agentconnect-cli 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentconnect/__init__.py +3 -0
- agentconnect/a2a_adapter/__init__.py +6 -0
- agentconnect/a2a_adapter/adapter.py +565 -0
- agentconnect/a2a_adapter/cards.py +149 -0
- agentconnect/a2a_adapter/convert.py +157 -0
- agentconnect/a2a_adapter/facade.py +560 -0
- agentconnect/a2a_adapter/seen.py +49 -0
- agentconnect/a2a_adapter/serve.py +67 -0
- agentconnect/adapter.py +319 -0
- agentconnect/adapters.py +52 -0
- agentconnect/apiclient.py +51 -0
- agentconnect/backoff.py +42 -0
- agentconnect/callback.py +314 -0
- agentconnect/cli.py +2324 -0
- agentconnect/cli_json.py +470 -0
- agentconnect/cli_support.py +444 -0
- agentconnect/clock.py +161 -0
- agentconnect/config.py +1213 -0
- agentconnect/daemon/__init__.py +41 -0
- agentconnect/deliveries.py +295 -0
- agentconnect/directory_client.py +255 -0
- agentconnect/failpoints.py +99 -0
- agentconnect/identity.py +111 -0
- agentconnect/isolation.py +345 -0
- agentconnect/keystore.py +279 -0
- agentconnect/limits.py +457 -0
- agentconnect/log.py +143 -0
- agentconnect/mcp/__init__.py +30 -0
- agentconnect/mcp/backend.py +405 -0
- agentconnect/mcp/server.py +382 -0
- agentconnect/mcp/session.py +373 -0
- agentconnect/mcp/tools.py +583 -0
- agentconnect/ownerfile.py +238 -0
- agentconnect/payloads.py +109 -0
- agentconnect/policy.py +1066 -0
- agentconnect/provision.py +239 -0
- agentconnect/py.typed +0 -0
- agentconnect/relay.py +405 -0
- agentconnect/runtime/__init__.py +46 -0
- agentconnect/runtime/agent.py +225 -0
- agentconnect/runtime/api.py +2043 -0
- agentconnect/runtime/engine.py +5359 -0
- agentconnect/runtime/errors.py +47 -0
- agentconnect/runtime/floor.py +649 -0
- agentconnect/runtime/groups.py +1351 -0
- agentconnect/runtime/knocks.py +1422 -0
- agentconnect/runtime/ledger.py +94 -0
- agentconnect/runtime/listing.py +1307 -0
- agentconnect/runtime/registry.py +53 -0
- agentconnect/runtime/runtime.py +158 -0
- agentconnect/runtime/sas.py +1002 -0
- agentconnect/sas_emoji.py +83 -0
- agentconnect/schemas.py +381 -0
- agentconnect/shapes.py +56 -0
- agentconnect/sockets.py +86 -0
- agentconnect/sse.py +234 -0
- agentconnect/store.py +3282 -0
- agentconnect/tasks.py +35 -0
- agentconnect/tokens.py +63 -0
- agentconnect/webhook.py +199 -0
- agentconnect_cli-0.1.0a1.dist-info/METADATA +103 -0
- agentconnect_cli-0.1.0a1.dist-info/RECORD +64 -0
- agentconnect_cli-0.1.0a1.dist-info/WHEEL +4 -0
- agentconnect_cli-0.1.0a1.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,1351 @@
|
|
|
1
|
+
"""Group membership in the runtime: who is in a group, how they leave or
|
|
2
|
+
are removed, who holds which role, and what each member is told (SPEC 9.2,
|
|
3
|
+
9.4, 11; docs/groups.md).
|
|
4
|
+
|
|
5
|
+
**Roles and the policy.** A role change, an ownership transfer and any other
|
|
6
|
+
policy change is a commit. It is queued as what it changes (a delta), not
|
|
7
|
+
as a whole policy: :func:`planned_policy` applies it to the policy in force
|
|
8
|
+
when the commit is built, so a rebuild after an epoch conflict keeps a
|
|
9
|
+
change another member committed meanwhile. Before queueing, the core's dry
|
|
10
|
+
run (``Client.check_policy_change``) says whether this instance may make
|
|
11
|
+
the change now; the core checks the commit again when it is built, and so
|
|
12
|
+
does every member that receives it. These operations are the human's
|
|
13
|
+
(``docs/api.md``): no agent-scope operation reaches them. After a commit:
|
|
14
|
+
an identity that may no longer invite closes its open invites for the
|
|
15
|
+
group, and the owner's instance drops departed identities from ``admins``
|
|
16
|
+
and ``floor_moderator`` (``groups.cleanup_departed_roles``).
|
|
17
|
+
|
|
18
|
+
**Leaves.** A leaving instance sends ``control.leave``, a notice only, then
|
|
19
|
+
a Remove proposal for its own leaf; membership changes only with the
|
|
20
|
+
commit that carries it. Which leaves are pending comes from the core
|
|
21
|
+
(``Client.pending_leaves``), never from memory, so a restart loses nothing:
|
|
22
|
+
the member with the lowest leaf that isn't itself leaving commits them, and
|
|
23
|
+
if they are still pending :data:`LEAVE_COMMIT_GRACE_SECS` after the first
|
|
24
|
+
proposal of the epoch arrived, the next one does, and so on. The check runs
|
|
25
|
+
when a proposal arrives, on every (re)connect and on a timer.
|
|
26
|
+
|
|
27
|
+
A leave can end without a commit, and the leaver can tell: when every
|
|
28
|
+
instance of the owner proposes a leave at once, the one with the lowest leaf
|
|
29
|
+
stays (SPEC 9.2), and a transfer of ownership committed without the new
|
|
30
|
+
owner's pending leave ends it with the epoch. Either way the instance is now
|
|
31
|
+
the owner's last one, which may not leave before it hands over ownership.
|
|
32
|
+
It keeps its leave and reports ``leave_blocked`` once; the leave goes on
|
|
33
|
+
once ownership has moved. A leave that no commit completes within
|
|
34
|
+
``groups.leave_timeout_secs`` ends locally (``left`` with ``confirmed:
|
|
35
|
+
false``), but only once the runtime has caught up with the relay, and never
|
|
36
|
+
for an instance of the owner, whose view can't tell whether it became the
|
|
37
|
+
owner's last one: it keeps its state and reports ``leave_blocked`` with
|
|
38
|
+
``timed_out``. A group that is out of sync commits and times out no leaves.
|
|
39
|
+
|
|
40
|
+
**Events** come from the commit's summary, not from a diff of cached state,
|
|
41
|
+
so a replay of the same journal entry (ADR-004) reports the same:
|
|
42
|
+
``member_joined``, ``instance_joined``, ``member_left`` (every instance it
|
|
43
|
+
had left on its own), ``member_removed``, ``instance_left``,
|
|
44
|
+
``instance_removed``, ``policy_changed`` and ``role_changed``, each with the
|
|
45
|
+
committer as ``by``.
|
|
46
|
+
|
|
47
|
+
**Updates.** Each instance renews its leaf key at most 24 hours or a number
|
|
48
|
+
of messages after its last commit. The number is drawn per instance and
|
|
49
|
+
epoch in [900, 1000] (SPEC 3.1 asks for 1000 at most), so the members of a
|
|
50
|
+
busy group don't all commit at once.
|
|
51
|
+
|
|
52
|
+
Peer commits are untrusted: the core checks each against the group policy
|
|
53
|
+
before it is merged, on every member. Nothing here decides who may do what.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
from __future__ import annotations
|
|
57
|
+
|
|
58
|
+
import asyncio
|
|
59
|
+
import contextlib
|
|
60
|
+
import hashlib
|
|
61
|
+
import json
|
|
62
|
+
import re
|
|
63
|
+
from collections.abc import Iterable
|
|
64
|
+
from typing import Any
|
|
65
|
+
|
|
66
|
+
import agentconnect_core as ac
|
|
67
|
+
from agentconnect import log
|
|
68
|
+
from agentconnect.backoff import Backoff
|
|
69
|
+
from agentconnect.relay import RelayError, b64, unb64
|
|
70
|
+
from agentconnect.runtime.engine import (
|
|
71
|
+
TRANSFER_TIMEOUT_SECS,
|
|
72
|
+
UNCHANGED,
|
|
73
|
+
UPDATE_AFTER_MESSAGES,
|
|
74
|
+
UPDATE_AFTER_SECS,
|
|
75
|
+
DaemonError,
|
|
76
|
+
Engine,
|
|
77
|
+
ac_identity_key,
|
|
78
|
+
canonical,
|
|
79
|
+
new_msg_id,
|
|
80
|
+
)
|
|
81
|
+
from agentconnect.store import GroupRow, MemberRow, OutboxRow
|
|
82
|
+
from agentconnect.tasks import cancel_and_wait
|
|
83
|
+
|
|
84
|
+
LOG = log.get("agentconnect.groups")
|
|
85
|
+
|
|
86
|
+
# How long a pending leave waits for the member before the next one commits it.
|
|
87
|
+
LEAVE_COMMIT_GRACE_SECS = 60
|
|
88
|
+
# The update threshold is drawn in [UPDATE_AFTER_MESSAGES - spread, UPDATE_AFTER_MESSAGES].
|
|
89
|
+
UPDATE_JITTER_MESSAGES = 100
|
|
90
|
+
UPDATE_THRESHOLD_INFO = "agentconnect/update-threshold"
|
|
91
|
+
INSTANCE_ID = re.compile(r"^[0-9a-f]{16}$")
|
|
92
|
+
MAX_INSTANCES = 50 # SPEC 8.3: the group size
|
|
93
|
+
# Relay answers to an invite revocation that may succeed later.
|
|
94
|
+
RELAY_RETRY = frozenset({"relay_unreachable", "rate_limited", "internal"})
|
|
95
|
+
OWNER_MUST_TRANSFER = "owner_must_transfer"
|
|
96
|
+
OWNER_INSTANCE = "owner_instance"
|
|
97
|
+
LEAVE_BLOCKED_WARNING = (
|
|
98
|
+
"This instance is the group owner's last one, so it can't leave yet (SPEC 9.2). "
|
|
99
|
+
"Hand ownership over to another member first; the leave goes on after that."
|
|
100
|
+
)
|
|
101
|
+
OWNER_LEAVE_WARNING = (
|
|
102
|
+
"No member committed this instance's leave, and an instance of the group's owner "
|
|
103
|
+
"never times out on its own: it may not have seen that it is the owner's last one. "
|
|
104
|
+
"Hand ownership over, or remove this instance from another instance of the owner."
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def update_threshold(group_id: str, epoch: int, instance_id: str) -> int:
|
|
109
|
+
"""Messages after which this instance renews its leaf key in this epoch:
|
|
110
|
+
drawn per instance and epoch in [900, 1000], so the members of a busy
|
|
111
|
+
group don't reach it together. Derived, so it survives a restart."""
|
|
112
|
+
digest = hashlib.sha256(canonical([UPDATE_THRESHOLD_INFO, group_id, epoch, instance_id]))
|
|
113
|
+
spread = int.from_bytes(digest.digest()[:8], "big") % (UPDATE_JITTER_MESSAGES + 1)
|
|
114
|
+
return UPDATE_AFTER_MESSAGES - UPDATE_JITTER_MESSAGES + spread
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# The fields of a policy change (POST /v0/groups/{gid}/policy, `group policy`).
|
|
118
|
+
POLICY_FIELDS = frozenset(
|
|
119
|
+
{
|
|
120
|
+
"require_verified",
|
|
121
|
+
"invite",
|
|
122
|
+
"remove",
|
|
123
|
+
"max_members",
|
|
124
|
+
"floor",
|
|
125
|
+
"floor_pinned",
|
|
126
|
+
"floor_moderator",
|
|
127
|
+
"max_agent_turns",
|
|
128
|
+
}
|
|
129
|
+
)
|
|
130
|
+
ROLES = ("admin", "member") # what a role change sets; the owner changes by transfer
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def policy_changes(fields: dict[str, Any]) -> dict[str, Any]:
|
|
134
|
+
"""The policy changes that the fields of a request ask for, as they sit
|
|
135
|
+
in the policy: ``max_agent_turns`` goes in ``loop_guard``, and setting
|
|
136
|
+
``floor`` pins it (SPEC 9.1: no switch to ``mention`` when a third
|
|
137
|
+
identity joins) unless the request says ``floor_pinned`` itself. Values
|
|
138
|
+
are the core's to check."""
|
|
139
|
+
unknown = sorted(set(fields) - POLICY_FIELDS)
|
|
140
|
+
if unknown:
|
|
141
|
+
raise DaemonError("bad_request", f"unknown policy field(s): {', '.join(unknown)}")
|
|
142
|
+
if not fields:
|
|
143
|
+
raise DaemonError("bad_request", "nothing to change")
|
|
144
|
+
changes = dict(fields)
|
|
145
|
+
if "max_agent_turns" in changes:
|
|
146
|
+
changes["loop_guard"] = {"max_agent_turns": changes.pop("max_agent_turns")}
|
|
147
|
+
if "floor" in changes:
|
|
148
|
+
changes.setdefault("floor_pinned", True)
|
|
149
|
+
return changes
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def may_invite(policy: dict[str, Any], identity: str) -> bool:
|
|
153
|
+
"""Whether the policy lets ``identity`` add members (SPEC 9.2)."""
|
|
154
|
+
invite = policy.get("invite")
|
|
155
|
+
admins = policy.get("admins")
|
|
156
|
+
return bool(
|
|
157
|
+
invite == "members"
|
|
158
|
+
or identity == policy.get("owner")
|
|
159
|
+
or (invite == "admins" and isinstance(admins, list) and identity in admins)
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def planned_policy(
|
|
164
|
+
current: dict[str, Any], intent: dict[str, Any], present: set[str]
|
|
165
|
+
) -> dict[str, Any]:
|
|
166
|
+
"""The policy a policy intent makes of ``current``, the policy in force
|
|
167
|
+
when its commit is built. ``present`` are the identities in the group.
|
|
168
|
+
|
|
169
|
+
* ``policy``: ``changes`` over the current fields. A row queued before
|
|
170
|
+
changes were deltas holds a whole ``policy``: only its
|
|
171
|
+
``require_verified`` is applied, the one field such a row could change.
|
|
172
|
+
* ``owner``: ``identity`` becomes the owner and leaves ``admins``; with
|
|
173
|
+
``keep_admin`` the old owner becomes an admin.
|
|
174
|
+
* ``role``: ``identity`` joins ``admins`` (``admin``) or leaves it
|
|
175
|
+
(``member``).
|
|
176
|
+
* ``drop_roles``: those of ``identities`` that are no longer in the group
|
|
177
|
+
leave ``admins`` and ``floor_moderator``: it only narrows roles."""
|
|
178
|
+
t = intent["type"]
|
|
179
|
+
if t == "policy":
|
|
180
|
+
changes = intent.get("changes")
|
|
181
|
+
legacy = intent.get("policy")
|
|
182
|
+
if changes is None and isinstance(legacy, dict) and "require_verified" in legacy:
|
|
183
|
+
# Queued before changes were deltas: the only change such a row
|
|
184
|
+
# could hold is require_verified (the API took nothing else).
|
|
185
|
+
# Never the rest of the policy it cached, such as the owner.
|
|
186
|
+
changes = {"require_verified": legacy["require_verified"]}
|
|
187
|
+
return {**current, **(changes if isinstance(changes, dict) else {})}
|
|
188
|
+
admins = [a for a in current.get("admins") or [] if isinstance(a, str)]
|
|
189
|
+
if t == "owner":
|
|
190
|
+
new_owner, old = str(intent["identity"]), current.get("owner")
|
|
191
|
+
if new_owner == old:
|
|
192
|
+
return current
|
|
193
|
+
admins = [a for a in admins if a != new_owner]
|
|
194
|
+
if intent.get("keep_admin", True) and isinstance(old, str) and old not in admins:
|
|
195
|
+
admins.append(old)
|
|
196
|
+
return {**current, "owner": new_owner, "admins": admins}
|
|
197
|
+
if t == "role":
|
|
198
|
+
who = str(intent["identity"])
|
|
199
|
+
if who == current.get("owner"):
|
|
200
|
+
raise DaemonError("bad_request", "the owner's role changes only with a transfer")
|
|
201
|
+
if intent.get("role") == "admin":
|
|
202
|
+
if who not in present:
|
|
203
|
+
# It left after the change was asked for (SPEC 9.2: roles
|
|
204
|
+
# name members); don't hand a role to an absent identity.
|
|
205
|
+
raise DaemonError("not_permitted", "that identity is no longer a member")
|
|
206
|
+
return {**current, "admins": admins if who in admins else [*admins, who]}
|
|
207
|
+
return {**current, "admins": [a for a in admins if a != who]}
|
|
208
|
+
if t == "drop_roles":
|
|
209
|
+
gone = {str(i) for i in intent.get("identities") or []} - present
|
|
210
|
+
out = {**current, "admins": [a for a in admins if a not in gone]}
|
|
211
|
+
if current.get("floor_moderator") in gone:
|
|
212
|
+
out["floor_moderator"] = None
|
|
213
|
+
return out
|
|
214
|
+
raise DaemonError("bad_intent", f"unknown policy intent {t}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _role_holders(policy: dict[str, Any]) -> set[str]:
|
|
218
|
+
"""The identities a policy gives a role below the owner's."""
|
|
219
|
+
admins = policy.get("admins")
|
|
220
|
+
holders: set[str] = {str(a) for a in admins} if isinstance(admins, list) else set()
|
|
221
|
+
moderator = policy.get("floor_moderator")
|
|
222
|
+
if isinstance(moderator, str):
|
|
223
|
+
holders.add(moderator)
|
|
224
|
+
return holders
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def role_of(policy: dict[str, Any], identity: str) -> str:
|
|
228
|
+
if identity == policy.get("owner"):
|
|
229
|
+
return "owner"
|
|
230
|
+
admins = policy.get("admins")
|
|
231
|
+
return "admin" if isinstance(admins, list) and identity in admins else "member"
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _leaf_key(m: dict[str, Any]) -> tuple[str, str]:
|
|
235
|
+
"""An instance as ``(instance_id, mailbox_id)``: a leaf that repeats an
|
|
236
|
+
instance id with another key (the core refuses one, SPEC 9.2) is never
|
|
237
|
+
taken for that instance."""
|
|
238
|
+
return str(m["instance_id_hex"]), str(m.get("mailbox_id", ""))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _by_identity(members: Iterable[dict[str, Any]]) -> dict[str, list[tuple[str, str]]]:
|
|
242
|
+
out: dict[str, list[tuple[str, str]]] = {}
|
|
243
|
+
for m in members:
|
|
244
|
+
out.setdefault(str(m["identity"]), []).append(_leaf_key(m))
|
|
245
|
+
return {identity: sorted(keys) for identity, keys in out.items()}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def membership_events(
|
|
249
|
+
summary: dict[str, Any], current: list[dict[str, Any]]
|
|
250
|
+
) -> list[tuple[str, dict[str, Any]]]:
|
|
251
|
+
"""The membership events of a merged commit, from its summary and the
|
|
252
|
+
members after it (``Client.members``), in a stable order. Instances are
|
|
253
|
+
told apart by id and mailbox (:func:`_leaf_key`).
|
|
254
|
+
|
|
255
|
+
An identity all of whose instances the commit added joined
|
|
256
|
+
(``member_joined``); a new instance of an identity already in the group
|
|
257
|
+
is ``instance_joined``. An identity with no instance left is
|
|
258
|
+
``member_left`` when every removed instance proposed its own leave
|
|
259
|
+
(``summary["left"]``), else ``member_removed``; one that keeps an
|
|
260
|
+
instance gets ``instance_left`` or ``instance_removed`` per instance."""
|
|
261
|
+
by = str(summary["committer"]["identity"])
|
|
262
|
+
after = _by_identity(current)
|
|
263
|
+
left = {_leaf_key(m) for m in summary.get("left") or []}
|
|
264
|
+
out: list[tuple[str, dict[str, Any]]] = []
|
|
265
|
+
for identity, keys in sorted(_by_identity(summary.get("added") or []).items()):
|
|
266
|
+
if set(after.get(identity, [])) <= set(keys):
|
|
267
|
+
ids = [i for i, _ in keys]
|
|
268
|
+
out.append(("member_joined", {"identity": identity, "instances": ids, "by": by}))
|
|
269
|
+
continue
|
|
270
|
+
out.extend(
|
|
271
|
+
("instance_joined", {"identity": identity, "instance_id": i, "by": by}) for i, _ in keys
|
|
272
|
+
)
|
|
273
|
+
for identity, keys in sorted(_by_identity(summary.get("removed") or []).items()):
|
|
274
|
+
if identity not in after:
|
|
275
|
+
kind = "member_left" if all(k in left for k in keys) else "member_removed"
|
|
276
|
+
out.append((kind, {"identity": identity, "by": by}))
|
|
277
|
+
continue
|
|
278
|
+
for k in keys:
|
|
279
|
+
kind = "instance_left" if k in left else "instance_removed"
|
|
280
|
+
out.append((kind, {"identity": identity, "instance_id": k[0], "by": by}))
|
|
281
|
+
return out
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def policy_events(
|
|
285
|
+
summary: dict[str, Any], current: dict[str, Any]
|
|
286
|
+
) -> list[tuple[str, dict[str, Any]]]:
|
|
287
|
+
"""``policy_changed`` and one ``role_changed`` per identity whose role
|
|
288
|
+
(owner, admin, member) the commit changed, or that became or stopped
|
|
289
|
+
being the ``floor_moderator`` (``moderator_from``, ``moderator_to``),
|
|
290
|
+
from the policies in the summary. ``current`` is the policy the core
|
|
291
|
+
holds now.
|
|
292
|
+
|
|
293
|
+
A summary journalled before the core reported the policies says
|
|
294
|
+
``policy_changed`` with neither: the policy it set and whether any role
|
|
295
|
+
changed are unknown. Its ``policy_changed`` carries the current policy,
|
|
296
|
+
``previous_policy: None`` and ``roles_known: False``, and no
|
|
297
|
+
``role_changed`` is made up for it."""
|
|
298
|
+
by = str(summary["committer"]["identity"])
|
|
299
|
+
previous_raw, policy_raw = summary.get("previous_policy"), summary.get("policy")
|
|
300
|
+
if previous_raw is None or policy_raw is None:
|
|
301
|
+
data = {"policy": current, "previous_policy": None, "roles_known": False, "by": by}
|
|
302
|
+
return [("policy_changed", data)]
|
|
303
|
+
previous, policy = json.loads(previous_raw), json.loads(policy_raw)
|
|
304
|
+
out: list[tuple[str, dict[str, Any]]] = [
|
|
305
|
+
(
|
|
306
|
+
"policy_changed",
|
|
307
|
+
{"policy": policy, "previous_policy": previous, "roles_known": True, "by": by},
|
|
308
|
+
)
|
|
309
|
+
]
|
|
310
|
+
named = {previous.get("owner"), policy.get("owner")}
|
|
311
|
+
for p in (previous, policy):
|
|
312
|
+
admins = p.get("admins")
|
|
313
|
+
named.update(admins if isinstance(admins, list) else [])
|
|
314
|
+
named.add(p.get("floor_moderator"))
|
|
315
|
+
for identity in sorted(str(i) for i in named if isinstance(i, str)):
|
|
316
|
+
before, now = role_of(previous, identity), role_of(policy, identity)
|
|
317
|
+
was = previous.get("floor_moderator") == identity
|
|
318
|
+
moderator = policy.get("floor_moderator") == identity
|
|
319
|
+
if before != now or was != moderator:
|
|
320
|
+
data = {
|
|
321
|
+
"identity": identity,
|
|
322
|
+
"from": before,
|
|
323
|
+
"to": now,
|
|
324
|
+
"moderator_from": was,
|
|
325
|
+
"moderator_to": moderator,
|
|
326
|
+
"by": by,
|
|
327
|
+
}
|
|
328
|
+
out.append(("role_changed", data))
|
|
329
|
+
return out
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class GroupService:
|
|
333
|
+
"""Membership for one :class:`Engine`: member state, commits' events,
|
|
334
|
+
leaves, removals and the owner's hand-over after a SAS mismatch."""
|
|
335
|
+
|
|
336
|
+
def __init__(self, engine: Engine) -> None:
|
|
337
|
+
self._engine = engine
|
|
338
|
+
self._wake = asyncio.Event()
|
|
339
|
+
# Between tries of an invite revocation the relay refused for now.
|
|
340
|
+
self._revoke_backoff = Backoff(base=1.0, cap=300.0, rng=engine.options.backoff_rng)
|
|
341
|
+
|
|
342
|
+
# -- plumbing -------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
@property
|
|
345
|
+
def _client(self) -> ac.Client:
|
|
346
|
+
return self._engine.client
|
|
347
|
+
|
|
348
|
+
def _now(self) -> int:
|
|
349
|
+
return self._engine.now()
|
|
350
|
+
|
|
351
|
+
def _kv(self, key: str) -> str:
|
|
352
|
+
return self._engine.store.kv_get(key) or ""
|
|
353
|
+
|
|
354
|
+
def wake(self) -> None:
|
|
355
|
+
self._wake.set()
|
|
356
|
+
|
|
357
|
+
# -- member state ---------------------------------------------------------
|
|
358
|
+
|
|
359
|
+
def refresh(self, group_id: str) -> None:
|
|
360
|
+
"""Re-read members, roles, policy and epoch from the core."""
|
|
361
|
+
e = self._engine
|
|
362
|
+
gid = unb64(group_id)
|
|
363
|
+
policy = json.loads(self._client.policy(gid))
|
|
364
|
+
epoch = self._client.epoch(gid)
|
|
365
|
+
# The epoch each instance was first seen in (G8), kept while it stays;
|
|
366
|
+
# by id and mailbox, so a leaf repeating an id never inherits it.
|
|
367
|
+
seen = {
|
|
368
|
+
(str(i["instance_id"]), str(i.get("mailbox_id", ""))): int(i.get("joined_epoch", 0))
|
|
369
|
+
for m in e.store.members(group_id)
|
|
370
|
+
for i in m.instances
|
|
371
|
+
}
|
|
372
|
+
grouped: dict[str, list[dict[str, Any]]] = {}
|
|
373
|
+
for m in self._client.members(gid):
|
|
374
|
+
grouped.setdefault(m["identity"], []).append(
|
|
375
|
+
{
|
|
376
|
+
"instance_id": m["instance_id_hex"],
|
|
377
|
+
"mailbox_id": m["mailbox_id"],
|
|
378
|
+
"leaf_index": m["leaf_index"],
|
|
379
|
+
"joined_epoch": seen.get(_leaf_key(m), epoch),
|
|
380
|
+
}
|
|
381
|
+
)
|
|
382
|
+
members = {
|
|
383
|
+
identity: (role_of(policy, identity), instances)
|
|
384
|
+
for identity, instances in grouped.items()
|
|
385
|
+
}
|
|
386
|
+
e.store.set_members(group_id, members)
|
|
387
|
+
e.store.update_group(group_id, policy=policy, epoch=epoch)
|
|
388
|
+
|
|
389
|
+
def trust_state(self, group_id: str) -> dict[str, Any]:
|
|
390
|
+
members = self._engine.store.members(group_id)
|
|
391
|
+
others = [m for m in members if m.identity != self._engine.me]
|
|
392
|
+
verified = sum(1 for m in others if m.verified)
|
|
393
|
+
return {
|
|
394
|
+
"self": "verified" if others and verified == len(others) else "unverified",
|
|
395
|
+
"verified_members": verified,
|
|
396
|
+
"members": len(members),
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
def update_due(self, group: GroupRow) -> bool:
|
|
400
|
+
"""SPEC 3.1: renew this instance's leaf a day, or a drawn number of
|
|
401
|
+
messages (:func:`update_threshold`), after its last commit."""
|
|
402
|
+
threshold = update_threshold(
|
|
403
|
+
group.group_id, group.epoch, self._engine.instance.instance_id_hex
|
|
404
|
+
)
|
|
405
|
+
return (
|
|
406
|
+
self._now() - group.last_own_commit_at >= UPDATE_AFTER_SECS
|
|
407
|
+
or group.msgs_since_commit >= threshold
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
def _own_leaf(self, members: list[dict[str, Any]]) -> int | None:
|
|
411
|
+
mailbox = self._engine.mailbox
|
|
412
|
+
return next((int(m["leaf_index"]) for m in members if m["mailbox_id"] == mailbox), None)
|
|
413
|
+
|
|
414
|
+
def owner_must_transfer(self, gid: bytes) -> bool:
|
|
415
|
+
"""Whether this instance is the owner's last one, counting the
|
|
416
|
+
instances whose leave is pending as gone: it may not propose a leave
|
|
417
|
+
until it hands ownership over (SPEC 9.2)."""
|
|
418
|
+
if json.loads(self._client.policy(gid)).get("owner") != self._engine.me:
|
|
419
|
+
return False
|
|
420
|
+
pending = {int(m["leaf_index"]) for m in self._client.pending_leaves(gid)}
|
|
421
|
+
members = self._client.members(gid)
|
|
422
|
+
own = self._own_leaf(members)
|
|
423
|
+
return not any(
|
|
424
|
+
m["identity"] == self._engine.me
|
|
425
|
+
and int(m["leaf_index"]) != own
|
|
426
|
+
and int(m["leaf_index"]) not in pending
|
|
427
|
+
for m in members
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
# -- roles and the policy ---------------------------------------------------
|
|
431
|
+
|
|
432
|
+
def _policy(self, gid: bytes) -> dict[str, Any]:
|
|
433
|
+
return dict(json.loads(self._client.policy(gid)))
|
|
434
|
+
|
|
435
|
+
def _identities(self, gid: bytes) -> set[str]:
|
|
436
|
+
return {str(m["identity"]) for m in self._client.members(gid)}
|
|
437
|
+
|
|
438
|
+
@staticmethod
|
|
439
|
+
def policy_changes_of(fields: dict[str, Any]) -> dict[str, Any]:
|
|
440
|
+
""":func:`policy_changes` (for the engine, which can't import it)."""
|
|
441
|
+
return policy_changes(fields)
|
|
442
|
+
|
|
443
|
+
def initial_policy(self, fields: dict[str, Any]) -> bytes:
|
|
444
|
+
"""A new group's policy: the SPEC 9.1 defaults with this identity as
|
|
445
|
+
the owner, and the changes ``fields`` ask for (:func:`policy_changes`)."""
|
|
446
|
+
defaults = json.loads(ac.default_group_policy(self._engine.me))
|
|
447
|
+
return canonical({**defaults, **policy_changes(fields)})
|
|
448
|
+
|
|
449
|
+
def change_policy(self, group_id: str, intent: dict[str, Any]) -> str:
|
|
450
|
+
"""Queue the commit of a policy intent (:func:`planned_policy`):
|
|
451
|
+
``committing``, then ``policy_changed`` or ``commit_failed``; or
|
|
452
|
+
``unchanged`` when the policy already says so. The core's dry run of
|
|
453
|
+
the change, as it would be made now, refuses one this instance may
|
|
454
|
+
not make with ``not_permitted`` before anything is queued."""
|
|
455
|
+
e = self._engine
|
|
456
|
+
group = e._group_or_raise(group_id)
|
|
457
|
+
gid = unb64(group.group_id)
|
|
458
|
+
current = self._policy(gid)
|
|
459
|
+
new = planned_policy(current, intent, self._identities(gid))
|
|
460
|
+
if new == current:
|
|
461
|
+
return UNCHANGED
|
|
462
|
+
self._dry_run(gid, new)
|
|
463
|
+
e._enqueue("commit", group.group_id, {"intent": intent})
|
|
464
|
+
return "committing"
|
|
465
|
+
|
|
466
|
+
def _dry_run(self, gid: bytes, policy: dict[str, Any]) -> None:
|
|
467
|
+
try:
|
|
468
|
+
self._client.check_policy_change(gid, canonical(policy))
|
|
469
|
+
except ac.AgentConnectError as err:
|
|
470
|
+
kind = getattr(err, "kind", "other")
|
|
471
|
+
if kind == "commit_pending":
|
|
472
|
+
# An own commit is under way; the policy passed validation
|
|
473
|
+
# (checked first). The core decides again when this one is built.
|
|
474
|
+
return
|
|
475
|
+
if kind in ("unauthorized", "not_permitted"):
|
|
476
|
+
raise DaemonError("not_permitted", str(err)) from None
|
|
477
|
+
if kind in ("malformed", "unsupported_version"):
|
|
478
|
+
raise DaemonError("bad_request", str(err)) from None
|
|
479
|
+
raise
|
|
480
|
+
|
|
481
|
+
def build_policy_commit(self, gid: bytes, intent: dict[str, Any]) -> dict[str, Any]:
|
|
482
|
+
"""Build a policy intent's commit against the policy in force now; a
|
|
483
|
+
change it already has fails as :data:`UNCHANGED` (nothing to tell)."""
|
|
484
|
+
current = self._policy(gid)
|
|
485
|
+
present = self._identities(gid)
|
|
486
|
+
if intent["type"] == "drop_roles":
|
|
487
|
+
# One that came back since it departed is new: it doesn't keep
|
|
488
|
+
# the role (only a later grant gives it one again).
|
|
489
|
+
present -= self._returned(b64(gid), set(intent.get("identities") or []))
|
|
490
|
+
new = planned_policy(current, intent, present)
|
|
491
|
+
if new == current:
|
|
492
|
+
raise DaemonError(UNCHANGED, "the policy already says so")
|
|
493
|
+
return self._client.set_policy(gid, canonical(new))
|
|
494
|
+
|
|
495
|
+
def _departed(self, group_id: str) -> dict[str, int]:
|
|
496
|
+
"""Identities that held a role when a commit took their last
|
|
497
|
+
instance, each with the epoch that commit started (kv, per group)."""
|
|
498
|
+
raw = json.loads(self._kv(f"departed_roles/{group_id}") or "{}")
|
|
499
|
+
return {str(k): int(v) for k, v in raw.items()} if isinstance(raw, dict) else {}
|
|
500
|
+
|
|
501
|
+
def _returned(self, group_id: str, identities: set[str]) -> set[str]:
|
|
502
|
+
"""Those of ``identities`` recorded as departed whose every instance
|
|
503
|
+
joined after the departure: they came back."""
|
|
504
|
+
departed = self._departed(group_id)
|
|
505
|
+
out = set()
|
|
506
|
+
for m in self._engine.store.members(group_id):
|
|
507
|
+
since = departed.get(m.identity)
|
|
508
|
+
if (
|
|
509
|
+
m.identity in identities
|
|
510
|
+
and since is not None
|
|
511
|
+
and m.instances
|
|
512
|
+
and all(int(i.get("joined_epoch", 0)) > since for i in m.instances)
|
|
513
|
+
):
|
|
514
|
+
out.add(m.identity)
|
|
515
|
+
return out
|
|
516
|
+
|
|
517
|
+
def set_role(self, group_id: str, identity: str, role: str) -> str:
|
|
518
|
+
"""Make a member an admin or a plain member (the owner's role changes
|
|
519
|
+
with :meth:`transfer_to`). Only the owner may, which the core checks."""
|
|
520
|
+
if role not in ROLES:
|
|
521
|
+
raise DaemonError("bad_request", "role must be admin or member")
|
|
522
|
+
gid = unb64(self._engine._group_or_raise(group_id).group_id)
|
|
523
|
+
if identity not in self._identities(gid):
|
|
524
|
+
raise DaemonError("not_found", "no such member")
|
|
525
|
+
current = self._policy(gid)
|
|
526
|
+
if identity == current.get("owner"):
|
|
527
|
+
msg = "that member is the owner: hand ownership over to change its role"
|
|
528
|
+
raise DaemonError("bad_request", msg)
|
|
529
|
+
grant = {"type": "role", "identity": identity, "role": role}
|
|
530
|
+
if (
|
|
531
|
+
role == "admin"
|
|
532
|
+
and identity in self._departed(group_id)
|
|
533
|
+
and identity in _role_holders(current)
|
|
534
|
+
):
|
|
535
|
+
# It departed and came back, and the cleanup that drops its role
|
|
536
|
+
# hasn't landed: answering "unchanged" would leave the role to
|
|
537
|
+
# that cleanup. Drop the stale role, then grant it: every member
|
|
538
|
+
# reads the second commit as a grant and forgets the departure.
|
|
539
|
+
without = {
|
|
540
|
+
**current,
|
|
541
|
+
"admins": [a for a in current.get("admins") or [] if a != identity],
|
|
542
|
+
}
|
|
543
|
+
self._dry_run(gid, without)
|
|
544
|
+
e = self._engine
|
|
545
|
+
e._enqueue(
|
|
546
|
+
"commit", group_id, {"intent": {"type": "drop_roles", "identities": [identity]}}
|
|
547
|
+
)
|
|
548
|
+
e._enqueue("commit", group_id, {"intent": grant})
|
|
549
|
+
return "committing"
|
|
550
|
+
return self.change_policy(group_id, grant)
|
|
551
|
+
|
|
552
|
+
def transfer_to(self, group_id: str, identity: str, *, keep_admin: bool = True) -> str:
|
|
553
|
+
"""Hand ownership over to another member (SPEC 9.2); with
|
|
554
|
+
``keep_admin`` this identity stays an admin. A leave of this
|
|
555
|
+
instance that waits for it goes on once it lands."""
|
|
556
|
+
gid = unb64(self._engine._group_or_raise(group_id).group_id)
|
|
557
|
+
if identity not in self._identities(gid):
|
|
558
|
+
raise DaemonError("not_found", "no such member")
|
|
559
|
+
if identity == self._policy(gid).get("owner"):
|
|
560
|
+
raise DaemonError("bad_request", "that member is the owner already")
|
|
561
|
+
intent = {"type": "owner", "identity": identity, "keep_admin": keep_admin}
|
|
562
|
+
return self.change_policy(group_id, intent)
|
|
563
|
+
|
|
564
|
+
def room(self, group_id: str) -> int | None:
|
|
565
|
+
"""How many more instances the group's ``max_members`` allows, counting
|
|
566
|
+
leaves already pending as gone and the Adds under way as in; ``None``
|
|
567
|
+
for a group that isn't active."""
|
|
568
|
+
e = self._engine
|
|
569
|
+
group = e.store.group(group_id)
|
|
570
|
+
if group is None or group.status != "active":
|
|
571
|
+
return None
|
|
572
|
+
gid = unb64(group_id)
|
|
573
|
+
members = self._client.members(gid)
|
|
574
|
+
staying = len(members) - len(self._client.pending_leaves(gid))
|
|
575
|
+
limit = int(self._policy(gid).get("max_members", MAX_INSTANCES))
|
|
576
|
+
# An Add that merged already counts in the members until its Welcome
|
|
577
|
+
# went out and its approval is done: count it once.
|
|
578
|
+
leaves = {str(m["mailbox_id"]) for m in members}
|
|
579
|
+
adding = [mb for mb in e.store.adding_to(group_id) if mb not in leaves]
|
|
580
|
+
return limit - staying - len(adding)
|
|
581
|
+
|
|
582
|
+
def _follow_policy(self, group_id: str, summary: dict[str, Any]) -> None:
|
|
583
|
+
"""After a merged commit: close this identity's open invites for the
|
|
584
|
+
group once the policy no longer lets it invite (SPEC 9.2: the core
|
|
585
|
+
would refuse their Adds), and on the owner's first instance queue a
|
|
586
|
+
commit that drops identities the commit removed from ``admins`` and
|
|
587
|
+
``floor_moderator`` (SPEC 9.2, "Departed roles")."""
|
|
588
|
+
e = self._engine
|
|
589
|
+
gid = unb64(group_id)
|
|
590
|
+
policy = self._policy(gid)
|
|
591
|
+
if not may_invite(policy, e.me):
|
|
592
|
+
self.close_invites(group_id, keep_replica=True)
|
|
593
|
+
group = e.store.group(group_id)
|
|
594
|
+
out_of_sync = group is not None and group.out_of_sync # its commits can't land
|
|
595
|
+
if not e.config.groups_cleanup_departed_roles or self.leaving(group_id) or out_of_sync:
|
|
596
|
+
return
|
|
597
|
+
self._record_departures(group_id, summary, policy)
|
|
598
|
+
members = self._client.members(gid)
|
|
599
|
+
owner = policy.get("owner")
|
|
600
|
+
owner_leaves = sorted(int(m["leaf_index"]) for m in members if m["identity"] == owner)
|
|
601
|
+
if not owner_leaves or self._own_leaf(members) != owner_leaves[0]:
|
|
602
|
+
return
|
|
603
|
+
present = {str(m["identity"]) for m in members}
|
|
604
|
+
departed = set(self._departed(group_id))
|
|
605
|
+
gone = sorted((departed - present) | self._returned(group_id, departed))
|
|
606
|
+
if gone and not self._drop_queued(group_id):
|
|
607
|
+
LOG.info("departed_roles", group=log.hid(group_id), count=len(gone))
|
|
608
|
+
e._enqueue("commit", group_id, {"intent": {"type": "drop_roles", "identities": gone}})
|
|
609
|
+
|
|
610
|
+
def _record_departures(
|
|
611
|
+
self, group_id: str, summary: dict[str, Any], policy: dict[str, Any]
|
|
612
|
+
) -> None:
|
|
613
|
+
"""Keep the departed-role record (:meth:`_departed`) current: an
|
|
614
|
+
identity that held a role and lost its last instance in this commit
|
|
615
|
+
is recorded with the commit's epoch; one that holds no role any
|
|
616
|
+
more, or was granted one again in this commit, is forgotten."""
|
|
617
|
+
e = self._engine
|
|
618
|
+
roles = _role_holders(policy)
|
|
619
|
+
present = self._identities(unb64(group_id))
|
|
620
|
+
removed = {str(m["identity"]) for m in summary.get("removed") or []}
|
|
621
|
+
before = json.loads(summary["previous_policy"]) if summary.get("previous_policy") else None
|
|
622
|
+
granted: set[str] = set()
|
|
623
|
+
if before is not None:
|
|
624
|
+
granted = roles - _role_holders(before)
|
|
625
|
+
record = {
|
|
626
|
+
k: v for k, v in self._departed(group_id).items() if k in roles and k not in granted
|
|
627
|
+
}
|
|
628
|
+
for identity in (removed - present) & roles:
|
|
629
|
+
record[identity] = int(summary["epoch"])
|
|
630
|
+
key = f"departed_roles/{group_id}"
|
|
631
|
+
if record != self._departed(group_id):
|
|
632
|
+
e.store.kv_set(key, json.dumps(record, sort_keys=True) if record else "")
|
|
633
|
+
|
|
634
|
+
def _drop_queued(self, group_id: str) -> bool:
|
|
635
|
+
return any(
|
|
636
|
+
r.kind == "commit"
|
|
637
|
+
and r.group_id == group_id
|
|
638
|
+
and "env" not in r.payload
|
|
639
|
+
and (r.payload.get("intent") or {}).get("type") == "drop_roles"
|
|
640
|
+
for r in self._engine.store.outbox()
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
def close_invites(self, group_id: str, *, keep_replica: bool = False) -> None:
|
|
644
|
+
"""Close this instance's open invites for the group, because it may
|
|
645
|
+
no longer invite or is no longer in the group. They are ``revoking``
|
|
646
|
+
at once, so a join request for one is ignored, and the timer revokes
|
|
647
|
+
them at the relay (:meth:`revoke_closed_invites`), so a joiner that
|
|
648
|
+
redeemed one hears it was used instead of waiting. ``keep_replica``
|
|
649
|
+
keeps the replica invites open: adding an instance of one's own
|
|
650
|
+
identity needs no `invite` permission (SPEC 9.2)."""
|
|
651
|
+
e = self._engine
|
|
652
|
+
ids = [
|
|
653
|
+
str(i["invite_id"])
|
|
654
|
+
for i in e.store.invites("open")
|
|
655
|
+
if i["group_id"] == group_id and not (keep_replica and i["replica"])
|
|
656
|
+
]
|
|
657
|
+
if not ids:
|
|
658
|
+
return
|
|
659
|
+
for invite_id in ids:
|
|
660
|
+
e.store.set_invite_status(invite_id, "revoking")
|
|
661
|
+
LOG.info("invites_closed", group=log.hid(group_id), count=len(ids))
|
|
662
|
+
e._defer(self.wake)
|
|
663
|
+
|
|
664
|
+
async def revoke_closed_invites(self) -> float | None:
|
|
665
|
+
"""Drop the secrets of the invites :meth:`close_invites` closed and
|
|
666
|
+
revoke them at the relay. One the relay refused for now (while this
|
|
667
|
+
runtime is connected) stays ``revoking``: returns when to try again,
|
|
668
|
+
after a backoff. Offline, the reconnect wakes the timer."""
|
|
669
|
+
e = self._engine
|
|
670
|
+
retry = False
|
|
671
|
+
for invite in e.store.invites("revoking"):
|
|
672
|
+
invite_id = str(invite["invite_id"])
|
|
673
|
+
e._forget_secret(f"invite/{invite_id}")
|
|
674
|
+
e._forget_invite_requests(invite_id)
|
|
675
|
+
if not e.link.connected:
|
|
676
|
+
continue
|
|
677
|
+
try:
|
|
678
|
+
await e.rest.revoke_invite(invite_id)
|
|
679
|
+
except RelayError as err:
|
|
680
|
+
if err.code in RELAY_RETRY:
|
|
681
|
+
LOG.info("invite_revoke_retry", code=err.code)
|
|
682
|
+
retry = True
|
|
683
|
+
continue
|
|
684
|
+
e.store.set_invite_status(invite_id, "revoked")
|
|
685
|
+
if not retry:
|
|
686
|
+
self._revoke_backoff.reset()
|
|
687
|
+
return None
|
|
688
|
+
return float(self._now() + self._revoke_backoff.next())
|
|
689
|
+
|
|
690
|
+
# -- commits --------------------------------------------------------------
|
|
691
|
+
|
|
692
|
+
async def after_commit(
|
|
693
|
+
self, group_id: str, summary: dict[str, Any], env: dict[str, Any] | None, own: bool
|
|
694
|
+
) -> None:
|
|
695
|
+
"""Bookkeeping after any merged commit, own or another member's.
|
|
696
|
+
|
|
697
|
+
The cached members, roles, policy and epoch are re-read from the core
|
|
698
|
+
first. The events come from the commit's summary (see the module
|
|
699
|
+
docs), so a replay of the same journal entry emits the same ones."""
|
|
700
|
+
e = self._engine
|
|
701
|
+
gid = unb64(group_id)
|
|
702
|
+
self.refresh(group_id)
|
|
703
|
+
if env is not None:
|
|
704
|
+
# Shapes were checked before processing (_envelope_problem).
|
|
705
|
+
hinted = env.get("routing") or {}
|
|
706
|
+
expected = summary["routing"]
|
|
707
|
+
if sorted(hinted.get("add", [])) != sorted(expected["add"]) or sorted(
|
|
708
|
+
hinted.get("remove", [])
|
|
709
|
+
) != sorted(expected["remove"]):
|
|
710
|
+
await e.emit(
|
|
711
|
+
group_id,
|
|
712
|
+
"relay_misbehaviour",
|
|
713
|
+
reason="routing_mismatch",
|
|
714
|
+
epoch=summary["epoch"],
|
|
715
|
+
)
|
|
716
|
+
if env.get("epoch") is not None and env["epoch"] + 1 != summary["epoch"]:
|
|
717
|
+
await e.emit(
|
|
718
|
+
group_id, "relay_misbehaviour", reason="epoch_mismatch", epoch=summary["epoch"]
|
|
719
|
+
)
|
|
720
|
+
if own:
|
|
721
|
+
e.store.update_group(group_id, last_own_commit_at=self._now(), msgs_since_commit=0)
|
|
722
|
+
joined: list[str] = []
|
|
723
|
+
for kind, data in membership_events(summary, self._client.members(gid)):
|
|
724
|
+
await e.emit(group_id, kind, **data)
|
|
725
|
+
if kind == "member_joined":
|
|
726
|
+
await e._send_card(group_id, [data["identity"]])
|
|
727
|
+
if kind in ("member_joined", "instance_joined"):
|
|
728
|
+
joined.append(str(data["identity"]))
|
|
729
|
+
if summary.get("policy_changed"):
|
|
730
|
+
current = json.loads(self._client.policy(gid))
|
|
731
|
+
for kind, data in policy_events(summary, current):
|
|
732
|
+
await e.emit(group_id, kind, **data)
|
|
733
|
+
await e.floor.after_commit(group_id, summary, joined)
|
|
734
|
+
self._follow_policy(group_id, summary)
|
|
735
|
+
trust = self.trust_state(group_id)
|
|
736
|
+
await e.emit(
|
|
737
|
+
group_id,
|
|
738
|
+
"epoch_changed",
|
|
739
|
+
epoch=summary["epoch"],
|
|
740
|
+
verified=f"{trust['verified_members']} of {trust['members'] - 1} verified",
|
|
741
|
+
)
|
|
742
|
+
# SPEC 11.3: runs of the earlier epoch end. A commit of ours that
|
|
743
|
+
# carried another member's leave is that member's doing.
|
|
744
|
+
committer = str(summary["committer"]["identity"])
|
|
745
|
+
others = [str(m["identity"]) for m in summary.get("left") or [] if m["identity"] != e.me]
|
|
746
|
+
e.sas.on_epoch_changed(group_id, others[0] if committer == e.me and others else committer)
|
|
747
|
+
if self.leaving(group_id):
|
|
748
|
+
await self._propose_again(group_id)
|
|
749
|
+
e._defer(self.wake)
|
|
750
|
+
|
|
751
|
+
async def _propose_again(self, group_id: str) -> None:
|
|
752
|
+
"""After a commit while leaving: the epoch ended this instance's
|
|
753
|
+
proposal, so it proposes again, unless it is now the owner's last
|
|
754
|
+
instance."""
|
|
755
|
+
e = self._engine
|
|
756
|
+
if self.owner_must_transfer(unb64(group_id)):
|
|
757
|
+
await self._blocked(group_id)
|
|
758
|
+
return
|
|
759
|
+
if self._kv(f"leave_blocked/{group_id}"):
|
|
760
|
+
# Unblocked: the timeout counts from this proposal.
|
|
761
|
+
e.store.kv_set(f"leave_blocked/{group_id}", "")
|
|
762
|
+
e.store.kv_set(f"leaving_since/{group_id}", str(self._now()))
|
|
763
|
+
if not self._proposal_queued(group_id):
|
|
764
|
+
# One queued row proposes in whatever epoch it runs in.
|
|
765
|
+
e._enqueue("proposal", group_id, {})
|
|
766
|
+
|
|
767
|
+
async def on_removed(
|
|
768
|
+
self, group_id: str, summary: dict[str, Any], *, legacy: bool = False
|
|
769
|
+
) -> None:
|
|
770
|
+
"""A commit removed this instance: a leave if the commit carried its
|
|
771
|
+
own leave proposal, otherwise a removal, as every other member reads
|
|
772
|
+
the same summary. Only a summary replayed from a journal entry an
|
|
773
|
+
older daemon wrote (``legacy``) may predate the core's ``left``, which
|
|
774
|
+
then reads back empty: there, a leaving instance reports a leave."""
|
|
775
|
+
mailbox = self._engine.mailbox
|
|
776
|
+
by = str(summary["committer"]["identity"])
|
|
777
|
+
own_leave = any(m["mailbox_id"] == mailbox for m in summary.get("left") or [])
|
|
778
|
+
if own_leave or (legacy and self.leaving(group_id)):
|
|
779
|
+
await self.gone(group_id, "left", confirmed=True, by=by)
|
|
780
|
+
else:
|
|
781
|
+
await self.gone(group_id, "removed", by=by)
|
|
782
|
+
|
|
783
|
+
async def gone(
|
|
784
|
+
self, group_id: str, kind: str, *, cause: str | None = None, **data: Any
|
|
785
|
+
) -> None:
|
|
786
|
+
"""This instance is no longer in the group (removed or left).
|
|
787
|
+
|
|
788
|
+
The daemon's record goes first and the core's state after it (a
|
|
789
|
+
crash in between is cleaned up by ``Engine._recover``): the other way
|
|
790
|
+
round, a crash would leave an "active" group with no MLS state."""
|
|
791
|
+
e = self._engine
|
|
792
|
+
gid = unb64(group_id)
|
|
793
|
+
e.store.update_group(group_id, status=kind)
|
|
794
|
+
await e._drop_group_holds(group_id)
|
|
795
|
+
e.store.delete_group_data(group_id)
|
|
796
|
+
self._end_leave(group_id)
|
|
797
|
+
self.close_invites(group_id)
|
|
798
|
+
await e.emit(group_id, kind, cause=cause, **data)
|
|
799
|
+
|
|
800
|
+
def forget() -> None:
|
|
801
|
+
e.safety.forget_group(group_id)
|
|
802
|
+
with contextlib.suppress(ac.AgentConnectError, ac.CoreError):
|
|
803
|
+
self._client.delete_group(gid)
|
|
804
|
+
|
|
805
|
+
e._defer(forget)
|
|
806
|
+
|
|
807
|
+
# -- leaves ---------------------------------------------------------------
|
|
808
|
+
|
|
809
|
+
def leaving(self, group_id: str) -> bool:
|
|
810
|
+
return self._kv(f"leaving/{group_id}") == "1"
|
|
811
|
+
|
|
812
|
+
def leave_state(self, group_id: str) -> dict[str, Any] | None:
|
|
813
|
+
"""``{since, blocked}`` while this instance is leaving the group."""
|
|
814
|
+
if not self.leaving(group_id):
|
|
815
|
+
return None
|
|
816
|
+
return {
|
|
817
|
+
"since": int(self._kv(f"leaving_since/{group_id}") or "0"),
|
|
818
|
+
"blocked": self._kv(f"leave_blocked/{group_id}") != "",
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
def _end_leave(self, group_id: str) -> None:
|
|
822
|
+
store = self._engine.store
|
|
823
|
+
store.kv_set(f"leaving/{group_id}", "0")
|
|
824
|
+
for key in ("leaving_since", "leave_blocked", "leave_seen"):
|
|
825
|
+
store.kv_set(f"{key}/{group_id}", "")
|
|
826
|
+
|
|
827
|
+
def recover(self, group: GroupRow) -> None:
|
|
828
|
+
"""After a restart: a leave recorded before its start time was kept
|
|
829
|
+
starts its timeout now."""
|
|
830
|
+
if self.leaving(group.group_id) and not self._kv(f"leaving_since/{group.group_id}"):
|
|
831
|
+
self._engine.store.kv_set(f"leaving_since/{group.group_id}", str(self._now()))
|
|
832
|
+
|
|
833
|
+
async def leave(self, group_id: str) -> str:
|
|
834
|
+
"""Leave the group (SPEC 9.2): a sole member deletes its state; the
|
|
835
|
+
owner's last instance must hand ownership over first; anyone else
|
|
836
|
+
sends ``control.leave`` and its leave proposal, and leaves once a
|
|
837
|
+
member commits it."""
|
|
838
|
+
e = self._engine
|
|
839
|
+
e._group_or_raise(group_id)
|
|
840
|
+
gid = unb64(group_id)
|
|
841
|
+
if len(self._client.members(gid)) == 1:
|
|
842
|
+
# Sole member: delete, recorded with its event in one transaction.
|
|
843
|
+
await self._gone_atomically("leave", group_id, "left", confirmed=True)
|
|
844
|
+
return "left"
|
|
845
|
+
if self.owner_must_transfer(gid):
|
|
846
|
+
raise DaemonError("forbidden", "transfer ownership before leaving (SPEC 9.2)")
|
|
847
|
+
if self.leaving(group_id):
|
|
848
|
+
return "leaving"
|
|
849
|
+
with e.store.tx():
|
|
850
|
+
e.store.kv_set(f"leaving/{group_id}", "1")
|
|
851
|
+
e.store.kv_set(f"leaving_since/{group_id}", str(self._now()))
|
|
852
|
+
e.store.kv_set(f"leave_blocked/{group_id}", "")
|
|
853
|
+
# SPEC 7.3: a notice right before the proposal; it changes nothing.
|
|
854
|
+
e.send_message(group_id, {}, kind="control.leave", internal=True)
|
|
855
|
+
e._enqueue("proposal", group_id, {})
|
|
856
|
+
self.wake()
|
|
857
|
+
return "leaving"
|
|
858
|
+
|
|
859
|
+
async def _gone_atomically(
|
|
860
|
+
self, path: str, group_id: str, kind: str, *, cause: str | None = None, **data: Any
|
|
861
|
+
) -> None:
|
|
862
|
+
""":meth:`gone` in one daemon transaction (failpoints ``<path>.in_tx``
|
|
863
|
+
and ``<path>.after_tx``): a crash leaves either the group and its leave
|
|
864
|
+
as they were, or the group gone and its event recorded, which the
|
|
865
|
+
adapter is told of after a restart."""
|
|
866
|
+
|
|
867
|
+
async def body() -> str:
|
|
868
|
+
await self.gone(group_id, kind, cause=cause, **data)
|
|
869
|
+
return kind
|
|
870
|
+
|
|
871
|
+
await self._engine._apply(path, body)
|
|
872
|
+
|
|
873
|
+
async def _blocked(
|
|
874
|
+
self, group_id: str, *, timed_out: bool = False, reason: str = OWNER_MUST_TRANSFER
|
|
875
|
+
) -> None:
|
|
876
|
+
"""Tell the human (once, and once more at the timeout) that this
|
|
877
|
+
leave waits for an ownership transfer."""
|
|
878
|
+
key = f"leave_blocked/{group_id}"
|
|
879
|
+
state, want = self._kv(key), "timeout" if timed_out else "blocked"
|
|
880
|
+
if state in (want, "timeout"):
|
|
881
|
+
return
|
|
882
|
+
self._engine.store.kv_set(key, want)
|
|
883
|
+
LOG.info("leave_blocked", group=log.hid(group_id), timed_out=timed_out)
|
|
884
|
+
since = self._kv(f"leaving_since/{group_id}")
|
|
885
|
+
await self._engine.emit(
|
|
886
|
+
group_id,
|
|
887
|
+
"leave_blocked",
|
|
888
|
+
cause=f"leave_blocked/{since}/{want}",
|
|
889
|
+
reason=reason,
|
|
890
|
+
timed_out=timed_out,
|
|
891
|
+
warning=LEAVE_BLOCKED_WARNING if reason == OWNER_MUST_TRANSFER else OWNER_LEAVE_WARNING,
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
async def run_proposal(self, row: OutboxRow, group: GroupRow) -> str:
|
|
895
|
+
"""Send this instance's leave proposal (an outbox row)."""
|
|
896
|
+
e = self._engine
|
|
897
|
+
gid = unb64(group.group_id)
|
|
898
|
+
payload = dict(row.payload)
|
|
899
|
+
if "env" in payload and int(payload["env"]["epoch"]) != int(self._client.epoch(gid)):
|
|
900
|
+
payload.pop("env") # built in an epoch that ended: propose in this one
|
|
901
|
+
if "env" not in payload:
|
|
902
|
+
if not self.leaving(group.group_id):
|
|
903
|
+
return "failed" # the leave ended meanwhile
|
|
904
|
+
if self.owner_must_transfer(gid):
|
|
905
|
+
await self._blocked(group.group_id)
|
|
906
|
+
return "failed"
|
|
907
|
+
members = self._client.members(gid)
|
|
908
|
+
own = self._own_leaf(members)
|
|
909
|
+
if any(int(m["leaf_index"]) == own for m in self._client.pending_leaves(gid)):
|
|
910
|
+
return "done" # proposed in this epoch already: once is enough
|
|
911
|
+
try:
|
|
912
|
+
proposal = self._client.propose_leave(gid)
|
|
913
|
+
except ac.AgentConnectError as e_:
|
|
914
|
+
LOG.info("leave_not_possible", reason=str(e_)[:60])
|
|
915
|
+
if self.owner_must_transfer(gid):
|
|
916
|
+
await self._blocked(group.group_id)
|
|
917
|
+
return "failed"
|
|
918
|
+
payload["env"] = {
|
|
919
|
+
"v": 0,
|
|
920
|
+
"kind": "proposal",
|
|
921
|
+
"group_id": group.group_id,
|
|
922
|
+
"epoch": self._client.epoch(gid),
|
|
923
|
+
"msg_id": new_msg_id(),
|
|
924
|
+
"sender": e.mailbox,
|
|
925
|
+
"ct": b64(proposal),
|
|
926
|
+
}
|
|
927
|
+
e.store.update_outbox(row.seq, payload=payload)
|
|
928
|
+
try:
|
|
929
|
+
await e._request({"t": "send", "env": payload["env"]}, str(payload["env"]["msg_id"]))
|
|
930
|
+
except RelayError as err:
|
|
931
|
+
# A stale proposal: the next commit we process triggers a resend.
|
|
932
|
+
LOG.info("proposal_failed", code=err.code)
|
|
933
|
+
await e.on_group_refused(group.group_id, err.code)
|
|
934
|
+
return "failed"
|
|
935
|
+
return "done"
|
|
936
|
+
|
|
937
|
+
async def on_leave_proposed(self, group_id: str, member: dict[str, Any]) -> None:
|
|
938
|
+
"""Another member proposed its own removal (inside the journal
|
|
939
|
+
entry's transaction)."""
|
|
940
|
+
e = self._engine
|
|
941
|
+
await e.emit(
|
|
942
|
+
group_id,
|
|
943
|
+
"member_leaving",
|
|
944
|
+
identity=str(member["identity"]),
|
|
945
|
+
instance_id=str(member["instance_id_hex"]),
|
|
946
|
+
)
|
|
947
|
+
self.check_leaves(group_id)
|
|
948
|
+
if self.leaving(group_id) and self.owner_must_transfer(unb64(group_id)):
|
|
949
|
+
# Every owner instance proposed at once and this one, the lowest,
|
|
950
|
+
# stays (SPEC 9.2): its own leave is not pending any more.
|
|
951
|
+
await self._blocked(group_id)
|
|
952
|
+
e._defer(self.wake)
|
|
953
|
+
|
|
954
|
+
def _proposal_queued(self, group_id: str) -> bool:
|
|
955
|
+
"""Whether a leave proposal for the current epoch waits in the outbox:
|
|
956
|
+
one not built yet (it proposes in whatever epoch it runs in), or built
|
|
957
|
+
in this epoch and being sent. One built in an earlier epoch is for an
|
|
958
|
+
epoch that ended, so it doesn't count."""
|
|
959
|
+
epoch = int(self._client.epoch(unb64(group_id)))
|
|
960
|
+
return any(
|
|
961
|
+
r.kind == "proposal"
|
|
962
|
+
and r.group_id == group_id
|
|
963
|
+
and ("env" not in r.payload or int(r.payload["env"]["epoch"]) == epoch)
|
|
964
|
+
for r in self._engine.store.outbox()
|
|
965
|
+
)
|
|
966
|
+
|
|
967
|
+
def _proposals_queued(self, group_id: str) -> bool:
|
|
968
|
+
return any(
|
|
969
|
+
r.kind == "commit"
|
|
970
|
+
and r.group_id == group_id
|
|
971
|
+
and (r.payload.get("intent") or {}).get("type") == "proposals"
|
|
972
|
+
for r in self._engine.store.outbox()
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
def check_leaves(self, group_id: str) -> float | None:
|
|
976
|
+
"""Commit the group's pending leaves if this instance is the one to:
|
|
977
|
+
the lowest leaf not itself leaving at once, the next one after
|
|
978
|
+
:data:`LEAVE_COMMIT_GRACE_SECS`, and so on, counted from when the
|
|
979
|
+
epoch's first leave proposal reached this instance. Returns when to
|
|
980
|
+
check again, or ``None``."""
|
|
981
|
+
e = self._engine
|
|
982
|
+
group = e.store.group(group_id)
|
|
983
|
+
if group is None or group.out_of_sync:
|
|
984
|
+
return None # its commits can't land: a retry would only fail again
|
|
985
|
+
gid = unb64(group_id)
|
|
986
|
+
key = f"leave_seen/{group_id}"
|
|
987
|
+
pending = self._client.pending_leaves(gid)
|
|
988
|
+
if not pending:
|
|
989
|
+
if self._kv(key):
|
|
990
|
+
e.store.kv_set(key, "")
|
|
991
|
+
return None
|
|
992
|
+
now, epoch = self._now(), int(self._client.epoch(gid))
|
|
993
|
+
seen = json.loads(self._kv(key) or "null")
|
|
994
|
+
if not isinstance(seen, dict) or seen.get("epoch") != epoch:
|
|
995
|
+
seen = {"epoch": epoch, "at": now}
|
|
996
|
+
e.store.kv_set(key, json.dumps(seen))
|
|
997
|
+
leaving = {int(m["leaf_index"]) for m in pending}
|
|
998
|
+
members = self._client.members(gid)
|
|
999
|
+
candidates = sorted(int(m["leaf_index"]) for m in members if m["leaf_index"] not in leaving)
|
|
1000
|
+
own = self._own_leaf(members)
|
|
1001
|
+
if own is None or own not in candidates:
|
|
1002
|
+
return None
|
|
1003
|
+
if self.leaving(group_id) and not self.owner_must_transfer(gid):
|
|
1004
|
+
# Its own leave is queued or pending: the core refuses its commits.
|
|
1005
|
+
# (The owner's kept instance, whose leave was dropped, commits.)
|
|
1006
|
+
return None
|
|
1007
|
+
due = int(seen["at"]) + candidates.index(own) * LEAVE_COMMIT_GRACE_SECS
|
|
1008
|
+
tried = seen.get("tried")
|
|
1009
|
+
if isinstance(tried, int):
|
|
1010
|
+
due = max(due, tried + LEAVE_COMMIT_GRACE_SECS)
|
|
1011
|
+
if now < due:
|
|
1012
|
+
return float(due)
|
|
1013
|
+
if not self._proposals_queued(group_id):
|
|
1014
|
+
e._enqueue("commit", group_id, {"intent": {"type": "proposals"}})
|
|
1015
|
+
seen["tried"] = now
|
|
1016
|
+
e.store.kv_set(key, json.dumps(seen))
|
|
1017
|
+
# Check again later: the commit may lose a race or fail.
|
|
1018
|
+
return float(now + LEAVE_COMMIT_GRACE_SECS)
|
|
1019
|
+
|
|
1020
|
+
async def _leave_timeout(self, group: GroupRow) -> float | None:
|
|
1021
|
+
"""D10: a leave no commit completes ends locally after
|
|
1022
|
+
``groups.leave_timeout_secs``.
|
|
1023
|
+
|
|
1024
|
+
Never for an instance of the owner (in the latest state it knows):
|
|
1025
|
+
a sibling's leave, or the commit that made it the owner's only
|
|
1026
|
+
instance, may not have reached it, so it keeps its state and reports
|
|
1027
|
+
``leave_blocked`` with ``timed_out``. Anyone else times out only once
|
|
1028
|
+
its view is current (``Engine.caught_up``: connected, the reconnect
|
|
1029
|
+
drain handled, the journal replayed), and only if it is then still a
|
|
1030
|
+
member and still not the owner: a transfer to an instance that slept
|
|
1031
|
+
through its deadline reaches it first."""
|
|
1032
|
+
e = self._engine
|
|
1033
|
+
group_id = group.group_id
|
|
1034
|
+
if not self.leaving(group_id):
|
|
1035
|
+
return None
|
|
1036
|
+
since = int(self._kv(f"leaving_since/{group_id}") or str(self._now()))
|
|
1037
|
+
due = since + e.config.groups_leave_timeout_secs
|
|
1038
|
+
if self._now() < due:
|
|
1039
|
+
return float(due)
|
|
1040
|
+
gid = unb64(group_id)
|
|
1041
|
+
current = e.caught_up()
|
|
1042
|
+
if json.loads(self._client.policy(gid)).get("owner") == e.me:
|
|
1043
|
+
reason = OWNER_MUST_TRANSFER if self.owner_must_transfer(gid) else OWNER_INSTANCE
|
|
1044
|
+
await self._blocked(group_id, timed_out=True, reason=reason)
|
|
1045
|
+
return None
|
|
1046
|
+
if not current or self._own_leaf(self._client.members(gid)) is None:
|
|
1047
|
+
return None # catching up wakes the timer
|
|
1048
|
+
LOG.info("leave_timed_out", group=log.hid(group_id))
|
|
1049
|
+
await self._gone_atomically(
|
|
1050
|
+
"leave_timeout", group_id, "left", cause=f"left/{group_id}/{since}", confirmed=False
|
|
1051
|
+
)
|
|
1052
|
+
return None
|
|
1053
|
+
|
|
1054
|
+
async def step(self) -> float | None:
|
|
1055
|
+
"""One pass of the timer over the active groups, after revoking the
|
|
1056
|
+
invites closed meanwhile; when the next is due."""
|
|
1057
|
+
revoke_again = await self.revoke_closed_invites()
|
|
1058
|
+
times: list[float] = [] if revoke_again is None else [revoke_again]
|
|
1059
|
+
for group in self._engine.store.groups():
|
|
1060
|
+
if group.status != "active" or group.out_of_sync:
|
|
1061
|
+
continue # an out-of-sync view decides nothing
|
|
1062
|
+
try:
|
|
1063
|
+
due = [self.check_leaves(group.group_id), await self._leave_timeout(group)]
|
|
1064
|
+
except (ac.AgentConnectError, ac.CoreError) as err:
|
|
1065
|
+
LOG.info("group_timer_skipped", reason=type(err).__name__)
|
|
1066
|
+
continue
|
|
1067
|
+
times.extend(d for d in due if d is not None)
|
|
1068
|
+
return min(times) if times else None
|
|
1069
|
+
|
|
1070
|
+
async def run(self) -> None:
|
|
1071
|
+
"""The membership timer: pending leaves this instance commits after
|
|
1072
|
+
the grace period, and leave timeouts. A pass that fails is logged by
|
|
1073
|
+
type and retried after a backoff."""
|
|
1074
|
+
e = self._engine
|
|
1075
|
+
backoff = Backoff(base=1.0, cap=300.0, rng=e.options.backoff_rng)
|
|
1076
|
+
while True:
|
|
1077
|
+
self._wake.clear()
|
|
1078
|
+
try:
|
|
1079
|
+
due = await self.step()
|
|
1080
|
+
backoff.reset()
|
|
1081
|
+
except Exception as err:
|
|
1082
|
+
LOG.warning("group_timer_failed", error=type(err).__name__)
|
|
1083
|
+
due = self._now() + backoff.next()
|
|
1084
|
+
woken = asyncio.ensure_future(self._wake.wait())
|
|
1085
|
+
waits: set[asyncio.Future[Any]] = {woken}
|
|
1086
|
+
if due is not None:
|
|
1087
|
+
waits.add(asyncio.ensure_future(e.clock.sleep_until(due, deadline=True)))
|
|
1088
|
+
try:
|
|
1089
|
+
await asyncio.sleep(0)
|
|
1090
|
+
e._pulse()
|
|
1091
|
+
await asyncio.wait(waits, return_when=asyncio.FIRST_COMPLETED)
|
|
1092
|
+
finally:
|
|
1093
|
+
for w in waits:
|
|
1094
|
+
w.cancel()
|
|
1095
|
+
|
|
1096
|
+
# -- removal --------------------------------------------------------------
|
|
1097
|
+
|
|
1098
|
+
def remove_member(self, group_id: str, identity: str) -> None:
|
|
1099
|
+
"""Commit the removal of every instance of ``identity`` (SPEC 9.2;
|
|
1100
|
+
the core checks this instance's role)."""
|
|
1101
|
+
group = self._engine._group_or_raise(group_id)
|
|
1102
|
+
self._engine._enqueue(
|
|
1103
|
+
"commit", group.group_id, {"intent": {"type": "remove", "identity": identity}}
|
|
1104
|
+
)
|
|
1105
|
+
|
|
1106
|
+
def remove_instances(self, group_id: str, identity: str, instance_ids: list[str]) -> None:
|
|
1107
|
+
"""Commit the removal of some of ``identity``'s instances, such as a
|
|
1108
|
+
dead replica (SPEC 9.2). One's own identity's other instances need
|
|
1109
|
+
no permission; anyone else's need ``remove``, which the core checks.
|
|
1110
|
+
This instance can't remove itself: it leaves."""
|
|
1111
|
+
e = self._engine
|
|
1112
|
+
group = e._group_or_raise(group_id)
|
|
1113
|
+
if not 0 < len(instance_ids) <= MAX_INSTANCES or not all(
|
|
1114
|
+
isinstance(i, str) and INSTANCE_ID.match(i) for i in instance_ids
|
|
1115
|
+
):
|
|
1116
|
+
msg = f"instances must list 1 to {MAX_INSTANCES} instance ids (16 hex characters)"
|
|
1117
|
+
raise DaemonError("bad_request", msg)
|
|
1118
|
+
member = next((m for m in e.store.members(group_id) if m.identity == identity), None)
|
|
1119
|
+
if member is None:
|
|
1120
|
+
raise DaemonError("not_found", "no such member")
|
|
1121
|
+
known = {str(i["instance_id"]) for i in member.instances}
|
|
1122
|
+
if not set(instance_ids) <= known:
|
|
1123
|
+
raise DaemonError("not_found", "no such instance of this member")
|
|
1124
|
+
if e.instance.instance_id_hex in instance_ids:
|
|
1125
|
+
raise DaemonError("bad_request", "this instance can't remove itself: leave instead")
|
|
1126
|
+
e._enqueue(
|
|
1127
|
+
"commit",
|
|
1128
|
+
group.group_id,
|
|
1129
|
+
{
|
|
1130
|
+
"intent": {
|
|
1131
|
+
"type": "remove_instances",
|
|
1132
|
+
"identity": identity,
|
|
1133
|
+
"instances": sorted(set(instance_ids)),
|
|
1134
|
+
}
|
|
1135
|
+
},
|
|
1136
|
+
)
|
|
1137
|
+
|
|
1138
|
+
# -- SAS mismatch (SPEC 11) -------------------------------------------------
|
|
1139
|
+
|
|
1140
|
+
async def sas_mismatch(self, group_id: str, run: str | None = None) -> str:
|
|
1141
|
+
"""**Human-only.** The human compared the code of a SAS run and the
|
|
1142
|
+
codes differ, or took the mismatch action of a run that never paired
|
|
1143
|
+
or of the stop (SPEC 11.5): the group view may be forked. Always
|
|
1144
|
+
accepted, whatever happened since. Leave at once and raise
|
|
1145
|
+
``sas_mismatch``.
|
|
1146
|
+
|
|
1147
|
+
* The owner's last instance first makes a best-effort ownership
|
|
1148
|
+
transfer (SPEC 9.2): a policy commit naming the lowest-index admin,
|
|
1149
|
+
or else the lowest-index other member, sent and awaited for at most
|
|
1150
|
+
``TRANSFER_TIMEOUT_SECS``. If the relay confirms it, this
|
|
1151
|
+
instance is now a plain member and proposes its own removal.
|
|
1152
|
+
* Anyone else sends a leave proposal when one is allowed.
|
|
1153
|
+
* A sole member just deletes.
|
|
1154
|
+
|
|
1155
|
+
The local group state is deleted in every case, since nothing more
|
|
1156
|
+
from this view can be trusted."""
|
|
1157
|
+
e = self._engine
|
|
1158
|
+
group = e._group_or_raise(group_id)
|
|
1159
|
+
current = e._core_epoch(unb64(group_id))
|
|
1160
|
+
transfer: dict[str, Any] | None = None
|
|
1161
|
+
if self._is_last_owner_instance(group):
|
|
1162
|
+
transfer = await self.transfer_ownership(group)
|
|
1163
|
+
if transfer is None or transfer["confirmed"]:
|
|
1164
|
+
self._propose_own_leave(group)
|
|
1165
|
+
await self.gone(group_id, "left", confirmed=False)
|
|
1166
|
+
e.store.update_group(group_id, out_of_sync=True, out_of_sync_reason="sas_mismatch")
|
|
1167
|
+
data: dict[str, Any] = {"epoch": current}
|
|
1168
|
+
if run is not None:
|
|
1169
|
+
data["run"] = run
|
|
1170
|
+
if transfer is not None:
|
|
1171
|
+
data["transfer"] = transfer
|
|
1172
|
+
await e.emit(group_id, "sas_mismatch", **data)
|
|
1173
|
+
return "left"
|
|
1174
|
+
|
|
1175
|
+
def _is_last_owner_instance(self, group: GroupRow) -> bool:
|
|
1176
|
+
me = self._engine.me
|
|
1177
|
+
if group.policy.get("owner") != me:
|
|
1178
|
+
return False
|
|
1179
|
+
mine = [m for m in self._client.members(unb64(group.group_id)) if m["identity"] == me]
|
|
1180
|
+
return len(mine) == 1
|
|
1181
|
+
|
|
1182
|
+
def _next_owner(self, group: GroupRow) -> str | None:
|
|
1183
|
+
"""The lowest-index admin, else the lowest-index other member."""
|
|
1184
|
+
me = self._engine.me
|
|
1185
|
+
members = sorted(
|
|
1186
|
+
(m for m in self._client.members(unb64(group.group_id)) if m["identity"] != me),
|
|
1187
|
+
key=lambda m: m["leaf_index"],
|
|
1188
|
+
)
|
|
1189
|
+
admins = set(group.policy.get("admins", []))
|
|
1190
|
+
for m in members:
|
|
1191
|
+
if m["identity"] in admins:
|
|
1192
|
+
return str(m["identity"])
|
|
1193
|
+
return str(members[0]["identity"]) if members else None
|
|
1194
|
+
|
|
1195
|
+
async def transfer_ownership(self, group: GroupRow) -> dict[str, Any] | None:
|
|
1196
|
+
"""Commit ``owner = next owner`` and wait (bounded) for the relay.
|
|
1197
|
+
``None`` for a sole member; otherwise who and whether it landed."""
|
|
1198
|
+
e = self._engine
|
|
1199
|
+
new_owner = self._next_owner(group)
|
|
1200
|
+
if new_owner is None:
|
|
1201
|
+
return None
|
|
1202
|
+
gid = unb64(group.group_id)
|
|
1203
|
+
policy = {**json.loads(self._client.policy(gid)), "owner": new_owner}
|
|
1204
|
+
policy["admins"] = [a for a in policy.get("admins", []) if a != new_owner]
|
|
1205
|
+
result: dict[str, Any] = {"new_owner": new_owner, "confirmed": False}
|
|
1206
|
+
try:
|
|
1207
|
+
out = self._client.set_policy(gid, canonical(policy))
|
|
1208
|
+
except (ac.AgentConnectError, ac.CoreError) as err:
|
|
1209
|
+
LOG.info("ownership_transfer_not_built", reason=str(err)[:60])
|
|
1210
|
+
return result
|
|
1211
|
+
env = {
|
|
1212
|
+
"v": 0,
|
|
1213
|
+
"kind": "commit",
|
|
1214
|
+
"group_id": group.group_id,
|
|
1215
|
+
"epoch": out["epoch"],
|
|
1216
|
+
"msg_id": out["msg_id"],
|
|
1217
|
+
"sender": e.mailbox,
|
|
1218
|
+
"routing": out["routing"],
|
|
1219
|
+
"ct": b64(out["commit"]),
|
|
1220
|
+
}
|
|
1221
|
+
send = asyncio.ensure_future(e._request({"t": "send", "env": env}, out["msg_id"]))
|
|
1222
|
+
timer = asyncio.ensure_future(e._sleep(TRANSFER_TIMEOUT_SECS))
|
|
1223
|
+
try:
|
|
1224
|
+
await asyncio.wait({send, timer}, return_when=asyncio.FIRST_COMPLETED)
|
|
1225
|
+
except BaseException:
|
|
1226
|
+
await cancel_and_wait(send, timer, ignore=(Exception,))
|
|
1227
|
+
raise
|
|
1228
|
+
timer.cancel()
|
|
1229
|
+
if send.done() and not send.cancelled() and send.exception() is None:
|
|
1230
|
+
# Accepted; the fan-out echo may already have merged it.
|
|
1231
|
+
with contextlib.suppress(ac.AgentConnectError, ac.CoreError):
|
|
1232
|
+
self._client.confirm_pending_commit(gid, str(out["msg_id"]))
|
|
1233
|
+
try:
|
|
1234
|
+
confirmed = json.loads(self._client.policy(gid)).get("owner") == new_owner
|
|
1235
|
+
except (ac.AgentConnectError, ac.CoreError, ValueError):
|
|
1236
|
+
confirmed = False
|
|
1237
|
+
# Stops the send if it is still waiting; a cancellation of this task
|
|
1238
|
+
# meanwhile propagates (a restart discards the unsent commit, §8.3).
|
|
1239
|
+
await cancel_and_wait(send, timer, ignore=(Exception,))
|
|
1240
|
+
if not confirmed:
|
|
1241
|
+
with contextlib.suppress(ac.AgentConnectError, ac.CoreError):
|
|
1242
|
+
self._client.discard_pending_commit(gid)
|
|
1243
|
+
result["confirmed"] = confirmed
|
|
1244
|
+
LOG.info("ownership_transfer", confirmed=confirmed)
|
|
1245
|
+
return result
|
|
1246
|
+
|
|
1247
|
+
def _propose_own_leave(self, group: GroupRow) -> None:
|
|
1248
|
+
"""Queue a self-Remove proposal when SPEC 9.2 allows one."""
|
|
1249
|
+
e = self._engine
|
|
1250
|
+
gid = unb64(group.group_id)
|
|
1251
|
+
try:
|
|
1252
|
+
proposal = self._client.propose_leave(gid)
|
|
1253
|
+
except (ac.AgentConnectError, ac.CoreError) as err:
|
|
1254
|
+
LOG.info("sas_mismatch_no_proposal", reason=str(err)[:60])
|
|
1255
|
+
return
|
|
1256
|
+
msg_id = new_msg_id()
|
|
1257
|
+
env = {
|
|
1258
|
+
"v": 0,
|
|
1259
|
+
"kind": "proposal",
|
|
1260
|
+
"group_id": group.group_id,
|
|
1261
|
+
"epoch": e._core_epoch(gid),
|
|
1262
|
+
"msg_id": msg_id,
|
|
1263
|
+
"sender": e.mailbox,
|
|
1264
|
+
"ct": b64(proposal),
|
|
1265
|
+
}
|
|
1266
|
+
e._enqueue("frame", None, {"frame": {"t": "send", "env": env}, "msg_id": msg_id})
|
|
1267
|
+
|
|
1268
|
+
# -- the view -------------------------------------------------------------
|
|
1269
|
+
|
|
1270
|
+
@staticmethod
|
|
1271
|
+
def _since_verified(member: MemberRow) -> list[str] | None:
|
|
1272
|
+
"""The member's instances that joined after the human verified it
|
|
1273
|
+
(SPEC 11: the flag stays, and the view shows them as new). ``None``
|
|
1274
|
+
(unknown) for a member verified by a daemon that didn't record the
|
|
1275
|
+
epoch: verifying it again tells."""
|
|
1276
|
+
if not member.verified:
|
|
1277
|
+
return []
|
|
1278
|
+
if member.verified_epoch is None:
|
|
1279
|
+
return None
|
|
1280
|
+
return sorted(
|
|
1281
|
+
str(i["instance_id"])
|
|
1282
|
+
for i in member.instances
|
|
1283
|
+
if int(i.get("joined_epoch", 0)) > member.verified_epoch
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
def group_info(self, group_id: str) -> dict[str, Any]:
|
|
1287
|
+
e = self._engine
|
|
1288
|
+
group = e.store.group(group_id)
|
|
1289
|
+
if group is None:
|
|
1290
|
+
raise DaemonError("not_found", "no such group")
|
|
1291
|
+
policy = group.policy
|
|
1292
|
+
guard = policy.get("loop_guard")
|
|
1293
|
+
pending: list[dict[str, Any]] = []
|
|
1294
|
+
if group.status == "active":
|
|
1295
|
+
pending = [
|
|
1296
|
+
{"identity": str(m["identity"]), "instance_id": str(m["instance_id_hex"])}
|
|
1297
|
+
for m in self._client.pending_leaves(unb64(group_id))
|
|
1298
|
+
]
|
|
1299
|
+
info: dict[str, Any] = {
|
|
1300
|
+
"group_id": group.group_id,
|
|
1301
|
+
"name": group.name,
|
|
1302
|
+
"purpose": group.purpose,
|
|
1303
|
+
"status": group.status,
|
|
1304
|
+
"epoch": group.epoch,
|
|
1305
|
+
"policy": group.policy,
|
|
1306
|
+
"out_of_sync": group.out_of_sync,
|
|
1307
|
+
"out_of_sync_reason": group.out_of_sync_reason,
|
|
1308
|
+
"adapters": group.adapters,
|
|
1309
|
+
"delivers_to": self._engine.adapter_names(group_id),
|
|
1310
|
+
"delivery_held": self._engine.unavailable(group_id) is not None,
|
|
1311
|
+
"trust_state": self.trust_state(group_id),
|
|
1312
|
+
"leaving": self.leave_state(group_id),
|
|
1313
|
+
"policy_summary": {
|
|
1314
|
+
"invite": policy.get("invite"),
|
|
1315
|
+
"remove": policy.get("remove"),
|
|
1316
|
+
"max_members": policy.get("max_members"),
|
|
1317
|
+
"max_agent_turns": guard.get("max_agent_turns")
|
|
1318
|
+
if isinstance(guard, dict)
|
|
1319
|
+
else None,
|
|
1320
|
+
},
|
|
1321
|
+
"floor": e.floor.view(group_id, policy, group.status == "active"),
|
|
1322
|
+
"pending_leaves": pending,
|
|
1323
|
+
"members": [
|
|
1324
|
+
{
|
|
1325
|
+
"identity": m.identity,
|
|
1326
|
+
"fingerprint": ac.fingerprint(ac_identity_key(m.identity)),
|
|
1327
|
+
"display_name": m.display_name,
|
|
1328
|
+
"display_name_verified": False,
|
|
1329
|
+
"role": m.role,
|
|
1330
|
+
"verified": m.verified,
|
|
1331
|
+
"instances": [
|
|
1332
|
+
{
|
|
1333
|
+
**i,
|
|
1334
|
+
"joined_epoch": int(i.get("joined_epoch", 0)),
|
|
1335
|
+
"self": i.get("mailbox_id") == e.mailbox,
|
|
1336
|
+
}
|
|
1337
|
+
for i in m.instances
|
|
1338
|
+
],
|
|
1339
|
+
"instances_since_verified": self._since_verified(m),
|
|
1340
|
+
"self": m.identity == e.me,
|
|
1341
|
+
}
|
|
1342
|
+
for m in e.store.members(group_id)
|
|
1343
|
+
],
|
|
1344
|
+
# SPEC 10.1: peer-supplied, so framed as untrusted.
|
|
1345
|
+
"agent_cards": {
|
|
1346
|
+
identity: {"untrusted": True, "card": card}
|
|
1347
|
+
for identity, card in e.store.cards(group_id).items()
|
|
1348
|
+
},
|
|
1349
|
+
}
|
|
1350
|
+
e._knock_origin(group_id, info)
|
|
1351
|
+
return info
|