unaiverse 0.1.11__cp311-cp311-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of unaiverse might be problematic. Click here for more details.

Files changed (50) hide show
  1. unaiverse/__init__.py +19 -0
  2. unaiverse/agent.py +2090 -0
  3. unaiverse/agent_basics.py +1948 -0
  4. unaiverse/clock.py +221 -0
  5. unaiverse/dataprops.py +1236 -0
  6. unaiverse/hsm.py +1892 -0
  7. unaiverse/modules/__init__.py +18 -0
  8. unaiverse/modules/cnu/__init__.py +17 -0
  9. unaiverse/modules/cnu/cnus.py +536 -0
  10. unaiverse/modules/cnu/layers.py +261 -0
  11. unaiverse/modules/cnu/psi.py +60 -0
  12. unaiverse/modules/hl/__init__.py +15 -0
  13. unaiverse/modules/hl/hl_utils.py +411 -0
  14. unaiverse/modules/networks.py +1509 -0
  15. unaiverse/modules/utils.py +710 -0
  16. unaiverse/networking/__init__.py +16 -0
  17. unaiverse/networking/node/__init__.py +18 -0
  18. unaiverse/networking/node/connpool.py +1308 -0
  19. unaiverse/networking/node/node.py +2499 -0
  20. unaiverse/networking/node/profile.py +446 -0
  21. unaiverse/networking/node/tokens.py +79 -0
  22. unaiverse/networking/p2p/__init__.py +187 -0
  23. unaiverse/networking/p2p/go.mod +127 -0
  24. unaiverse/networking/p2p/go.sum +548 -0
  25. unaiverse/networking/p2p/golibp2p.py +18 -0
  26. unaiverse/networking/p2p/golibp2p.pyi +135 -0
  27. unaiverse/networking/p2p/lib.go +2662 -0
  28. unaiverse/networking/p2p/lib.go.sha256 +1 -0
  29. unaiverse/networking/p2p/lib_types.py +312 -0
  30. unaiverse/networking/p2p/message_pb2.py +50 -0
  31. unaiverse/networking/p2p/messages.py +362 -0
  32. unaiverse/networking/p2p/mylogger.py +77 -0
  33. unaiverse/networking/p2p/p2p.py +871 -0
  34. unaiverse/networking/p2p/proto-go/message.pb.go +846 -0
  35. unaiverse/networking/p2p/unailib.cpython-311-darwin.so +0 -0
  36. unaiverse/stats.py +1481 -0
  37. unaiverse/streamlib/__init__.py +15 -0
  38. unaiverse/streamlib/streamlib.py +210 -0
  39. unaiverse/streams.py +776 -0
  40. unaiverse/utils/__init__.py +16 -0
  41. unaiverse/utils/lone_wolf.json +24 -0
  42. unaiverse/utils/misc.py +310 -0
  43. unaiverse/utils/sandbox.py +293 -0
  44. unaiverse/utils/server.py +435 -0
  45. unaiverse/world.py +335 -0
  46. unaiverse-0.1.11.dist-info/METADATA +367 -0
  47. unaiverse-0.1.11.dist-info/RECORD +50 -0
  48. unaiverse-0.1.11.dist-info/WHEEL +6 -0
  49. unaiverse-0.1.11.dist-info/licenses/LICENSE +43 -0
  50. unaiverse-0.1.11.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1308 @@
1
+ """
2
+ █████ █████ ██████ █████ █████ █████ █████ ██████████ ███████████ █████████ ██████████
3
+ ░░███ ░░███ ░░██████ ░░███ ░░███ ░░███ ░░███ ░░███░░░░░█░░███░░░░░███ ███░░░░░███░░███░░░░░█
4
+ ░███ ░███ ░███░███ ░███ ██████ ░███ ░███ ░███ ░███ █ ░ ░███ ░███ ░███ ░░░ ░███ █ ░
5
+ ░███ ░███ ░███░░███░███ ░░░░░███ ░███ ░███ ░███ ░██████ ░██████████ ░░█████████ ░██████
6
+ ░███ ░███ ░███ ░░██████ ███████ ░███ ░░███ ███ ░███░░█ ░███░░░░░███ ░░░░░░░░███ ░███░░█
7
+ ░███ ░███ ░███ ░░█████ ███░░███ ░███ ░░░█████░ ░███ ░ █ ░███ ░███ ███ ░███ ░███ ░ █
8
+ ░░████████ █████ ░░█████░░████████ █████ ░░███ ██████████ █████ █████░░█████████ ██████████
9
+ ░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░ ░░░░░ ░░░ ░░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░ ░░░░░░░░░░
10
+ A Collectionless AI Project (https://collectionless.ai)
11
+ Registration/Login: https://unaiverse.io
12
+ Code Repositories: https://github.com/collectionlessai/
13
+ Main Developers: Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
14
+ """
15
+ import math
16
+ import base64
17
+ import binascii
18
+ from unaiverse.networking.p2p.messages import Msg
19
+ from unaiverse.networking.p2p.p2p import P2P, P2PError
20
+ from unaiverse.networking.node.tokens import TokenVerifier
21
+
22
+
23
+ class ConnectionPools:
24
+ DEBUG = True
25
+
26
+ def __init__(self, max_connections: int, pool_name_to_p2p_name_and_ratio: dict[str, [str, float]],
27
+ p2p_name_to_p2p: dict[str, P2P], public_key: str | None = None, token: str | None = None):
28
+ """Initializes a new instance of the ConnectionPools class.
29
+
30
+ Args:
31
+ max_connections: The maximum total number of connections allowed across all pools.
32
+ pool_name_to_p2p_name_and_ratio: A dictionary mapping pool names to a list containing the associated P2P
33
+ network name and its connection ratio.
34
+ p2p_name_to_p2p: A dictionary mapping P2P network names to their corresponding P2P objects.
35
+ public_key: An optional public key for token verification.
36
+ token: An optional initial token for authentication.
37
+
38
+ Returns:
39
+ None.
40
+
41
+ """
42
+ # Common terms: a "pool triple" is [pool_contents, max_connections_in_such_a_pool, p2p_object_of_the_pool
43
+ self.max_con = max_connections
44
+ self.pool_count = len(pool_name_to_p2p_name_and_ratio)
45
+ self.pool_names = list(pool_name_to_p2p_name_and_ratio.keys())
46
+ self.pool_ratios = [p2p_name_and_ratio[1] for p2p_name_and_ratio in pool_name_to_p2p_name_and_ratio.values()]
47
+
48
+ # Indices involving the P2P object or its name
49
+ self.p2p_name_to_p2p = p2p_name_to_p2p
50
+ self.p2p_name_and_pool_name_to_pool_triple = {}
51
+ self.p2p_to_pool_names = {}
52
+
53
+ # Indices rooted around the pool name
54
+ self.pool_name_to_pool_triple = {}
55
+ self.pool_name_to_added_in_last_update = {}
56
+ self.pool_name_to_removed_in_last_update = {}
57
+ self.pool_name_to_peer_infos = {p: {} for p in self.pool_names}
58
+
59
+ # Indices rooted around the peer ID
60
+ self.peer_id_to_pool_name = {}
61
+ self.peer_id_to_p2p = {}
62
+ self.peer_id_to_misc = {}
63
+ self.peer_id_to_token = {}
64
+
65
+ # Token-related stuff, super private
66
+ self.__token = token if token is not None else ""
67
+ self.__token_verifier = TokenVerifier(public_key) if public_key is not None else None
68
+
69
+ # Checking
70
+ for p2p_name_and_ratio in pool_name_to_p2p_name_and_ratio.values():
71
+ assert p2p_name_and_ratio[0] in self.p2p_name_to_p2p, f"Cannot find p2p named {p2p_name_and_ratio[0]} "
72
+ assert self.max_con >= len(self.pool_names), "Too small number of max connections"
73
+ assert sum([x for x in self.pool_ratios if x > 0]) == 1.0, "Pool ratios must sum to 1.0"
74
+
75
+ # Preparing the pool triples
76
+ self.pool_name_to_pool_triple = \
77
+ {k: [set(), 0, self.p2p_name_to_p2p[pool_name_to_p2p_name_and_ratio[k][0]]] for k in self.pool_names}
78
+ num_zero_ratio_pools = len([x for x in self.pool_ratios if x == 0])
79
+ assert num_zero_ratio_pools <= self.max_con, "Cannot create pools given the provided max connection count"
80
+
81
+ # Edit: to solve the teacher not engaging with more than two students.
82
+ pools_max_sizes = {k: max(math.floor(self.pool_ratios[i] * (self.max_con - num_zero_ratio_pools)),
83
+ 1 if self.pool_ratios[i] >= 0. else 0)
84
+ for i, k in enumerate(self.pool_names)}
85
+
86
+ # Pools_max_sizes = {k: self.max_con for k in self.pool_names}
87
+
88
+ # Fixing sizes
89
+ tot = 0
90
+ for i, (k, v) in enumerate(pools_max_sizes.items()):
91
+ assert v > 0 or self.pool_ratios[i] < 0, "Cannot create pools given the provided max connection count"
92
+
93
+ # Edit: to solve the teacher not engaging with more than two students.
94
+ tot += v
95
+ assert tot <= self.max_con, \
96
+ "Cannot create pools given the provided max connection count"
97
+ pools_max_sizes[self.pool_names[-1]] += (self.max_con - tot)
98
+
99
+ # Storing fixed sizes in the previously created pool triples & building additional index
100
+ for pool_name, pool_contents_max_con_and_p2p in self.pool_name_to_pool_triple.items():
101
+ pool_contents_max_con_and_p2p[1] = pools_max_sizes[pool_name] # Fixing the second element of the triple
102
+
103
+ pool, _, p2p = pool_contents_max_con_and_p2p
104
+ p2p_name = None
105
+ for k, v in self.p2p_name_to_p2p.items():
106
+ if v == p2p:
107
+ p2p_name = k
108
+ break
109
+ if p2p_name not in self.p2p_name_and_pool_name_to_pool_triple:
110
+ self.p2p_name_and_pool_name_to_pool_triple[p2p_name] = {}
111
+ self.p2p_to_pool_names[p2p] = []
112
+ self.p2p_name_and_pool_name_to_pool_triple[p2p_name][pool_name] = (
113
+ pool_contents_max_con_and_p2p)
114
+ self.p2p_to_pool_names[p2p].append(pool_name)
115
+
116
+ def __str__(self):
117
+ """Returns a human-readable string representation of the connection pools' status.
118
+
119
+ Args:
120
+ None.
121
+
122
+ Returns:
123
+ A formatted string showing the number of connections and peer IDs in each pool.
124
+ """
125
+ max_len = max(len(s) for s in self.pool_names)
126
+ s = f"[ConnectionPool] max_con={self.max_con}, pool_count={self.pool_count}"
127
+ for k, (pool, max_con, _) in self.pool_name_to_pool_triple.items():
128
+ kk = k + ","
129
+ s += f"\n\tpool={kk.ljust(max_len + 1)}\tmax_con={max_con},\tcurrent_con={len(pool)},\tpeer_ids="
130
+ ss = "("
131
+ first = True
132
+ for c in pool:
133
+ if not first:
134
+ ss += ", "
135
+ ss += c
136
+ first = False
137
+ ss += ")"
138
+ s += ss
139
+ return s
140
+
141
+ def __getitem__(self, p2p_name):
142
+ """Retrieves a P2P object by its name.
143
+
144
+ Args:
145
+ p2p_name: The name of the P2P network.
146
+
147
+ Returns:
148
+ The P2P object.
149
+ """
150
+ return self.p2p_name_to_p2p[p2p_name]
151
+
152
+ async def conn_routing_fcn(self, connected_peer_infos: list, p2p: P2P):
153
+ """A placeholder function that must be implemented to route connected peers to the correct pool (async).
154
+
155
+ Args:
156
+ connected_peer_infos: A list of dictionaries containing information about connected peers.
157
+ p2p: The P2P network object from which the peers were connected.
158
+
159
+ Returns:
160
+ A dictionary mapping pool names to a dictionary of peer IDs and their information.
161
+ """
162
+ raise NotImplementedError("You must implement conn_routing_fcn!")
163
+
164
+ @staticmethod
165
+ async def __connect(p2p: P2P, addresses: list[str]):
166
+ """Establishes a connection to a peer via a P2P network (async).
167
+
168
+ Args:
169
+ p2p: The P2P network object to use for the connection.
170
+ addresses: A list of addresses of the peer to connect to.
171
+
172
+ Returns:
173
+ A tuple containing the peer ID and a boolean indicating if the connection was established through a relay.
174
+ """
175
+ if ConnectionPools.DEBUG:
176
+ print(f"[DEBUG CONNECTIONS-POOL] Connecting to {addresses}")
177
+
178
+ try:
179
+ winning_addr_info_dict = p2p.connect_to(addresses)
180
+ peer_id = winning_addr_info_dict.get('ID')
181
+ connected_addr_str = winning_addr_info_dict.get('Addrs')[0]
182
+ through_relay = '/p2p-circuit/' in connected_addr_str
183
+ if ConnectionPools.DEBUG:
184
+ print(f"[DEBUG CONNECTIONS-POOL] Connected to peer {peer_id} via {connected_addr_str} "
185
+ f"(through relay: {through_relay})")
186
+
187
+ return peer_id, through_relay
188
+ except P2PError:
189
+ if ConnectionPools.DEBUG:
190
+ print(f"[DEBUG CONNECTIONS-POOL] Connection failed!")
191
+ return None, False
192
+
193
+ @staticmethod
194
+ async def disconnect(p2p: P2P, peer_id: str):
195
+ """Disconnects from a specific peer on a P2P network (async).
196
+
197
+ Args:
198
+ p2p: The P2P network object to use for disconnection.
199
+ peer_id: The peer ID to disconnect from.
200
+
201
+ Returns:
202
+ True if the disconnection is successful, otherwise False.
203
+ """
204
+ try:
205
+ p2p.disconnect_from(peer_id)
206
+ except P2PError:
207
+ return False
208
+ return True
209
+
210
+ def set_token(self, token: str):
211
+ """Sets the authentication token for the connection pools.
212
+
213
+ Args:
214
+ token: The new token string.
215
+ """
216
+ self.__token = token
217
+
218
+ async def verify_token(self, token: str, peer_id: str):
219
+ """Verifies a received token using the provided public key (async).
220
+
221
+ Args:
222
+ token: The token string to verify.
223
+ peer_id: The peer ID associated with the token.
224
+
225
+ Returns:
226
+ A tuple containing the node ID and CV hash if the token is valid, otherwise None.
227
+ """
228
+ if self.__token_verifier is None:
229
+ return None
230
+ else:
231
+ node_id, cv_hash = self.__token_verifier.verify_token(token, p2p_peer=peer_id)
232
+ return node_id, cv_hash # If the verification fails, this is None, None
233
+
234
+ async def connect(self, addresses: list[str], p2p_name: str):
235
+ """Connects to a peer on a specified P2P network (async).
236
+
237
+ Args:
238
+ addresses: A list of addresses of the peer to connect to.
239
+ p2p_name: The name of the P2P network to use.
240
+
241
+ Returns:
242
+ A tuple containing the peer ID of the connected peer and a boolean indicating if a relay was used.
243
+ """
244
+ p2p = self.p2p_name_to_p2p[p2p_name]
245
+
246
+ # Connecting
247
+ peer_id, through_relay = await ConnectionPools.__connect(p2p, addresses)
248
+ return peer_id, through_relay
249
+
250
+ def add(self, peer_info: dict, pool_name: str):
251
+ """Adds a connected peer to a specified connection pool.
252
+
253
+ Args:
254
+ peer_info: A dictionary containing information about the peer.
255
+ pool_name: The name of the pool to add the peer to.
256
+
257
+ Returns:
258
+ True if the peer is successfully added, otherwise False.
259
+ """
260
+ peer_id = peer_info['id']
261
+ pool, max_size, p2p = self.pool_name_to_pool_triple[pool_name]
262
+ if len(pool) < max_size:
263
+
264
+ # "hoping" peer IDs are unique, and stopping duplicate cases
265
+ if peer_id in self.peer_id_to_pool_name and self.peer_id_to_pool_name[peer_id] != pool_name:
266
+ return False
267
+
268
+ self.peer_id_to_pool_name[peer_id] = pool_name
269
+ self.peer_id_to_p2p[peer_id] = p2p
270
+
271
+ # Setting 'misc' field (default is 0, where 0 means public)
272
+ peer_info['misc'] = self.peer_id_to_misc.get(peer_id, 0)
273
+
274
+ # Storing (only)
275
+ pool.add(peer_id)
276
+ self.pool_name_to_peer_infos[pool_name][peer_id] = peer_info
277
+ return True
278
+ else:
279
+ return False
280
+
281
+ async def remove(self, peer_id: str):
282
+ """Removes a peer from its connection pool and disconnects from it (async).
283
+
284
+ Args:
285
+ peer_id: The peer ID to remove.
286
+
287
+ Returns:
288
+ True if the peer is successfully removed, otherwise False.
289
+ """
290
+ if peer_id in self.peer_id_to_pool_name:
291
+ pool_name = self.peer_id_to_pool_name[peer_id]
292
+ pool, _, p2p = self.pool_name_to_pool_triple[pool_name]
293
+
294
+ # Disconnecting
295
+ disc = await ConnectionPools.disconnect(p2p, peer_id)
296
+ pool.remove(peer_id)
297
+ del self.pool_name_to_peer_infos[pool_name][peer_id]
298
+ del self.peer_id_to_pool_name[peer_id]
299
+ del self.peer_id_to_p2p[peer_id]
300
+ if peer_id in self.peer_id_to_misc:
301
+ del self.peer_id_to_misc[peer_id]
302
+ if peer_id in self.peer_id_to_token:
303
+ del self.peer_id_to_token[peer_id]
304
+ return disc
305
+ else:
306
+ return False
307
+
308
+ def get_all_connected_peer_infos(self, pool_name: str):
309
+ """Retrieves a list of peer information dictionaries for a given pool.
310
+
311
+ Args:
312
+ pool_name: The name of the pool to query.
313
+
314
+ Returns:
315
+ A list of dictionaries, each containing information about a peer in the pool.
316
+ """
317
+ return list(self.pool_name_to_peer_infos[pool_name].values())
318
+
319
+ def get_pool_status(self):
320
+ """Returns a dictionary showing the set of peer IDs in each pool.
321
+
322
+ Args:
323
+ None.
324
+
325
+ Returns:
326
+ A dictionary mapping pool names to the set of peer IDs in that pool.
327
+ """
328
+ return {k: v[0] for k, v in self.pool_name_to_pool_triple.items()}
329
+
330
+ def get_all_connected_peer_ids(self):
331
+ """Retrieves a list of all peer IDs currently connected across all pools.
332
+
333
+ Args:
334
+ None.
335
+
336
+ Returns:
337
+ A list of all connected peer IDs.
338
+ """
339
+ return list(self.peer_id_to_pool_name.keys())
340
+
341
+ async def update(self):
342
+ """Refreshes the connection pools by checking for new and lost connections (async).
343
+
344
+ Args:
345
+ None.
346
+
347
+ Returns:
348
+ A tuple containing two dictionaries: one for newly added peers and one for removed peers, both keyed by
349
+ pool name.
350
+ """
351
+ self.pool_name_to_added_in_last_update = {}
352
+ self.pool_name_to_removed_in_last_update = {}
353
+
354
+ for p2p_name, p2p in self.p2p_name_to_p2p.items():
355
+ connected_peer_infos = p2p.get_connected_peers_info()
356
+
357
+ if connected_peer_infos is not None:
358
+
359
+ # Routing to the right queue / filtering
360
+ pool_name_and_peer_ids_to_peer_info = await self.conn_routing_fcn(connected_peer_infos, p2p)
361
+
362
+ # Parsing the generated index
363
+ for pool_name, connected_peer_ids_to_connected_peer_infos \
364
+ in pool_name_and_peer_ids_to_peer_info.items():
365
+ pool, _, pool_p2p = self.p2p_name_and_pool_name_to_pool_triple[p2p_name][pool_name]
366
+ connected_peer_ids = connected_peer_ids_to_connected_peer_infos.keys()
367
+ new_peer_ids = connected_peer_ids - pool
368
+ lost_peer_ids = pool - connected_peer_ids
369
+
370
+ # Clearing disconnected agents
371
+ for lost_peer_id in lost_peer_ids:
372
+ self.pool_name_to_removed_in_last_update.setdefault(pool_name, set()).add(lost_peer_id)
373
+
374
+ # Adding new agents
375
+ for new_peer_id in new_peer_ids:
376
+ peer_info = connected_peer_ids_to_connected_peer_infos[new_peer_id]
377
+ if not self.add(peer_info, pool_name=pool_name):
378
+ break
379
+ self.pool_name_to_added_in_last_update.setdefault(pool_name, set()).add(new_peer_id)
380
+
381
+ return self.pool_name_to_added_in_last_update, self.pool_name_to_removed_in_last_update
382
+
383
+ async def get_messages(self, p2p_name: str, allowed_not_connected_peers: set | None = None) -> list[Msg]:
384
+ """Retrieves and verifies all messages from a specified P2P network (async).
385
+
386
+ Args:
387
+ p2p_name: The name of the P2P network to fetch messages from.
388
+ allowed_not_connected_peers: An optional set of peer IDs to allow messages from, even if they are not
389
+ in the pools.
390
+
391
+ Returns:
392
+ A list of verified and processed message objects.
393
+ """
394
+ # Pop all messages
395
+ byte_messages: list[bytes] = self[p2p_name].pop_messages() # Pop all messages
396
+ # Process the list of message dictionaries
397
+ processed_messages: list[Msg] = []
398
+ for i, msg_dict in enumerate(byte_messages):
399
+ try:
400
+ # Extract and validate required fields from the Go message structure
401
+ # Go structure: {"from":"Qm...", "data":"BASE64_ENCODED_DATA"}
402
+ verified_sender_id = msg_dict.get("from")
403
+ base64_data = msg_dict.get("data")
404
+
405
+ # Decode data
406
+ decoded_data = base64.b64decode(base64_data)
407
+
408
+ # Attempt to create the higher-level Msg object
409
+ # This assumes Msg.from_bytes can parse your message protocol from decoded_data
410
+ # and that Msg objects store sender, type, channel intrinsically or can be set.
411
+ msg_obj = Msg.from_bytes(decoded_data)
412
+
413
+ # --- CRITICAL SECURITY CHECK ---
414
+ # Verify that the sender claimed inside the message payload
415
+ # matches the cryptographically verified sender from the network layer.
416
+ if msg_obj.sender != verified_sender_id:
417
+ print(f"[DEBUG CONNECTIONS-POOL] SENDER MISMATCH! Network sender '{verified_sender_id}' "
418
+ f"does not match payload sender '{msg_obj.sender}'. Discarding message.")
419
+
420
+ # In a real-world scenario, you might also want to penalize or disconnect
421
+ # from a peer that sends such malformed/spoofed messages.
422
+ continue # Discard this message
423
+
424
+ # filter only valid messages to return
425
+ if (msg_obj.sender in self.peer_id_to_pool_name or # Check if expected sender
426
+ (allowed_not_connected_peers is not None and msg_obj.sender in allowed_not_connected_peers)):
427
+
428
+ try:
429
+ token_with_inspector_final_bit = msg_obj.piggyback
430
+ token = token_with_inspector_final_bit[0:-1]
431
+ inspector_mode = token_with_inspector_final_bit[-1]
432
+ node_id, _ = await self.verify_token(token, msg_obj.sender)
433
+ if node_id is not None:
434
+ # Replacing piggyback with the node ID and the flag telling if it is inspector
435
+ msg_obj.piggyback = node_id + inspector_mode
436
+ processed_messages.append(msg_obj)
437
+ if msg_obj.sender in self.peer_id_to_pool_name:
438
+ self.peer_id_to_token[msg_obj.sender] = token
439
+ else:
440
+ print("Received a message missing expected info in the token payload (discarding it)")
441
+ except Exception as e:
442
+ print(f"Received a message with an invalid piggyback token! (discarding it) [{e}]")
443
+
444
+ except ValueError as ve:
445
+ print(f"[DEBUG CONNECTIONS-POOL]Invalid message created, stopping. Error: {ve}")
446
+ continue # Skip problematic message
447
+ except (TypeError, binascii.Error) as decode_err:
448
+ print(f"[DEBUG CONNECTIONS-POOL] Failed to decode Base64 data for a message in batch: {decode_err}. "
449
+ f"Message dict: {msg_dict}")
450
+ continue # Skip problematic message
451
+ except Exception as msg_proc_err: # Catch errors from Msg.from_bytes or attribute setting
452
+ print(f"[DEBUG CONNECTIONS-POOL] Error processing popped message item {i}: {msg_proc_err}. "
453
+ f"Message dict: {msg_dict}")
454
+ continue # Skip problematic message
455
+
456
+ return processed_messages
457
+
458
+ def get_added_after_updating(self, pool_name: str | None = None):
459
+ """Retrieves the peers that were added in the last update cycle.
460
+
461
+ Args:
462
+ pool_name: The name of a specific pool to query. If None, returns data for all pools.
463
+
464
+ Returns:
465
+ A set of added peer IDs for the specified pool, or a dictionary of sets for all pools.
466
+ """
467
+ if pool_name is not None:
468
+ return self.pool_name_to_added_in_last_update[pool_name]
469
+ else:
470
+ return self.pool_name_to_added_in_last_update
471
+
472
+ def get_removed_after_updating(self, pool_name: str | None = None):
473
+ """Retrieves the peers that were removed in the last update cycle.
474
+
475
+ Args:
476
+ pool_name: The name of a specific pool to query. If None, returns data for all pools.
477
+
478
+ Returns:
479
+ A set of removed peer IDs for the specified pool, or a dictionary of sets for all pools.
480
+ """
481
+ if pool_name is not None:
482
+ return self.pool_name_to_removed_in_last_update[pool_name]
483
+ else:
484
+ return self.pool_name_to_removed_in_last_update
485
+
486
+ def get_last_token(self, peer_id):
487
+ """Retrieves the last known token for a given peer.
488
+
489
+ Args:
490
+ peer_id: The peer ID to query.
491
+
492
+ Returns:
493
+ The token string if found, otherwise None.
494
+ """
495
+ return self.peer_id_to_token[peer_id] if peer_id in self.peer_id_to_token else None
496
+
497
+ def is_connected(self, peer_id: str, pool_name: str | None = None):
498
+ """Checks if a peer is currently connected, optionally in a specific pool.
499
+
500
+ Args:
501
+ peer_id: The peer ID to check.
502
+ pool_name: An optional pool name to check within.
503
+
504
+ Returns:
505
+ True if the peer is connected, otherwise False.
506
+ """
507
+ if pool_name is None:
508
+ return peer_id in self.peer_id_to_pool_name
509
+ else:
510
+ return peer_id in self.peer_id_to_pool_name and pool_name == self.peer_id_to_pool_name[peer_id]
511
+
512
+ def get_pool_of(self, peer_id: str):
513
+ """Gets the pool name for a given connected peer.
514
+
515
+ Args:
516
+ peer_id: The peer ID to query.
517
+
518
+ Returns:
519
+ The name of the pool the peer is in.
520
+ """
521
+ return self.peer_id_to_pool_name[peer_id]
522
+
523
+ def size(self, pool_name: str | None = None):
524
+ """Returns the number of connections in a specific pool or the total number across all pools.
525
+
526
+ Args:
527
+ pool_name: An optional pool name to get the size of. If None, returns the total size.
528
+
529
+ Returns:
530
+ The size of the pool or the total number of connections.
531
+ """
532
+ if pool_name is not None:
533
+ return len(self.pool_name_to_pool_triple[pool_name])
534
+ else:
535
+ c = 0
536
+ for v in self.pool_name_to_pool_triple.values():
537
+ c += len(v)
538
+ return c
539
+
540
+ async def send(self, peer_id: str, channel_trail: str | None,
541
+ content_type: str, content: bytes | dict | None = None, p2p: P2P | None = None):
542
+ """Sends a direct message to a specific peer (async).
543
+
544
+ Args:
545
+ peer_id: The peer ID to send the message to.
546
+ channel_trail: An optional string to append to the channel name.
547
+ content_type: The type of content in the message.
548
+ content: The message content.
549
+ p2p: An optional P2P object to use for sending. If None, it is derived from the peer_id.
550
+
551
+ Returns:
552
+ True if the message is sent successfully, otherwise False.
553
+ """
554
+ # Getting the right p2p object
555
+ if p2p is None:
556
+ p2p = self.peer_id_to_p2p[peer_id] if peer_id in self.peer_id_to_p2p else None
557
+ if p2p is None:
558
+ if ConnectionPools.DEBUG:
559
+ print("[DEBUG CONNECTIONS-POOL] P2P non found for peer id: " + str(peer_id))
560
+ return False
561
+
562
+ # Defining channel
563
+ if channel_trail is not None and len(channel_trail) > 0:
564
+ channel = f"{p2p.peer_id}::dm:{peer_id}-{content_type}~{channel_trail}"
565
+ else:
566
+ channel = f"{p2p.peer_id}::dm:{peer_id}-{content_type}"
567
+
568
+ # Adding sender info here
569
+ msg = Msg(sender=p2p.peer_id,
570
+ content_type=content_type,
571
+ content=content,
572
+ channel=channel,
573
+ piggyback=self.__token + "0") # Adding inspector-mode bit (dummy bit here)
574
+ if ConnectionPools.DEBUG:
575
+ print("[DEBUG CONNECTIONS-POOL] Sending message: " + str(msg))
576
+
577
+ # Sending direct message
578
+ try:
579
+ p2p.send_message_to_peer(channel, msg_bytes=msg.to_bytes())
580
+
581
+ # If the line above executes without raising an error, it was successful.
582
+ return True
583
+ except P2PError as e:
584
+
585
+ # If send_message_to_peer fails, it will raise a P2PError. We catch it here.
586
+ if ConnectionPools.DEBUG:
587
+ print("[DEBUG CONNECTIONS-POOL] Sending error is: " + str(e))
588
+ return False
589
+
590
+ async def subscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
591
+ """Subscribes to a topic/channel on a P2P network (async).
592
+
593
+ Args:
594
+ peer_id: The peer ID associated with the topic/channel.
595
+ channel: The name of the channel to subscribe to.
596
+ default_p2p_name: An optional P2P network name to use if the peer's network is unknown.
597
+
598
+ Returns:
599
+ True if the subscription is successful, otherwise False.
600
+ """
601
+
602
+ # Getting the right p2p object
603
+ p2p = None
604
+ for _p2p in self.p2p_to_pool_names.keys():
605
+ if _p2p.peer_id == peer_id:
606
+ p2p = _p2p
607
+ break
608
+ if p2p is None and peer_id in self.peer_id_to_p2p:
609
+ p2p = self.peer_id_to_p2p[peer_id]
610
+ if p2p is None:
611
+ if default_p2p_name is not None:
612
+ p2p = self.p2p_name_to_p2p[default_p2p_name]
613
+ else:
614
+ return False
615
+
616
+ try:
617
+ p2p.subscribe_to_topic(channel)
618
+ except (P2PError, ValueError):
619
+ return False
620
+ return True
621
+
622
+ async def unsubscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
623
+ """Unsubscribes from a topic/channel on a P2P network (async).
624
+
625
+ Args:
626
+ peer_id: The peer ID associated with the topic/channel.
627
+ channel: The name of the channel to unsubscribe from.
628
+ default_p2p_name: An optional P2P network name to use if the peer's network is unknown.
629
+
630
+ Returns:
631
+ True if the unsubscription is successful, otherwise False.
632
+ """
633
+
634
+ # Getting the right p2p object
635
+ p2p = None
636
+ for _p2p in self.p2p_to_pool_names.keys():
637
+ if _p2p.peer_id == peer_id:
638
+ p2p = _p2p
639
+ break
640
+ if p2p is None and peer_id in self.peer_id_to_p2p:
641
+ p2p = self.peer_id_to_p2p[peer_id]
642
+ if p2p is None:
643
+ if default_p2p_name is not None:
644
+ p2p = self.p2p_name_to_p2p[default_p2p_name]
645
+ else:
646
+ return False
647
+
648
+ try:
649
+ p2p.unsubscribe_from_topic(channel)
650
+ except (P2PError, ValueError):
651
+ return False
652
+ return True
653
+
654
+ async def publish(self, peer_id: str, channel: str,
655
+ content_type: str, content: bytes | dict | tuple | None = None):
656
+ """Publishes a message to a topic/channel on a P2P network (async).
657
+
658
+ Args:
659
+ peer_id: The peer ID associated with the topic/channel.
660
+ channel: The name of the channel to publish to.
661
+ content_type: The type of content in the message.
662
+ content: The message content.
663
+
664
+ Returns:
665
+ True if the message is published successfully, otherwise False.
666
+ """
667
+
668
+ # Getting the right p2p object
669
+ p2p = None
670
+ for _p2p in self.p2p_to_pool_names.keys():
671
+ if _p2p.peer_id == peer_id:
672
+ p2p = _p2p
673
+ break
674
+ if p2p is None:
675
+ p2p = self.peer_id_to_p2p[peer_id]
676
+ if p2p is None:
677
+ return False
678
+
679
+ # Adding sender info here
680
+ msg = Msg(sender=p2p.peer_id,
681
+ content_type=content_type,
682
+ content=content,
683
+ channel=channel,
684
+ piggyback=self.__token + "0") # Adding inspector-mode bit (dummy bit here)
685
+ if ConnectionPools.DEBUG:
686
+ print("[DEBUG CONNECTIONS-POOL] Sending (publish) message: " + str(msg))
687
+
688
+ # Sending message via GossipSub
689
+ try:
690
+ p2p.broadcast_message(channel, msg_bytes=msg.to_bytes())
691
+
692
+ # If the line above executes without raising an error, it was successful.
693
+ return True
694
+ except P2PError:
695
+
696
+ # If send_message_to_peer fails, it will raise a P2PError. We catch it here.
697
+ return False
698
+
699
+
700
+ class NodeConn(ConnectionPools):
701
+
702
+ # Basic name
703
+ __ALL_UNIVERSE = "all_universe"
704
+ __WORLD_AGENTS_ONLY = "world_agents"
705
+ __WORLD_NODE_ONLY = "world_node"
706
+ __WORLD_MASTERS_ONLY = "world_masters"
707
+
708
+ # Suffixes
709
+ __PUBLIC_NET = "_public"
710
+ __PRIVATE_NET = "_private"
711
+
712
+ # Prefixes
713
+ __INBOUND = "in_"
714
+ __OUTBOUND = "out_"
715
+
716
+ # P2p names
717
+ P2P_PUBLIC = "p2p_public"
718
+ P2P_WORLD = "p2p_world"
719
+
720
+ # All pools (prefix + basic name + suffix)
721
+ IN_PUBLIC = __INBOUND + __ALL_UNIVERSE + __PUBLIC_NET
722
+ OUT_PUBLIC = __OUTBOUND + __ALL_UNIVERSE + __PUBLIC_NET
723
+ IN_WORLD_AGENTS = __INBOUND + __WORLD_AGENTS_ONLY + __PRIVATE_NET
724
+ OUT_WORLD_AGENTS = __OUTBOUND + __WORLD_AGENTS_ONLY + __PRIVATE_NET
725
+ IN_WORLD_NODE = __INBOUND + __WORLD_NODE_ONLY + __PRIVATE_NET
726
+ OUT_WORLD_NODE = __OUTBOUND + __WORLD_NODE_ONLY + __PRIVATE_NET
727
+ IN_WORLD_MASTERS = __INBOUND + __WORLD_MASTERS_ONLY + __PRIVATE_NET
728
+ OUT_WORLD_MASTERS = __OUTBOUND + __WORLD_MASTERS_ONLY + __PRIVATE_NET
729
+
730
+ # Aggregated pools
731
+ PUBLIC = {IN_PUBLIC, OUT_PUBLIC}
732
+ WORLD_NODE = {IN_WORLD_NODE, OUT_WORLD_NODE}
733
+ WORLD_AGENTS = {IN_WORLD_AGENTS, OUT_WORLD_AGENTS}
734
+ WORLD_MASTERS = {IN_WORLD_MASTERS, OUT_WORLD_MASTERS}
735
+ WORLD = WORLD_NODE | WORLD_AGENTS | WORLD_MASTERS
736
+ ALL = PUBLIC | WORLD
737
+ OUTGOING = {OUT_PUBLIC, OUT_WORLD_NODE, OUT_WORLD_AGENTS, OUT_WORLD_MASTERS}
738
+ INCOMING = {IN_PUBLIC, IN_WORLD_NODE, IN_WORLD_AGENTS, IN_WORLD_MASTERS}
739
+
740
+ def __init__(self, max_connections: int, p2p_u: P2P, p2p_w: P2P,
741
+ is_world_node: bool, public_key: str, token: str):
742
+ """Initializes a new instance of the NodeConn class.
743
+
744
+ Args:
745
+ max_connections: The total number of connections the node can handle.
746
+ p2p_u: The P2P object for the public network.
747
+ p2p_w: The P2P object for the world/private network.
748
+ is_world_node: A boolean flag indicating if this node is a world node.
749
+ public_key: The public key for token verification.
750
+ token: The node's authentication token.
751
+ """
752
+ super().__init__(max_connections=max_connections,
753
+ p2p_name_to_p2p={
754
+ NodeConn.P2P_PUBLIC: p2p_u,
755
+ NodeConn.P2P_WORLD: p2p_w,
756
+ },
757
+ pool_name_to_p2p_name_and_ratio={
758
+ NodeConn.IN_PUBLIC: [NodeConn.P2P_PUBLIC, 0.25 / 2. if not is_world_node else 0.25 / 2.],
759
+ NodeConn.OUT_PUBLIC: [NodeConn.P2P_PUBLIC, 0.25 / 2. if not is_world_node else 0.25 / 2.],
760
+ NodeConn.IN_WORLD_AGENTS: [NodeConn.P2P_WORLD, .75 / 2 if not is_world_node else 0.5 / 2],
761
+ NodeConn.OUT_WORLD_AGENTS: [NodeConn.P2P_WORLD, .75 / 2 if not is_world_node else 0.5 / 2],
762
+ NodeConn.IN_WORLD_NODE: [NodeConn.P2P_WORLD, 0. if not is_world_node else -1.],
763
+ NodeConn.OUT_WORLD_NODE: [NodeConn.P2P_WORLD, 0. if not is_world_node else -1],
764
+ NodeConn.IN_WORLD_MASTERS: [NodeConn.P2P_WORLD, 0. if not is_world_node else 0.25 / 2.],
765
+ NodeConn.OUT_WORLD_MASTERS: [NodeConn.P2P_WORLD, 0. if not is_world_node else 0.25 / 2.]
766
+ },
767
+ public_key=public_key, token=token)
768
+
769
+ # Just for convenience
770
+ self.p2p_public = p2p_u
771
+ self.p2p_world = p2p_w
772
+
773
+ # These are the list of all the possible agents that might try to connect when we are in world
774
+ self.world_agents_list = set()
775
+ self.world_masters_list = set()
776
+ self.world_agents_and_world_masters_list = set()
777
+ self.world_node_peer_id = None
778
+ self.inspector_peer_id = None
779
+ self.role_to_peer_ids = {}
780
+ self.peer_id_to_addrs = {}
781
+
782
+ # Rendezvous
783
+ self.rendezvous_tag = -1
784
+
785
+ def reset_rendezvous_tag(self):
786
+ """Resets the rendezvous tag to its initial state."""
787
+ self.rendezvous_tag = -1
788
+
789
+ async def conn_routing_fcn(self, connected_peer_infos: list, p2p: P2P):
790
+ """Routes connected peers to the correct connection pool based on their network and role (async).
791
+
792
+ Args:
793
+ connected_peer_infos: A list of dictionaries with information about connected peers.
794
+ p2p: The P2P network object where the connections were found.
795
+
796
+ Returns:
797
+ A dictionary mapping pool names to a dictionary of peer IDs and their information.
798
+ """
799
+ pool_name_and_peer_id_to_peer_info = {k: {} for k in self.p2p_to_pool_names[p2p]}
800
+ public = p2p == self.p2p_public
801
+
802
+ for c in connected_peer_infos:
803
+ inbound = c['direction'] == "incoming"
804
+ outbound = c['direction'] == "outgoing"
805
+ peer_id = c['id'] # Other fields are: c['addrs'], c['connected_at']
806
+
807
+ if public:
808
+ if inbound:
809
+ pool_name_and_peer_id_to_peer_info[NodeConn.IN_PUBLIC][peer_id] = c
810
+ elif outbound:
811
+ pool_name_and_peer_id_to_peer_info[NodeConn.OUT_PUBLIC][peer_id] = c
812
+ else:
813
+ raise ValueError(f"Connection direction is undefined: {c['direction']}")
814
+ else:
815
+ is_world_agent = peer_id in self.world_agents_list
816
+ is_world_master = peer_id in self.world_masters_list
817
+ is_world_node = self.world_node_peer_id is not None and peer_id == self.world_node_peer_id
818
+ is_inspector = self.inspector_peer_id is not None and peer_id == self.inspector_peer_id
819
+ if not is_world_node and not is_world_master and not is_world_agent and not is_inspector:
820
+ if ConnectionPools.DEBUG:
821
+ print("[DEBUG CONNECTIONS-POOL] World agents list: " + str(self.world_agents_list))
822
+ print("[DEBUG CONNECTIONS-POOL] World masters list: " + str(self.world_masters_list))
823
+ print("[DEBUG CONNECTIONS-POOL] World node peer id: " + str(self.world_node_peer_id))
824
+ print("[DEBUG CONNECTIONS-POOL] Inspector peer id: " + str(self.inspector_peer_id))
825
+ print(f"[DEBUG CONNECTIONS-POOL] Unable to determine the peer type for {peer_id}: "
826
+ f"cannot say if world agent, master, world node, inspector (disconnecting it)")
827
+ await ConnectionPools.disconnect(p2p, peer_id)
828
+ continue
829
+
830
+ if inbound:
831
+ pool_name_and_peer_id_to_peer_info[NodeConn.IN_WORLD_AGENTS if is_world_agent else (
832
+ NodeConn.IN_WORLD_NODE if is_world_node else
833
+ NodeConn.IN_WORLD_MASTERS)][peer_id] = c
834
+ elif outbound:
835
+ pool_name_and_peer_id_to_peer_info[NodeConn.OUT_WORLD_AGENTS if is_world_agent else (
836
+ NodeConn.OUT_WORLD_NODE if is_world_node else
837
+ NodeConn.OUT_WORLD_MASTERS)][peer_id] = c
838
+ else:
839
+ raise ValueError(f"Connection direction is undefined: {c}")
840
+
841
+ return pool_name_and_peer_id_to_peer_info
842
+
843
+ def set_world(self, world_peer_id: str | None):
844
+ """Sets the peer ID of the world node.
845
+
846
+ Args:
847
+ world_peer_id: The peer ID of the world node, or None to clear it.
848
+ """
849
+ self.world_node_peer_id = world_peer_id
850
+
851
+ def set_inspector(self, inspector_peer_id: str | None):
852
+ """Sets the peer ID of the inspector.
853
+
854
+ Args:
855
+ inspector_peer_id: The peer ID of the inspector node.
856
+ """
857
+ self.inspector_peer_id = inspector_peer_id
858
+
859
+ def get_world_peer_id(self):
860
+ """Returns the peer ID of the world node.
861
+
862
+ Args:
863
+ None.
864
+
865
+ Returns:
866
+ The world node's peer ID.
867
+ """
868
+ return self.world_node_peer_id
869
+
870
+ def set_addresses_in_peer_info(self, peer_id, addresses):
871
+ """Updates the list of addresses for a given peer.
872
+
873
+ Args:
874
+ peer_id: The peer ID to update.
875
+ addresses: A new list of addresses for the peer.
876
+ """
877
+ if self.in_connection_queues(peer_id):
878
+ addrs = self.pool_name_to_peer_infos[self.get_pool_of(peer_id)][peer_id]['addrs']
879
+ addrs.clear() # Warning: do not allocate a new list, keep the current one (it is referenced by others)
880
+ for _addrs in addresses:
881
+ addrs.append(_addrs)
882
+
883
+ def set_role(self, peer_id, new_role: int):
884
+ """Updates the role of a peer and its associated role-based lists.
885
+
886
+ Args:
887
+ peer_id: The peer ID to update.
888
+ new_role: The new role for the peer.
889
+ """
890
+ cur_role = self.get_role(peer_id)
891
+
892
+ # Updating
893
+ self.peer_id_to_misc[peer_id] = new_role
894
+
895
+ if self.in_connection_queues(peer_id):
896
+ self.pool_name_to_peer_infos[self.get_pool_of(peer_id)][peer_id]['misc'] = new_role
897
+
898
+ # Updating
899
+ if cur_role in self.role_to_peer_ids:
900
+ if peer_id in self.role_to_peer_ids[cur_role]:
901
+ self.role_to_peer_ids[cur_role].remove(peer_id)
902
+ if len(self.role_to_peer_ids[cur_role]) == 0:
903
+ del self.role_to_peer_ids[cur_role]
904
+ if new_role not in self.role_to_peer_ids:
905
+ self.role_to_peer_ids[new_role] = set()
906
+ self.role_to_peer_ids[new_role].add(peer_id)
907
+
908
+ def set_world_agents_list(self, world_agents_list_peer_infos: list[dict] | None):
909
+ """Sets the list of all world agents based on a provided list of peer information.
910
+
911
+ Args:
912
+ world_agents_list_peer_infos: A list of dictionaries containing peer information for world agents.
913
+ """
914
+
915
+ # Clearing previous information
916
+ to_remove = []
917
+ for peer_id, misc in self.peer_id_to_misc.items():
918
+ if misc & 1 == 1 and misc & 2 == 0:
919
+ to_remove.append((peer_id, misc))
920
+
921
+ for peer_id, misc in to_remove:
922
+ del self.peer_id_to_misc[peer_id]
923
+ if peer_id in self.peer_id_to_addrs:
924
+ del self.peer_id_to_addrs[peer_id]
925
+ self.role_to_peer_ids[misc].discard(peer_id)
926
+
927
+ # Setting new information
928
+ if world_agents_list_peer_infos is not None and len(world_agents_list_peer_infos) > 0:
929
+ self.world_agents_list = {x['id'] for x in world_agents_list_peer_infos}
930
+ for x in world_agents_list_peer_infos:
931
+ self.peer_id_to_addrs[x['id']] = x['addrs']
932
+ self.set_role(x['id'], x['misc'])
933
+ else:
934
+ self.world_agents_list = set()
935
+
936
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
937
+
938
+ def set_world_masters_list(self, world_masters_list_peer_infos: list[dict] | None):
939
+ """Sets the list of all world masters based on a provided list of peer information.
940
+
941
+ Args:
942
+ world_masters_list_peer_infos: A list of dictionaries containing peer information for world masters.
943
+ """
944
+
945
+ # Clearing previous information
946
+ to_remove = []
947
+ for peer_id, misc in self.peer_id_to_misc.items():
948
+ if misc & 1 == 1 and misc & 2 == 2:
949
+ to_remove.append((peer_id, misc))
950
+
951
+ for peer_id, misc in to_remove:
952
+ del self.peer_id_to_misc[peer_id]
953
+ if peer_id in self.peer_id_to_addrs:
954
+ del self.peer_id_to_addrs[peer_id]
955
+ self.role_to_peer_ids[misc].discard(peer_id)
956
+
957
+ # Setting new information
958
+ if world_masters_list_peer_infos is not None and len(world_masters_list_peer_infos) > 0:
959
+ self.world_masters_list = {x['id'] for x in world_masters_list_peer_infos}
960
+ for x in world_masters_list_peer_infos:
961
+ self.peer_id_to_addrs[x['id']] = x['addrs']
962
+ self.set_role(x['id'], x['misc'])
963
+ else:
964
+ self.world_masters_list = set()
965
+
966
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
967
+
968
+ def add_to_world_agents_list(self, peer_id: str, addrs: list[str], role: int = -1):
969
+ """Adds a new world agent to the list.
970
+
971
+ Args:
972
+ peer_id: The peer ID of the new agent.
973
+ addrs: A list of addresses for the new agent.
974
+ role: The role assigned to the agent.
975
+ """
976
+ self.world_agents_list.add(peer_id)
977
+
978
+ # This assumes that the WORLD MASTER/AGENT BIT is the first one
979
+ assert role & 1 == 1, "Expecting the first bit of the role to be 1 for world agents"
980
+ assert role & 2 == 0, "Expecting the second bit of the role to be 0 for world agents"
981
+ self.peer_id_to_addrs[peer_id] = addrs
982
+ self.set_role(peer_id, role)
983
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
984
+
985
+ def add_to_world_masters_list(self, peer_id: str, addrs: list[str], role: int = -1):
986
+ """Adds a new world master to the list.
987
+
988
+ Args:
989
+ peer_id: The peer ID of the new master.
990
+ addrs: A list of addresses for the new master.
991
+ role: The role assigned to the master.
992
+ """
993
+ self.world_masters_list.add(peer_id)
994
+
995
+ # This assumes that the WORLD MASTER/AGENT BIT is the first one
996
+ assert role & 1 == 1, "Expecting the first bit of the role to be 1 for world masters"
997
+ assert role & 2 == 2, "Expecting the second bit of the role to be 1 for world masters"
998
+ self.peer_id_to_addrs[peer_id] = addrs
999
+ self.set_role(peer_id, role)
1000
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
1001
+
1002
+ def get_added_after_updating(self, pool_names: list[str] | None = None):
1003
+ """Retrieves the set of peers added after the last update cycle for specified pools.
1004
+
1005
+ Args:
1006
+ pool_names: A list of pool names to check. If None, checks all pools.
1007
+
1008
+ Returns:
1009
+ A dictionary mapping pool names to sets of added peer IDs, or a single set if only one pool is specified.
1010
+ """
1011
+ if pool_names is not None:
1012
+ ret = {}
1013
+ for p in pool_names:
1014
+ ret[p] = super().get_added_after_updating(p)
1015
+ return ret
1016
+ else:
1017
+ return super().get_added_after_updating()
1018
+
1019
+ def get_removed_after_updating(self, pool_names: list[str] | None = None):
1020
+ """Retrieves the set of peers removed after the last update cycle for specified pools.
1021
+
1022
+ Args:
1023
+ pool_names: A list of pool names to check. If None, checks all pools.
1024
+
1025
+ Returns:
1026
+ A dictionary mapping pool names to sets of removed peer IDs, or a single set if only one pool is specified.
1027
+ """
1028
+ if pool_names is not None:
1029
+ ret = {}
1030
+ for p in pool_names:
1031
+ ret[p] = super().get_removed_after_updating(p)
1032
+ return ret
1033
+ else:
1034
+ return super().get_removed_after_updating()
1035
+
1036
+ def size(self, pool_names: list[str] | None = None):
1037
+ """Returns the total number of connections across all specified pools.
1038
+
1039
+ Args:
1040
+ pool_names: A list of pool names to sum the size of. If None, returns the total size of all pools.
1041
+
1042
+ Returns:
1043
+ The total number of connections.
1044
+ """
1045
+ if pool_names is not None:
1046
+ return super().size()
1047
+ else:
1048
+ c = 0
1049
+ for p in self.pool_names:
1050
+ c += super().size(p)
1051
+ return c
1052
+
1053
+ def is_connected(self, peer_id: str, pool_names: list[str] | None = None):
1054
+ """Checks if a peer is connected in any of the specified pools.
1055
+
1056
+ Args:
1057
+ peer_id: The peer ID to check.
1058
+ pool_names: A list of pool names to search within. If None, searches all pools.
1059
+
1060
+ Returns:
1061
+ True if the peer is found in any of the pools, otherwise False.
1062
+ """
1063
+ if pool_names is None:
1064
+ return super().is_connected(peer_id)
1065
+ else:
1066
+ for p in pool_names:
1067
+ if super().is_connected(peer_id, p):
1068
+ return True
1069
+ return False
1070
+
1071
+ def is_public(self, peer_id):
1072
+ """Checks if a peer is connected via the public network.
1073
+
1074
+ Args:
1075
+ peer_id: The peer ID to check.
1076
+
1077
+ Returns:
1078
+ True if the peer is in a public pool, otherwise False.
1079
+ """
1080
+ pool_name = self.get_pool_of(peer_id)
1081
+ return pool_name in NodeConn.PUBLIC
1082
+
1083
+ def is_world_master(self, peer_id):
1084
+ """Checks if a peer is a world master.
1085
+
1086
+ Args:
1087
+ peer_id: The peer ID to check.
1088
+
1089
+ Returns:
1090
+ True if the peer is in a world master pool, otherwise False.
1091
+ """
1092
+ pool_name = self.get_pool_of(peer_id)
1093
+ return pool_name in NodeConn.WORLD_MASTERS
1094
+
1095
+ def is_world_node(self, peer_id):
1096
+ """Checks if a peer is the world node.
1097
+
1098
+ Args:
1099
+ peer_id: The peer ID to check.
1100
+
1101
+ Returns:
1102
+ True if the peer is in a world node pool, otherwise False.
1103
+ """
1104
+ pool_name = self.get_pool_of(peer_id)
1105
+ return pool_name in NodeConn.WORLD_NODE
1106
+
1107
+ def is_in_world(self, peer_id):
1108
+ """Checks if a peer is connected to the world network.
1109
+
1110
+ Args:
1111
+ peer_id: The peer ID to check.
1112
+
1113
+ Returns:
1114
+ True if the peer is in any world pool, otherwise False.
1115
+ """
1116
+ pool_name = self.get_pool_of(peer_id)
1117
+ return pool_name in NodeConn.WORLD
1118
+
1119
+ def get_role(self, peer_id):
1120
+ """Retrieves the role of a given peer.
1121
+
1122
+ Args:
1123
+ peer_id: The peer ID to query.
1124
+
1125
+ Returns:
1126
+ The integer role of the peer.
1127
+ """
1128
+ role = self.peer_id_to_misc.get(peer_id, 0) # 0 means public
1129
+ assert role >= 0, "Expecting role to be >= 0"
1130
+ assert role & 1 != 0 or role == 0, "Expecting public role to be zero (all-zero-bits)"
1131
+ return role
1132
+
1133
+ def get_addrs(self, peer_id):
1134
+ """Retrieves the list of addresses for a given peer.
1135
+
1136
+ Args:
1137
+ peer_id: The peer ID to query.
1138
+
1139
+ Returns:
1140
+ A list of addresses for the peer.
1141
+ """
1142
+ return self.peer_id_to_addrs.get(peer_id)
1143
+
1144
+ def in_connection_queues(self, peer_id):
1145
+ """Checks if a peer ID exists in any connection pool.
1146
+
1147
+ Args:
1148
+ peer_id: The peer ID to check.
1149
+
1150
+ Returns:
1151
+ True if the peer is found in any pool, otherwise False.
1152
+ """
1153
+ return peer_id in self.peer_id_to_pool_name
1154
+
1155
+ def find_addrs_by_role(self, role, return_peer_ids_too: bool = False):
1156
+ """Finds all addresses of peers with a specific role.
1157
+
1158
+ Args:
1159
+ role: The integer role to search for.
1160
+ return_peer_ids_too: A boolean to also return the peer IDs.
1161
+
1162
+ Returns:
1163
+ A list of lists of addresses, and optionally a list of peer IDs.
1164
+ """
1165
+ if role in self.role_to_peer_ids:
1166
+ peer_ids = self.role_to_peer_ids[role]
1167
+ else:
1168
+ if not return_peer_ids_too:
1169
+ return []
1170
+ else:
1171
+ return [], []
1172
+ ret_addrs = []
1173
+ ret_peer_ids = []
1174
+ for peer_id in peer_ids:
1175
+ addrs = self.get_addrs(peer_id)
1176
+ if addrs is not None:
1177
+ ret_addrs.append(addrs)
1178
+ ret_peer_ids.append(peer_id)
1179
+ if not return_peer_ids_too:
1180
+ return ret_addrs
1181
+ else:
1182
+ return ret_addrs, ret_peer_ids
1183
+
1184
+ def count_by_role(self, role: int):
1185
+ """Counts the number of peers with a specific role.
1186
+
1187
+ Args:
1188
+ role: The integer role to count.
1189
+
1190
+ Returns:
1191
+ The number of peers with that role.
1192
+ """
1193
+ if role in self.role_to_peer_ids:
1194
+ return len(self.role_to_peer_ids[role])
1195
+ else:
1196
+ return 0
1197
+
1198
+ def get_all_connected_peer_infos(self, pool_names: list[str] | set[str]):
1199
+ """Retrieves a list of all peer info dictionaries for the specified pools.
1200
+
1201
+ Args:
1202
+ pool_names: A list or set of pool names to query.
1203
+
1204
+ Returns:
1205
+ A list of dictionaries containing peer information.
1206
+ """
1207
+ ret = []
1208
+ for p in pool_names:
1209
+ ret += super().get_all_connected_peer_infos(p)
1210
+ return ret
1211
+
1212
+ async def set_world_agents_and_world_masters_lists_from_rendezvous(self):
1213
+ """Updates the lists of world agents and masters using data from the rendezvous topic (async)."""
1214
+ rendezvous_state = self.p2p_world.get_rendezvous_peers_info()
1215
+
1216
+ if rendezvous_state is not None:
1217
+ tag = rendezvous_state.get('update_count', -1)
1218
+
1219
+ if tag > self.rendezvous_tag:
1220
+ self.rendezvous_tag = tag
1221
+ rendezvous_peer_infos = rendezvous_state.get('peers', [])
1222
+
1223
+ world_agents_peer_infos = []
1224
+ world_masters_peer_infos = []
1225
+
1226
+ if ConnectionPools.DEBUG:
1227
+ print(f"[DEBUG CONNECTIONS-POOL] Rendezvous peer infos (tag: {tag}, peers: "
1228
+ f"{len(rendezvous_peer_infos)} peers)")
1229
+
1230
+ for c in rendezvous_peer_infos:
1231
+ if c['addrs'] is None or len(c['addrs']) == 0:
1232
+ print(f"[DEBUG CONNECTIONS-POOL] Skipping a peer with missing addrs: {c}")
1233
+ continue
1234
+ if (c['misc'] & 1) == 1 and (c['misc'] & 2) == 0:
1235
+ world_agents_peer_infos.append(c)
1236
+ elif (c['misc'] & 1) == 1 and (c['misc'] & 2) == 2:
1237
+ world_masters_peer_infos.append(c)
1238
+ else:
1239
+ raise ValueError("Unexpected value of the 'misc' field: " + str(c))
1240
+
1241
+ # Updating lists
1242
+ self.set_world_agents_list(world_agents_peer_infos)
1243
+ self.set_world_masters_list(world_masters_peer_infos)
1244
+
1245
+ async def get_cv_hash_from_last_token(self, peer_id):
1246
+ """Retrieves the CV hash from the last token received from a peer (async).
1247
+
1248
+ Args:
1249
+ peer_id: The peer ID to query.
1250
+
1251
+ Returns:
1252
+ The CV hash string, or None if not found.
1253
+ """
1254
+ token = self.get_last_token(peer_id)
1255
+ if token is not None:
1256
+ _, cv_hash = await self.verify_token(token, peer_id)
1257
+ return cv_hash
1258
+ else:
1259
+ return None
1260
+
1261
+ async def remove(self, peer_id: str):
1262
+ """Removes a peer and its associated information from all lists and pools (async).
1263
+
1264
+ Args:
1265
+ peer_id: The peer ID to remove.
1266
+ """
1267
+ await super().remove(peer_id)
1268
+ if peer_id in self.peer_id_to_addrs:
1269
+ del self.peer_id_to_addrs[peer_id]
1270
+
1271
+ async def remove_all_world_agents(self):
1272
+ """Removes all connected world agents from the pools and role lists (async)."""
1273
+ peer_infos = self.get_all_connected_peer_infos(NodeConn.WORLD)
1274
+ for c in peer_infos:
1275
+ peer_id = c['id']
1276
+ await self.remove(peer_id)
1277
+ for role, peer_ids in self.role_to_peer_ids.items():
1278
+ if role & 1 == NodeConn.WORLD:
1279
+ peer_ids.remove(peer_id)
1280
+
1281
+ async def subscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
1282
+ """Subscribes to a channel, defaulting to the world P2P network if a network is not specified (async).
1283
+
1284
+ Args:
1285
+ peer_id: The peer ID associated with the channel.
1286
+ channel: The channel to subscribe to.
1287
+ default_p2p_name: An optional P2P name to use for the subscription.
1288
+
1289
+ Returns:
1290
+ True if successful, False otherwise.
1291
+ """
1292
+ return await super().subscribe(peer_id, channel,
1293
+ default_p2p_name=NodeConn.P2P_WORLD
1294
+ if default_p2p_name is None else default_p2p_name)
1295
+
1296
+ async def get_messages(self, p2p_name: str, allowed_not_connected_peers: set | None = None) -> list[Msg]:
1297
+ """Retrieves messages, allowing for messages from known world agents and masters even if not in a
1298
+ connection pool (async).
1299
+
1300
+ Args:
1301
+ p2p_name: The name of the P2P network to get messages from.
1302
+ allowed_not_connected_peers: This parameter is ignored in this implementation.
1303
+
1304
+ Returns:
1305
+ A list of verified and processed message objects.
1306
+ """
1307
+ assert allowed_not_connected_peers is None, "This param (allowed_not_connected_peers is ignored in NodeConn"
1308
+ return await super().get_messages(p2p_name, allowed_not_connected_peers=self.world_agents_and_world_masters_list)