unaiverse 0.1.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. unaiverse/__init__.py +19 -0
  2. unaiverse/agent.py +2226 -0
  3. unaiverse/agent_basics.py +2389 -0
  4. unaiverse/clock.py +234 -0
  5. unaiverse/dataprops.py +1282 -0
  6. unaiverse/hsm.py +2471 -0
  7. unaiverse/modules/__init__.py +18 -0
  8. unaiverse/modules/cnu/__init__.py +17 -0
  9. unaiverse/modules/cnu/cnus.py +536 -0
  10. unaiverse/modules/cnu/layers.py +261 -0
  11. unaiverse/modules/cnu/psi.py +60 -0
  12. unaiverse/modules/hl/__init__.py +15 -0
  13. unaiverse/modules/hl/hl_utils.py +411 -0
  14. unaiverse/modules/networks.py +1509 -0
  15. unaiverse/modules/utils.py +748 -0
  16. unaiverse/networking/__init__.py +16 -0
  17. unaiverse/networking/node/__init__.py +18 -0
  18. unaiverse/networking/node/connpool.py +1332 -0
  19. unaiverse/networking/node/node.py +2752 -0
  20. unaiverse/networking/node/profile.py +446 -0
  21. unaiverse/networking/node/tokens.py +79 -0
  22. unaiverse/networking/p2p/__init__.py +188 -0
  23. unaiverse/networking/p2p/go.mod +127 -0
  24. unaiverse/networking/p2p/go.sum +548 -0
  25. unaiverse/networking/p2p/golibp2p.py +18 -0
  26. unaiverse/networking/p2p/golibp2p.pyi +136 -0
  27. unaiverse/networking/p2p/lib.go +2765 -0
  28. unaiverse/networking/p2p/lib_types.py +311 -0
  29. unaiverse/networking/p2p/message_pb2.py +50 -0
  30. unaiverse/networking/p2p/messages.py +360 -0
  31. unaiverse/networking/p2p/mylogger.py +78 -0
  32. unaiverse/networking/p2p/p2p.py +900 -0
  33. unaiverse/networking/p2p/proto-go/message.pb.go +846 -0
  34. unaiverse/stats.py +1506 -0
  35. unaiverse/streamlib/__init__.py +15 -0
  36. unaiverse/streamlib/streamlib.py +210 -0
  37. unaiverse/streams.py +804 -0
  38. unaiverse/utils/__init__.py +16 -0
  39. unaiverse/utils/lone_wolf.json +28 -0
  40. unaiverse/utils/misc.py +441 -0
  41. unaiverse/utils/sandbox.py +292 -0
  42. unaiverse/world.py +384 -0
  43. unaiverse-0.1.12.dist-info/METADATA +366 -0
  44. unaiverse-0.1.12.dist-info/RECORD +47 -0
  45. unaiverse-0.1.12.dist-info/WHEEL +5 -0
  46. unaiverse-0.1.12.dist-info/licenses/LICENSE +177 -0
  47. unaiverse-0.1.12.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1332 @@
1
+ """
2
+ █████ █████ ██████ █████ █████ █████ █████ ██████████ ███████████ █████████ ██████████
3
+ ░░███ ░░███ ░░██████ ░░███ ░░███ ░░███ ░░███ ░░███░░░░░█░░███░░░░░███ ███░░░░░███░░███░░░░░█
4
+ ░███ ░███ ░███░███ ░███ ██████ ░███ ░███ ░███ ░███ █ ░ ░███ ░███ ░███ ░░░ ░███ █ ░
5
+ ░███ ░███ ░███░░███░███ ░░░░░███ ░███ ░███ ░███ ░██████ ░██████████ ░░█████████ ░██████
6
+ ░███ ░███ ░███ ░░██████ ███████ ░███ ░░███ ███ ░███░░█ ░███░░░░░███ ░░░░░░░░███ ░███░░█
7
+ ░███ ░███ ░███ ░░█████ ███░░███ ░███ ░░░█████░ ░███ ░ █ ░███ ░███ ███ ░███ ░███ ░ █
8
+ ░░████████ █████ ░░█████░░████████ █████ ░░███ ██████████ █████ █████░░█████████ ██████████
9
+ ░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░ ░░░░░ ░░░ ░░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░ ░░░░░░░░░░
10
+ A Collectionless AI Project (https://collectionless.ai)
11
+ Registration/Login: https://unaiverse.io
12
+ Code Repositories: https://github.com/collectionlessai/
13
+ Main Developers: Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
14
+ """
15
+ import math
16
+ import base64
17
+ import binascii
18
+ from unaiverse.networking.p2p.messages import Msg
19
+ from unaiverse.networking.p2p.p2p import P2P, P2PError
20
+ from unaiverse.networking.node.tokens import TokenVerifier
21
+
22
+
23
+ class ConnectionPools:
24
+ DEBUG = True
25
+
26
+ def __init__(self, max_connections: int, pool_name_to_p2p_name_and_ratio: dict[str, [str, float]],
27
+ p2p_name_to_p2p: dict[str, P2P], public_key: str | None = None, token: str | None = None):
28
+ """Initializes a new instance of the ConnectionPools class.
29
+
30
+ Args:
31
+ max_connections: The maximum total number of connections allowed across all pools.
32
+ pool_name_to_p2p_name_and_ratio: A dictionary mapping pool names to a list containing the associated P2P
33
+ network name and its connection ratio.
34
+ p2p_name_to_p2p: A dictionary mapping P2P network names to their corresponding P2P objects.
35
+ public_key: An optional public key for token verification.
36
+ token: An optional initial token for authentication.
37
+
38
+ Returns:
39
+ None.
40
+
41
+ """
42
+ # Common terms: a "pool triple" is [pool_contents, max_connections_in_such_a_pool, p2p_object_of_the_pool
43
+ self.max_con = max_connections
44
+ self.pool_count = len(pool_name_to_p2p_name_and_ratio)
45
+ self.pool_names = list(pool_name_to_p2p_name_and_ratio.keys())
46
+ self.pool_ratios = [p2p_name_and_ratio[1] for p2p_name_and_ratio in pool_name_to_p2p_name_and_ratio.values()]
47
+
48
+ # Indices involving the P2P object or its name
49
+ self.p2p_name_to_p2p = p2p_name_to_p2p
50
+ self.p2p_name_and_pool_name_to_pool_triple = {}
51
+ self.p2p_to_pool_names = {}
52
+
53
+ # Indices rooted around the pool name
54
+ self.pool_name_to_pool_triple = {}
55
+ self.pool_name_to_added_in_last_update = {}
56
+ self.pool_name_to_removed_in_last_update = {}
57
+ self.pool_name_to_peer_infos = {p: {} for p in self.pool_names}
58
+
59
+ # Indices rooted around the peer ID
60
+ self.peer_id_to_pool_name = {}
61
+ self.peer_id_to_p2p = {}
62
+ self.peer_id_to_misc = {}
63
+ self.peer_id_to_token = {}
64
+
65
+ # Token-related stuff, super private
66
+ self.__token = token if token is not None else ""
67
+ self.__token_verifier = TokenVerifier(public_key) if public_key is not None else None
68
+
69
+ # Checking
70
+ for p2p_name_and_ratio in pool_name_to_p2p_name_and_ratio.values():
71
+ assert p2p_name_and_ratio[0] in self.p2p_name_to_p2p, f"Cannot find p2p named {p2p_name_and_ratio[0]} "
72
+ assert self.max_con >= len(self.pool_names), "Too small number of max connections"
73
+ assert sum([x for x in self.pool_ratios if x > 0]) == 1.0, "Pool ratios must sum to 1.0"
74
+
75
+ # Preparing the pool triples
76
+ self.pool_name_to_pool_triple = \
77
+ {k: [set(), 0, self.p2p_name_to_p2p[pool_name_to_p2p_name_and_ratio[k][0]]] for k in self.pool_names}
78
+ num_zero_ratio_pools = len([x for x in self.pool_ratios if x == 0])
79
+ assert num_zero_ratio_pools <= self.max_con, "Cannot create pools given the provided max connection count"
80
+
81
+ # Edit: to solve the teacher not engaging with more than two students.
82
+ pools_max_sizes = {k: max(math.floor(self.pool_ratios[i] * (self.max_con - num_zero_ratio_pools)),
83
+ 1 if self.pool_ratios[i] >= 0. else 0)
84
+ for i, k in enumerate(self.pool_names)}
85
+
86
+ # Pools_max_sizes = {k: self.max_con for k in self.pool_names}
87
+
88
+ # Fixing sizes
89
+ tot = 0
90
+ for i, (k, v) in enumerate(pools_max_sizes.items()):
91
+ assert v > 0 or self.pool_ratios[i] < 0, "Cannot create pools given the provided max connection count"
92
+
93
+ # Edit: to solve the teacher not engaging with more than two students.
94
+ tot += v
95
+ assert tot <= self.max_con, \
96
+ "Cannot create pools given the provided max connection count"
97
+ pools_max_sizes[self.pool_names[-1]] += (self.max_con - tot)
98
+
99
+ # Storing fixed sizes in the previously created pool triples & building additional index
100
+ for pool_name, pool_contents_max_con_and_p2p in self.pool_name_to_pool_triple.items():
101
+ pool_contents_max_con_and_p2p[1] = pools_max_sizes[pool_name] # Fixing the second element of the triple
102
+
103
+ pool, _, p2p = pool_contents_max_con_and_p2p
104
+ p2p_name = None
105
+ for k, v in self.p2p_name_to_p2p.items():
106
+ if v == p2p:
107
+ p2p_name = k
108
+ break
109
+ if p2p_name not in self.p2p_name_and_pool_name_to_pool_triple:
110
+ self.p2p_name_and_pool_name_to_pool_triple[p2p_name] = {}
111
+ self.p2p_to_pool_names[p2p] = []
112
+ self.p2p_name_and_pool_name_to_pool_triple[p2p_name][pool_name] = (
113
+ pool_contents_max_con_and_p2p)
114
+ self.p2p_to_pool_names[p2p].append(pool_name)
115
+
116
+ def __str__(self):
117
+ """Returns a human-readable string representation of the connection pools' status.
118
+
119
+ Args:
120
+ None.
121
+
122
+ Returns:
123
+ A formatted string showing the number of connections and peer IDs in each pool.
124
+ """
125
+ max_len = max(len(s) for s in self.pool_names)
126
+ s = f"[ConnectionPool] max_con={self.max_con}, pool_count={self.pool_count}"
127
+ for k, (pool, max_con, _) in self.pool_name_to_pool_triple.items():
128
+ kk = k + ","
129
+ s += f"\n\tpool={kk.ljust(max_len + 1)}\tmax_con={max_con},\tcurrent_con={len(pool)},\tpeer_ids="
130
+ ss = "("
131
+ first = True
132
+ for c in pool:
133
+ if not first:
134
+ ss += ", "
135
+ ss += c
136
+ first = False
137
+ ss += ")"
138
+ s += ss
139
+ return s
140
+
141
+ def __getitem__(self, p2p_name):
142
+ """Retrieves a P2P object by its name.
143
+
144
+ Args:
145
+ p2p_name: The name of the P2P network.
146
+
147
+ Returns:
148
+ The P2P object.
149
+ """
150
+ return self.p2p_name_to_p2p[p2p_name]
151
+
152
+ async def conn_routing_fcn(self, connected_peer_infos: list, p2p: P2P):
153
+ """A placeholder function that must be implemented to route connected peers to the correct pool (async).
154
+
155
+ Args:
156
+ connected_peer_infos: A list of dictionaries containing information about connected peers.
157
+ p2p: The P2P network object from which the peers were connected.
158
+
159
+ Returns:
160
+ A dictionary mapping pool names to a dictionary of peer IDs and their information.
161
+ """
162
+ raise NotImplementedError("You must implement conn_routing_fcn!")
163
+
164
+ @staticmethod
165
+ async def __connect(p2p: P2P, addresses: list[str]):
166
+ """Establishes a connection to a peer via a P2P network (async).
167
+
168
+ Args:
169
+ p2p: The P2P network object to use for the connection.
170
+ addresses: A list of addresses of the peer to connect to.
171
+
172
+ Returns:
173
+ A tuple containing the peer ID and a boolean indicating if the connection was established through a relay.
174
+ """
175
+
176
+ force: str | None = None
177
+ # force: str | None = '/tls/ws'
178
+ if force is not None:
179
+ _addresses = [a for a in addresses if force in a]
180
+ addresses.clear()
181
+ for a in _addresses:
182
+ addresses.append(a)
183
+
184
+ if ConnectionPools.DEBUG:
185
+ print(f"[DEBUG CONNECTIONS-POOL] Connecting to {addresses}")
186
+
187
+ if addresses is None or len(addresses) == 0:
188
+ if ConnectionPools.DEBUG:
189
+ print(f"[DEBUG CONNECTIONS-POOL] Connection failed! (not-event tried, invalid addresses: {addresses})")
190
+ return None, False
191
+
192
+ try:
193
+ winning_addr_info_dict = p2p.connect_to(addresses)
194
+ peer_id = winning_addr_info_dict.get('ID')
195
+ connected_addr_str = winning_addr_info_dict.get('Addrs')[0]
196
+ through_relay = '/p2p-circuit/' in connected_addr_str
197
+ if ConnectionPools.DEBUG:
198
+ print(f"[DEBUG CONNECTIONS-POOL] Connected to peer {peer_id} via {connected_addr_str} "
199
+ f"(through relay: {through_relay})")
200
+
201
+ return peer_id, through_relay
202
+ except P2PError:
203
+ if ConnectionPools.DEBUG:
204
+ print(f"[DEBUG CONNECTIONS-POOL] Connection failed!")
205
+ return None, False
206
+
207
+ @staticmethod
208
+ async def disconnect(p2p: P2P, peer_id: str):
209
+ """Disconnects from a specific peer on a P2P network (async).
210
+
211
+ Args:
212
+ p2p: The P2P network object to use for disconnection.
213
+ peer_id: The peer ID to disconnect from.
214
+
215
+ Returns:
216
+ True if the disconnection is successful, otherwise False.
217
+ """
218
+ try:
219
+ p2p.disconnect_from(peer_id)
220
+ except P2PError:
221
+ return False
222
+ return True
223
+
224
+ def set_token(self, token: str):
225
+ """Sets the authentication token for the connection pools.
226
+
227
+ Args:
228
+ token: The new token string.
229
+ """
230
+ self.__token = token
231
+
232
+ async def verify_token(self, token: str, peer_id: str):
233
+ """Verifies a received token using the provided public key (async).
234
+
235
+ Args:
236
+ token: The token string to verify.
237
+ peer_id: The peer ID associated with the token.
238
+
239
+ Returns:
240
+ A tuple containing the node ID and CV hash if the token is valid, otherwise None.
241
+ """
242
+ if self.__token_verifier is None:
243
+ return None
244
+ else:
245
+ node_id, cv_hash = self.__token_verifier.verify_token(token, p2p_peer=peer_id)
246
+ return node_id, cv_hash # If the verification fails, this is None, None
247
+
248
+ async def connect(self, addresses: list[str], p2p_name: str):
249
+ """Connects to a peer on a specified P2P network (async).
250
+
251
+ Args:
252
+ addresses: A list of addresses of the peer to connect to.
253
+ p2p_name: The name of the P2P network to use.
254
+
255
+ Returns:
256
+ A tuple containing the peer ID of the connected peer and a boolean indicating if a relay was used.
257
+ """
258
+ p2p = self.p2p_name_to_p2p[p2p_name]
259
+
260
+ # Connecting
261
+ peer_id, through_relay = await ConnectionPools.__connect(p2p, addresses)
262
+ return peer_id, through_relay
263
+
264
+ def add(self, peer_info: dict, pool_name: str):
265
+ """Adds a connected peer to a specified connection pool.
266
+
267
+ Args:
268
+ peer_info: A dictionary containing information about the peer.
269
+ pool_name: The name of the pool to add the peer to.
270
+
271
+ Returns:
272
+ True if the peer is successfully added, otherwise False.
273
+ """
274
+ peer_id = peer_info['id']
275
+ pool, max_size, p2p = self.pool_name_to_pool_triple[pool_name]
276
+ if len(pool) < max_size:
277
+
278
+ # "hoping" peer IDs are unique, and stopping duplicate cases
279
+ if peer_id in self.peer_id_to_pool_name and self.peer_id_to_pool_name[peer_id] != pool_name:
280
+ print("VAFFANCULO")
281
+ return False
282
+
283
+ self.peer_id_to_pool_name[peer_id] = pool_name
284
+ self.peer_id_to_p2p[peer_id] = p2p
285
+
286
+ # Setting 'misc' field (default is 0, where 0 means public)
287
+ peer_info['misc'] = self.peer_id_to_misc.get(peer_id, 0)
288
+
289
+ # Storing (only)
290
+ pool.add(peer_id)
291
+ self.pool_name_to_peer_infos[pool_name][peer_id] = peer_info
292
+ return True
293
+ else:
294
+ print(f"VAFFANCULO pool_name={pool_name}, len(pool)={len(pool)}, max_size={max_size}")
295
+ return False
296
+
297
+ async def remove(self, peer_id: str):
298
+ """Removes a peer from its connection pool and disconnects from it (async).
299
+
300
+ Args:
301
+ peer_id: The peer ID to remove.
302
+
303
+ Returns:
304
+ True if the peer is successfully removed, otherwise False.
305
+ """
306
+ if peer_id in self.peer_id_to_pool_name:
307
+ pool_name = self.peer_id_to_pool_name[peer_id]
308
+ pool, _, p2p = self.pool_name_to_pool_triple[pool_name]
309
+
310
+ # Disconnecting
311
+ disc = await ConnectionPools.disconnect(p2p, peer_id)
312
+ pool.remove(peer_id)
313
+ del self.pool_name_to_peer_infos[pool_name][peer_id]
314
+ del self.peer_id_to_pool_name[peer_id]
315
+ del self.peer_id_to_p2p[peer_id]
316
+ if peer_id in self.peer_id_to_token:
317
+ del self.peer_id_to_token[peer_id]
318
+
319
+ # Remember to NOT del peer_id_to_misc[peer_id]!
320
+ return disc
321
+ else:
322
+ return False
323
+
324
+ def get_all_connected_peer_infos(self, pool_name: str):
325
+ """Retrieves a list of peer information dictionaries for a given pool.
326
+
327
+ Args:
328
+ pool_name: The name of the pool to query.
329
+
330
+ Returns:
331
+ A list of dictionaries, each containing information about a peer in the pool.
332
+ """
333
+ return list(self.pool_name_to_peer_infos[pool_name].values())
334
+
335
+ def get_pool_status(self):
336
+ """Returns a dictionary showing the set of peer IDs in each pool.
337
+
338
+ Args:
339
+ None.
340
+
341
+ Returns:
342
+ A dictionary mapping pool names to the set of peer IDs in that pool.
343
+ """
344
+ return {k: v[0] for k, v in self.pool_name_to_pool_triple.items()}
345
+
346
+ def get_all_connected_peer_ids(self):
347
+ """Retrieves a list of all peer IDs currently connected across all pools.
348
+
349
+ Args:
350
+ None.
351
+
352
+ Returns:
353
+ A list of all connected peer IDs.
354
+ """
355
+ return list(self.peer_id_to_pool_name.keys())
356
+
357
+ async def update(self):
358
+ """Refreshes the connection pools by checking for new and lost connections (async).
359
+
360
+ Args:
361
+ None.
362
+
363
+ Returns:
364
+ A tuple containing two dictionaries: one for newly added peers and one for removed peers, both keyed by
365
+ pool name.
366
+ """
367
+ self.pool_name_to_added_in_last_update = {}
368
+ self.pool_name_to_removed_in_last_update = {}
369
+
370
+ for p2p_name, p2p in self.p2p_name_to_p2p.items():
371
+ connected_peer_infos = p2p.get_connected_peers_info()
372
+
373
+ if connected_peer_infos is not None:
374
+
375
+ # Routing to the right queue / filtering
376
+ pool_name_and_peer_ids_to_peer_info = await self.conn_routing_fcn(connected_peer_infos, p2p)
377
+
378
+ # Parsing the generated index
379
+ for pool_name, connected_peer_ids_to_connected_peer_infos \
380
+ in pool_name_and_peer_ids_to_peer_info.items():
381
+ pool, _, pool_p2p = self.p2p_name_and_pool_name_to_pool_triple[p2p_name][pool_name]
382
+ connected_peer_ids = connected_peer_ids_to_connected_peer_infos.keys()
383
+ new_peer_ids = connected_peer_ids - pool
384
+ lost_peer_ids = pool - connected_peer_ids
385
+
386
+ # Clearing disconnected agents
387
+ for lost_peer_id in lost_peer_ids:
388
+ self.pool_name_to_removed_in_last_update.setdefault(pool_name, set()).add(lost_peer_id)
389
+
390
+ # Adding new agents
391
+ for new_peer_id in new_peer_ids:
392
+ peer_info = connected_peer_ids_to_connected_peer_infos[new_peer_id]
393
+ if not self.add(peer_info, pool_name=pool_name):
394
+ break
395
+ self.pool_name_to_added_in_last_update.setdefault(pool_name, set()).add(new_peer_id)
396
+
397
+ return self.pool_name_to_added_in_last_update, self.pool_name_to_removed_in_last_update
398
+
399
+ async def get_messages(self, p2p_name: str, allowed_not_connected_peers: set | None = None) -> list[Msg]:
400
+ """Retrieves and verifies all messages from a specified P2P network (async).
401
+
402
+ Args:
403
+ p2p_name: The name of the P2P network to fetch messages from.
404
+ allowed_not_connected_peers: An optional set of peer IDs to allow messages from, even if they are not
405
+ in the pools.
406
+
407
+ Returns:
408
+ A list of verified and processed message objects.
409
+ """
410
+ # Pop all messages
411
+ byte_messages: list[bytes] = self[p2p_name].pop_messages() # Pop all messages
412
+ # Process the list of message dictionaries
413
+ processed_messages: list[Msg] = []
414
+ for i, msg_dict in enumerate(byte_messages):
415
+ try:
416
+ # Extract and validate required fields from the Go message structure
417
+ # Go structure: {"from":"Qm...", "data":"BASE64_ENCODED_DATA"}
418
+ verified_sender_id = msg_dict.get("from")
419
+ base64_data = msg_dict.get("data")
420
+
421
+ # Decode data
422
+ decoded_data = base64.b64decode(base64_data)
423
+
424
+ # Attempt to create the higher-level Msg object
425
+ # This assumes Msg.from_bytes can parse your message protocol from decoded_data
426
+ # and that Msg objects store sender, type, channel intrinsically or can be set.
427
+ msg_obj = Msg.from_bytes(decoded_data)
428
+
429
+ # --- CRITICAL SECURITY CHECK ---
430
+ # Verify that the sender claimed inside the message payload
431
+ # matches the cryptographically verified sender from the network layer.
432
+ if msg_obj.sender != verified_sender_id:
433
+ print(f"[DEBUG CONNECTIONS-POOL] SENDER MISMATCH! Network sender '{verified_sender_id}' "
434
+ f"does not match payload sender '{msg_obj.sender}'. Discarding message.")
435
+
436
+ # In a real-world scenario, you might also want to penalize or disconnect
437
+ # from a peer that sends such malformed/spoofed messages.
438
+ continue # Discard this message
439
+
440
+ # filter only valid messages to return
441
+ if (msg_obj.sender in self.peer_id_to_pool_name or # Check if expected sender
442
+ (allowed_not_connected_peers is not None and msg_obj.sender in allowed_not_connected_peers)):
443
+
444
+ try:
445
+ token_with_inspector_final_bit = msg_obj.piggyback
446
+ token = token_with_inspector_final_bit[0:-1]
447
+ inspector_mode = token_with_inspector_final_bit[-1]
448
+ node_id, _ = await self.verify_token(token, msg_obj.sender)
449
+ if node_id is not None:
450
+ # Replacing piggyback with the node ID and the flag telling if it is inspector
451
+ msg_obj.piggyback = node_id + inspector_mode
452
+ processed_messages.append(msg_obj)
453
+ if msg_obj.sender in self.peer_id_to_pool_name:
454
+ self.peer_id_to_token[msg_obj.sender] = token
455
+ else:
456
+ print("Received a message missing expected info in the token payload (discarding it)")
457
+ except Exception as e:
458
+ print(f"Received a message with an invalid piggyback token! (discarding it) [{e}]")
459
+
460
+ except ValueError as ve:
461
+ print(f"[DEBUG CONNECTIONS-POOL]Invalid message created, stopping. Error: {ve}")
462
+ continue # Skip problematic message
463
+ except (TypeError, binascii.Error) as decode_err:
464
+ print(f"[DEBUG CONNECTIONS-POOL] Failed to decode Base64 data for a message in batch: {decode_err}. "
465
+ f"Message dict: {msg_dict}")
466
+ continue # Skip problematic message
467
+ except Exception as msg_proc_err: # Catch errors from Msg.from_bytes or attribute setting
468
+ print(f"[DEBUG CONNECTIONS-POOL] Error processing popped message item {i}: {msg_proc_err}. "
469
+ f"Message dict: {msg_dict}")
470
+ continue # Skip problematic message
471
+
472
+ return processed_messages
473
+
474
+ def get_added_after_updating(self, pool_name: str | None = None):
475
+ """Retrieves the peers that were added in the last update cycle.
476
+
477
+ Args:
478
+ pool_name: The name of a specific pool to query. If None, returns data for all pools.
479
+
480
+ Returns:
481
+ A set of added peer IDs for the specified pool, or a dictionary of sets for all pools.
482
+ """
483
+ if pool_name is not None:
484
+ return self.pool_name_to_added_in_last_update[pool_name]
485
+ else:
486
+ return self.pool_name_to_added_in_last_update
487
+
488
+ def get_removed_after_updating(self, pool_name: str | None = None):
489
+ """Retrieves the peers that were removed in the last update cycle.
490
+
491
+ Args:
492
+ pool_name: The name of a specific pool to query. If None, returns data for all pools.
493
+
494
+ Returns:
495
+ A set of removed peer IDs for the specified pool, or a dictionary of sets for all pools.
496
+ """
497
+ if pool_name is not None:
498
+ return self.pool_name_to_removed_in_last_update[pool_name]
499
+ else:
500
+ return self.pool_name_to_removed_in_last_update
501
+
502
+ def get_last_token(self, peer_id):
503
+ """Retrieves the last known token for a given peer.
504
+
505
+ Args:
506
+ peer_id: The peer ID to query.
507
+
508
+ Returns:
509
+ The token string if found, otherwise None.
510
+ """
511
+ return self.peer_id_to_token[peer_id] if peer_id in self.peer_id_to_token else None
512
+
513
+ def is_connected(self, peer_id: str, pool_name: str | None = None):
514
+ """Checks if a peer is currently connected, optionally in a specific pool.
515
+
516
+ Args:
517
+ peer_id: The peer ID to check.
518
+ pool_name: An optional pool name to check within.
519
+
520
+ Returns:
521
+ True if the peer is connected, otherwise False.
522
+ """
523
+ if pool_name is None:
524
+ return peer_id in self.peer_id_to_pool_name
525
+ else:
526
+ return peer_id in self.peer_id_to_pool_name and pool_name == self.peer_id_to_pool_name[peer_id]
527
+
528
+ def get_pool_of(self, peer_id: str):
529
+ """Gets the pool name for a given connected peer.
530
+
531
+ Args:
532
+ peer_id: The peer ID to query.
533
+
534
+ Returns:
535
+ The name of the pool the peer is in.
536
+ """
537
+ if peer_id in self.peer_id_to_pool_name:
538
+ return self.peer_id_to_pool_name[peer_id]
539
+ else:
540
+ return None
541
+
542
+ def size(self, pool_name: str | None = None):
543
+ """Returns the number of connections in a specific pool or the total number across all pools.
544
+
545
+ Args:
546
+ pool_name: An optional pool name to get the size of. If None, returns the total size.
547
+
548
+ Returns:
549
+ The size of the pool or the total number of connections.
550
+ """
551
+ if pool_name is not None:
552
+ return len(self.pool_name_to_pool_triple[pool_name])
553
+ else:
554
+ c = 0
555
+ for v in self.pool_name_to_pool_triple.values():
556
+ c += len(v)
557
+ return c
558
+
559
+ async def send(self, peer_id: str, channel_trail: str | None,
560
+ content_type: str, content: bytes | dict | None = None, p2p: P2P | None = None):
561
+ """Sends a direct message to a specific peer (async).
562
+
563
+ Args:
564
+ peer_id: The peer ID to send the message to.
565
+ channel_trail: An optional string to append to the channel name.
566
+ content_type: The type of content in the message.
567
+ content: The message content.
568
+ p2p: An optional P2P object to use for sending. If None, it is derived from the peer_id.
569
+
570
+ Returns:
571
+ True if the message is sent successfully, otherwise False.
572
+ """
573
+ # Getting the right p2p object
574
+ if p2p is None:
575
+ p2p = self.peer_id_to_p2p[peer_id] if peer_id in self.peer_id_to_p2p else None
576
+ if p2p is None:
577
+ if ConnectionPools.DEBUG:
578
+ print("[DEBUG CONNECTIONS-POOL] P2P non found for peer id: " + str(peer_id))
579
+ return False
580
+
581
+ # Defining channel
582
+ if channel_trail is not None and len(channel_trail) > 0:
583
+ channel = f"{p2p.peer_id}::dm:{peer_id}-{content_type}~{channel_trail}"
584
+ else:
585
+ channel = f"{p2p.peer_id}::dm:{peer_id}-{content_type}"
586
+
587
+ # Adding sender info here
588
+ msg = Msg(sender=p2p.peer_id,
589
+ content_type=content_type,
590
+ content=content,
591
+ channel=channel,
592
+ piggyback=self.__token + "0") # Adding inspector-mode bit (dummy bit here)
593
+ if ConnectionPools.DEBUG:
594
+ print("[DEBUG CONNECTIONS-POOL] Sending message: " + str(msg))
595
+
596
+ # Sending direct message
597
+ try:
598
+ p2p.send_message_to_peer(channel, msg_bytes=msg.to_bytes())
599
+
600
+ # If the line above executes without raising an error, it was successful.
601
+ return True
602
+ except P2PError as e:
603
+
604
+ # If send_message_to_peer fails, it will raise a P2PError. We catch it here.
605
+ if ConnectionPools.DEBUG:
606
+ print("[DEBUG CONNECTIONS-POOL] Sending error is: " + str(e))
607
+ return False
608
+
609
+ async def subscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
610
+ """Subscribes to a topic/channel on a P2P network (async).
611
+
612
+ Args:
613
+ peer_id: The peer ID associated with the topic/channel.
614
+ channel: The name of the channel to subscribe to.
615
+ default_p2p_name: An optional P2P network name to use if the peer's network is unknown.
616
+
617
+ Returns:
618
+ True if the subscription is successful, otherwise False.
619
+ """
620
+
621
+ # Getting the right p2p object
622
+ p2p = None
623
+ for _p2p in self.p2p_to_pool_names.keys():
624
+ if _p2p.peer_id == peer_id:
625
+ p2p = _p2p
626
+ break
627
+ if p2p is None and peer_id in self.peer_id_to_p2p:
628
+ p2p = self.peer_id_to_p2p[peer_id]
629
+ if p2p is None:
630
+ if default_p2p_name is not None:
631
+ p2p = self.p2p_name_to_p2p[default_p2p_name]
632
+ else:
633
+ return False
634
+
635
+ try:
636
+ p2p.subscribe_to_topic(channel)
637
+ except (P2PError, ValueError):
638
+ return False
639
+ return True
640
+
641
+ async def unsubscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
642
+ """Unsubscribes from a topic/channel on a P2P network (async).
643
+
644
+ Args:
645
+ peer_id: The peer ID associated with the topic/channel.
646
+ channel: The name of the channel to unsubscribe from.
647
+ default_p2p_name: An optional P2P network name to use if the peer's network is unknown.
648
+
649
+ Returns:
650
+ True if the unsubscription is successful, otherwise False.
651
+ """
652
+
653
+ # Getting the right p2p object
654
+ p2p = None
655
+ for _p2p in self.p2p_to_pool_names.keys():
656
+ if _p2p.peer_id == peer_id:
657
+ p2p = _p2p
658
+ break
659
+ if p2p is None and peer_id in self.peer_id_to_p2p:
660
+ p2p = self.peer_id_to_p2p[peer_id]
661
+ if p2p is None:
662
+ if default_p2p_name is not None:
663
+ p2p = self.p2p_name_to_p2p[default_p2p_name]
664
+ else:
665
+ return False
666
+
667
+ try:
668
+ p2p.unsubscribe_from_topic(channel)
669
+ except (P2PError, ValueError):
670
+ return False
671
+ return True
672
+
673
+ async def publish(self, peer_id: str, channel: str,
674
+ content_type: str, content: bytes | dict | tuple | None = None):
675
+ """Publishes a message to a topic/channel on a P2P network (async).
676
+
677
+ Args:
678
+ peer_id: The peer ID associated with the topic/channel.
679
+ channel: The name of the channel to publish to.
680
+ content_type: The type of content in the message.
681
+ content: The message content.
682
+
683
+ Returns:
684
+ True if the message is published successfully, otherwise False.
685
+ """
686
+
687
+ # Getting the right p2p object
688
+ p2p = None
689
+ for _p2p in self.p2p_to_pool_names.keys():
690
+ if _p2p.peer_id == peer_id:
691
+ p2p = _p2p
692
+ break
693
+ if p2p is None:
694
+ p2p = self.peer_id_to_p2p[peer_id]
695
+ if p2p is None:
696
+ return False
697
+
698
+ # Adding sender info here
699
+ msg = Msg(sender=p2p.peer_id,
700
+ content_type=content_type,
701
+ content=content,
702
+ channel=channel,
703
+ piggyback=self.__token + "0") # Adding inspector-mode bit (dummy bit here)
704
+ if ConnectionPools.DEBUG:
705
+ print("[DEBUG CONNECTIONS-POOL] Sending (publish) message: " + str(msg))
706
+
707
+ # Sending message via GossipSub
708
+ try:
709
+ p2p.broadcast_message(channel, msg_bytes=msg.to_bytes())
710
+
711
+ # If the line above executes without raising an error, it was successful.
712
+ return True
713
+ except P2PError:
714
+
715
+ # If send_message_to_peer fails, it will raise a P2PError. We catch it here.
716
+ return False
717
+
718
+
719
+ class NodeConn(ConnectionPools):
720
+
721
+ # Basic name
722
+ __ALL_UNIVERSE = "all_universe"
723
+ __WORLD_AGENTS_ONLY = "world_agents"
724
+ __WORLD_NODE_ONLY = "world_node"
725
+ __WORLD_MASTERS_ONLY = "world_masters"
726
+
727
+ # Suffixes
728
+ __PUBLIC_NET = "_public"
729
+ __PRIVATE_NET = "_private"
730
+
731
+ # Prefixes
732
+ __INBOUND = "in_"
733
+ __OUTBOUND = "out_"
734
+
735
+ # P2p names
736
+ P2P_PUBLIC = "p2p_public"
737
+ P2P_WORLD = "p2p_world"
738
+
739
+ # All pools (prefix + basic name + suffix)
740
+ IN_PUBLIC = __INBOUND + __ALL_UNIVERSE + __PUBLIC_NET
741
+ OUT_PUBLIC = __OUTBOUND + __ALL_UNIVERSE + __PUBLIC_NET
742
+ IN_WORLD_AGENTS = __INBOUND + __WORLD_AGENTS_ONLY + __PRIVATE_NET
743
+ OUT_WORLD_AGENTS = __OUTBOUND + __WORLD_AGENTS_ONLY + __PRIVATE_NET
744
+ IN_WORLD_NODE = __INBOUND + __WORLD_NODE_ONLY + __PRIVATE_NET
745
+ OUT_WORLD_NODE = __OUTBOUND + __WORLD_NODE_ONLY + __PRIVATE_NET
746
+ IN_WORLD_MASTERS = __INBOUND + __WORLD_MASTERS_ONLY + __PRIVATE_NET
747
+ OUT_WORLD_MASTERS = __OUTBOUND + __WORLD_MASTERS_ONLY + __PRIVATE_NET
748
+
749
+ # Aggregated pools
750
+ PUBLIC = {IN_PUBLIC, OUT_PUBLIC}
751
+ WORLD_NODE = {IN_WORLD_NODE, OUT_WORLD_NODE}
752
+ WORLD_AGENTS = {IN_WORLD_AGENTS, OUT_WORLD_AGENTS}
753
+ WORLD_MASTERS = {IN_WORLD_MASTERS, OUT_WORLD_MASTERS}
754
+ WORLD = WORLD_NODE | WORLD_AGENTS | WORLD_MASTERS
755
+ ALL = PUBLIC | WORLD
756
+ OUTGOING = {OUT_PUBLIC, OUT_WORLD_NODE, OUT_WORLD_AGENTS, OUT_WORLD_MASTERS}
757
+ INCOMING = {IN_PUBLIC, IN_WORLD_NODE, IN_WORLD_AGENTS, IN_WORLD_MASTERS}
758
+
759
+ def __init__(self, max_connections: int, p2p_u: P2P, p2p_w: P2P,
760
+ is_world_node: bool, public_key: str, token: str):
761
+ """Initializes a new instance of the NodeConn class.
762
+
763
+ Args:
764
+ max_connections: The total number of connections the node can handle.
765
+ p2p_u: The P2P object for the public network.
766
+ p2p_w: The P2P object for the world/private network.
767
+ is_world_node: A boolean flag indicating if this node is a world node.
768
+ public_key: The public key for token verification.
769
+ token: The node's authentication token.
770
+ """
771
+ super().__init__(max_connections=max_connections,
772
+ p2p_name_to_p2p={
773
+ NodeConn.P2P_PUBLIC: p2p_u,
774
+ NodeConn.P2P_WORLD: p2p_w,
775
+ },
776
+ pool_name_to_p2p_name_and_ratio={
777
+ NodeConn.IN_PUBLIC: [NodeConn.P2P_PUBLIC, 0.25 / 2. if not is_world_node else 0.25 / 2.],
778
+ NodeConn.OUT_PUBLIC: [NodeConn.P2P_PUBLIC, 0.25 / 2. if not is_world_node else 0.25 / 2.],
779
+ NodeConn.IN_WORLD_AGENTS: [NodeConn.P2P_WORLD, .75 / 2 if not is_world_node else 0.5 / 2],
780
+ NodeConn.OUT_WORLD_AGENTS: [NodeConn.P2P_WORLD, .75 / 2 if not is_world_node else 0.5 / 2],
781
+ NodeConn.IN_WORLD_NODE: [NodeConn.P2P_WORLD, 0. if not is_world_node else -1.],
782
+ NodeConn.OUT_WORLD_NODE: [NodeConn.P2P_WORLD, 0. if not is_world_node else -1],
783
+ NodeConn.IN_WORLD_MASTERS: [NodeConn.P2P_WORLD, 0. if not is_world_node else 0.25 / 2.],
784
+ NodeConn.OUT_WORLD_MASTERS: [NodeConn.P2P_WORLD, 0. if not is_world_node else 0.25 / 2.]
785
+ },
786
+ public_key=public_key, token=token)
787
+
788
+ # Just for convenience
789
+ self.p2p_public = p2p_u
790
+ self.p2p_world = p2p_w
791
+
792
+ # These are the list of all the possible agents that might try to connect when we are in world
793
+ self.world_agents_list = set()
794
+ self.world_masters_list = set()
795
+ self.world_agents_and_world_masters_list = set()
796
+ self.world_node_peer_id = None
797
+ self.inspector_peer_id = None
798
+ self.role_to_peer_ids = {}
799
+ self.peer_id_to_addrs = {}
800
+
801
+ # Rendezvous
802
+ self.rendezvous_tag = -1
803
+
804
+ def reset_rendezvous_tag(self):
805
+ """Resets the rendezvous tag to its initial state."""
806
+ self.rendezvous_tag = -1
807
+
808
+ async def conn_routing_fcn(self, connected_peer_infos: list, p2p: P2P):
809
+ """Routes connected peers to the correct connection pool based on their network and role (async).
810
+
811
+ Args:
812
+ connected_peer_infos: A list of dictionaries with information about connected peers.
813
+ p2p: The P2P network object where the connections were found.
814
+
815
+ Returns:
816
+ A dictionary mapping pool names to a dictionary of peer IDs and their information.
817
+ """
818
+ pool_name_and_peer_id_to_peer_info = {k: {} for k in self.p2p_to_pool_names[p2p]}
819
+ public = p2p == self.p2p_public
820
+
821
+ for c in connected_peer_infos:
822
+ inbound = c['direction'] == "incoming"
823
+ outbound = c['direction'] == "outgoing"
824
+ peer_id = c['id'] # Other fields are: c['addrs'], c['connected_at']
825
+
826
+ if public:
827
+ if inbound:
828
+ pool_name_and_peer_id_to_peer_info[NodeConn.IN_PUBLIC][peer_id] = c
829
+ elif outbound:
830
+ pool_name_and_peer_id_to_peer_info[NodeConn.OUT_PUBLIC][peer_id] = c
831
+ else:
832
+ raise ValueError(f"Connection direction is undefined: {c['direction']}")
833
+ else:
834
+ is_world_agent = peer_id in self.world_agents_list
835
+ is_world_master = peer_id in self.world_masters_list
836
+ is_world_node = self.world_node_peer_id is not None and peer_id == self.world_node_peer_id
837
+ is_inspector = self.inspector_peer_id is not None and peer_id == self.inspector_peer_id
838
+ if not is_world_node and not is_world_master and not is_world_agent and not is_inspector:
839
+ if ConnectionPools.DEBUG:
840
+ print("[DEBUG CONNECTIONS-POOL] World agents list: " + str(self.world_agents_list))
841
+ print("[DEBUG CONNECTIONS-POOL] World masters list: " + str(self.world_masters_list))
842
+ print("[DEBUG CONNECTIONS-POOL] World node peer id: " + str(self.world_node_peer_id))
843
+ print("[DEBUG CONNECTIONS-POOL] Inspector peer id: " + str(self.inspector_peer_id))
844
+ print(f"[DEBUG CONNECTIONS-POOL] Unable to determine the peer type for {peer_id}: "
845
+ f"cannot say if world agent, master, world node, inspector (disconnecting it)")
846
+ await ConnectionPools.disconnect(p2p, peer_id)
847
+ continue
848
+
849
+ if inbound:
850
+ pool_name_and_peer_id_to_peer_info[NodeConn.IN_WORLD_AGENTS if is_world_agent else (
851
+ NodeConn.IN_WORLD_NODE if is_world_node else
852
+ NodeConn.IN_WORLD_MASTERS)][peer_id] = c
853
+ elif outbound:
854
+ pool_name_and_peer_id_to_peer_info[NodeConn.OUT_WORLD_AGENTS if is_world_agent else (
855
+ NodeConn.OUT_WORLD_NODE if is_world_node else
856
+ NodeConn.OUT_WORLD_MASTERS)][peer_id] = c
857
+ else:
858
+ raise ValueError(f"Connection direction is undefined: {c}")
859
+
860
+ return pool_name_and_peer_id_to_peer_info
861
+
862
+ def set_world(self, world_peer_id: str | None):
863
+ """Sets the peer ID of the world node.
864
+
865
+ Args:
866
+ world_peer_id: The peer ID of the world node, or None to clear it.
867
+ """
868
+ self.world_node_peer_id = world_peer_id
869
+
870
+ def set_inspector(self, inspector_peer_id: str | None):
871
+ """Sets the peer ID of the inspector.
872
+
873
+ Args:
874
+ inspector_peer_id: The peer ID of the inspector node.
875
+ """
876
+ self.inspector_peer_id = inspector_peer_id
877
+
878
+ def get_world_peer_id(self):
879
+ """Returns the peer ID of the world node.
880
+
881
+ Args:
882
+ None.
883
+
884
+ Returns:
885
+ The world node's peer ID.
886
+ """
887
+ return self.world_node_peer_id
888
+
889
+ def set_addresses_in_peer_info(self, peer_id, addresses):
890
+ """Updates the list of addresses for a given peer.
891
+
892
+ Args:
893
+ peer_id: The peer ID to update.
894
+ addresses: A new list of addresses for the peer.
895
+ """
896
+ if self.in_connection_queues(peer_id):
897
+ addrs = self.pool_name_to_peer_infos[self.get_pool_of(peer_id)][peer_id]['addrs']
898
+ addrs.clear() # Warning: do not allocate a new list, keep the current one (it is referenced by others)
899
+ for _addrs in addresses:
900
+ addrs.append(_addrs)
901
+
902
+ def set_role(self, peer_id, new_role: int):
903
+ """Updates the role of a peer and its associated role-based lists.
904
+
905
+ Args:
906
+ peer_id: The peer ID to update.
907
+ new_role: The new role for the peer.
908
+ """
909
+ cur_role = self.get_role(peer_id)
910
+
911
+ # Updating
912
+ self.peer_id_to_misc[peer_id] = new_role
913
+
914
+ if self.in_connection_queues(peer_id):
915
+ self.pool_name_to_peer_infos[self.get_pool_of(peer_id)][peer_id]['misc'] = new_role
916
+
917
+ # Updating
918
+ if cur_role in self.role_to_peer_ids:
919
+ if peer_id in self.role_to_peer_ids[cur_role]:
920
+ self.role_to_peer_ids[cur_role].remove(peer_id)
921
+ if len(self.role_to_peer_ids[cur_role]) == 0:
922
+ del self.role_to_peer_ids[cur_role]
923
+ if new_role not in self.role_to_peer_ids:
924
+ self.role_to_peer_ids[new_role] = set()
925
+ self.role_to_peer_ids[new_role].add(peer_id)
926
+
927
+ def set_world_agents_list(self, world_agents_list_peer_infos: list[dict] | None):
928
+ """Sets the list of all world agents based on a provided list of peer information.
929
+
930
+ Args:
931
+ world_agents_list_peer_infos: A list of dictionaries containing peer information for world agents.
932
+ """
933
+
934
+ # Clearing previous information
935
+ to_remove = []
936
+ for peer_id, misc in self.peer_id_to_misc.items():
937
+ if misc & 1 == 1 and misc & 2 == 0:
938
+ to_remove.append((peer_id, misc))
939
+
940
+ for peer_id, misc in to_remove:
941
+ del self.peer_id_to_misc[peer_id]
942
+ if peer_id in self.peer_id_to_addrs:
943
+ del self.peer_id_to_addrs[peer_id]
944
+ self.role_to_peer_ids[misc].discard(peer_id)
945
+
946
+ # Setting new information
947
+ if world_agents_list_peer_infos is not None and len(world_agents_list_peer_infos) > 0:
948
+ self.world_agents_list = {x['id'] for x in world_agents_list_peer_infos}
949
+ for x in world_agents_list_peer_infos:
950
+ self.peer_id_to_addrs[x['id']] = x['addrs']
951
+ self.set_role(x['id'], x['misc'])
952
+ else:
953
+ self.world_agents_list = set()
954
+
955
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
956
+
957
+ def set_world_masters_list(self, world_masters_list_peer_infos: list[dict] | None):
958
+ """Sets the list of all world masters based on a provided list of peer information.
959
+
960
+ Args:
961
+ world_masters_list_peer_infos: A list of dictionaries containing peer information for world masters.
962
+ """
963
+
964
+ # Clearing previous information
965
+ to_remove = []
966
+ for peer_id, misc in self.peer_id_to_misc.items():
967
+ if misc & 1 == 1 and misc & 2 == 2:
968
+ to_remove.append((peer_id, misc))
969
+
970
+ for peer_id, misc in to_remove:
971
+ del self.peer_id_to_misc[peer_id]
972
+ if peer_id in self.peer_id_to_addrs:
973
+ del self.peer_id_to_addrs[peer_id]
974
+ self.role_to_peer_ids[misc].discard(peer_id)
975
+
976
+ # Setting new information
977
+ if world_masters_list_peer_infos is not None and len(world_masters_list_peer_infos) > 0:
978
+ self.world_masters_list = {x['id'] for x in world_masters_list_peer_infos}
979
+ for x in world_masters_list_peer_infos:
980
+ self.peer_id_to_addrs[x['id']] = x['addrs']
981
+ self.set_role(x['id'], x['misc'])
982
+ else:
983
+ self.world_masters_list = set()
984
+
985
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
986
+
987
+ def add_to_world_agents_list(self, peer_id: str, addrs: list[str], role: int = -1):
988
+ """Adds a new world agent to the list.
989
+
990
+ Args:
991
+ peer_id: The peer ID of the new agent.
992
+ addrs: A list of addresses for the new agent.
993
+ role: The role assigned to the agent.
994
+ """
995
+ self.world_agents_list.add(peer_id)
996
+
997
+ # This assumes that the WORLD MASTER/AGENT BIT is the first one
998
+ assert role & 1 == 1, "Expecting the first bit of the role to be 1 for world agents"
999
+ assert role & 2 == 0, "Expecting the second bit of the role to be 0 for world agents"
1000
+ self.peer_id_to_addrs[peer_id] = addrs
1001
+ self.set_role(peer_id, role)
1002
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
1003
+
1004
+ def add_to_world_masters_list(self, peer_id: str, addrs: list[str], role: int = -1):
1005
+ """Adds a new world master to the list.
1006
+
1007
+ Args:
1008
+ peer_id: The peer ID of the new master.
1009
+ addrs: A list of addresses for the new master.
1010
+ role: The role assigned to the master.
1011
+ """
1012
+ self.world_masters_list.add(peer_id)
1013
+
1014
+ # This assumes that the WORLD MASTER/AGENT BIT is the first one
1015
+ assert role & 1 == 1, "Expecting the first bit of the role to be 1 for world masters"
1016
+ assert role & 2 == 2, "Expecting the second bit of the role to be 1 for world masters"
1017
+ self.peer_id_to_addrs[peer_id] = addrs
1018
+ self.set_role(peer_id, role)
1019
+ self.world_agents_and_world_masters_list = self.world_agents_list | self.world_masters_list
1020
+
1021
+ def get_added_after_updating(self, pool_names: list[str] | None = None):
1022
+ """Retrieves the set of peers added after the last update cycle for specified pools.
1023
+
1024
+ Args:
1025
+ pool_names: A list of pool names to check. If None, checks all pools.
1026
+
1027
+ Returns:
1028
+ A dictionary mapping pool names to sets of added peer IDs, or a single set if only one pool is specified.
1029
+ """
1030
+ if pool_names is not None:
1031
+ ret = {}
1032
+ for p in pool_names:
1033
+ ret[p] = super().get_added_after_updating(p)
1034
+ return ret
1035
+ else:
1036
+ return super().get_added_after_updating()
1037
+
1038
+ def get_removed_after_updating(self, pool_names: list[str] | None = None):
1039
+ """Retrieves the set of peers removed after the last update cycle for specified pools.
1040
+
1041
+ Args:
1042
+ pool_names: A list of pool names to check. If None, checks all pools.
1043
+
1044
+ Returns:
1045
+ A dictionary mapping pool names to sets of removed peer IDs, or a single set if only one pool is specified.
1046
+ """
1047
+ if pool_names is not None:
1048
+ ret = {}
1049
+ for p in pool_names:
1050
+ ret[p] = super().get_removed_after_updating(p)
1051
+ return ret
1052
+ else:
1053
+ return super().get_removed_after_updating()
1054
+
1055
+ def size(self, pool_names: list[str] | None = None):
1056
+ """Returns the total number of connections across all specified pools.
1057
+
1058
+ Args:
1059
+ pool_names: A list of pool names to sum the size of. If None, returns the total size of all pools.
1060
+
1061
+ Returns:
1062
+ The total number of connections.
1063
+ """
1064
+ if pool_names is not None:
1065
+ return super().size()
1066
+ else:
1067
+ c = 0
1068
+ for p in self.pool_names:
1069
+ c += super().size(p)
1070
+ return c
1071
+
1072
+ def is_connected(self, peer_id: str, pool_names: list[str] | None = None):
1073
+ """Checks if a peer is connected in any of the specified pools.
1074
+
1075
+ Args:
1076
+ peer_id: The peer ID to check.
1077
+ pool_names: A list of pool names to search within. If None, searches all pools.
1078
+
1079
+ Returns:
1080
+ True if the peer is found in any of the pools, otherwise False.
1081
+ """
1082
+ if pool_names is None:
1083
+ return super().is_connected(peer_id)
1084
+ else:
1085
+ for p in pool_names:
1086
+ if super().is_connected(peer_id, p):
1087
+ return True
1088
+ return False
1089
+
1090
+ def is_public(self, peer_id):
1091
+ """Checks if a peer is connected via the public network.
1092
+
1093
+ Args:
1094
+ peer_id: The peer ID to check.
1095
+
1096
+ Returns:
1097
+ True if the peer is in a public pool, otherwise False.
1098
+ """
1099
+ pool_name = self.get_pool_of(peer_id)
1100
+ return pool_name in NodeConn.PUBLIC
1101
+
1102
+ def is_world_master(self, peer_id):
1103
+ """Checks if a peer is a world master.
1104
+
1105
+ Args:
1106
+ peer_id: The peer ID to check.
1107
+
1108
+ Returns:
1109
+ True if the peer is in a world master pool, otherwise False.
1110
+ """
1111
+ pool_name = self.get_pool_of(peer_id)
1112
+ return pool_name in NodeConn.WORLD_MASTERS
1113
+
1114
+ def is_world_node(self, peer_id):
1115
+ """Checks if a peer is the world node.
1116
+
1117
+ Args:
1118
+ peer_id: The peer ID to check.
1119
+
1120
+ Returns:
1121
+ True if the peer is in a world node pool, otherwise False.
1122
+ """
1123
+ pool_name = self.get_pool_of(peer_id)
1124
+ return pool_name in NodeConn.WORLD_NODE
1125
+
1126
+ def is_in_world(self, peer_id):
1127
+ """Checks if a peer is connected to the world network.
1128
+
1129
+ Args:
1130
+ peer_id: The peer ID to check.
1131
+
1132
+ Returns:
1133
+ True if the peer is in any world pool, otherwise False.
1134
+ """
1135
+ pool_name = self.get_pool_of(peer_id)
1136
+ return pool_name in NodeConn.WORLD
1137
+
1138
+ def get_role(self, peer_id):
1139
+ """Retrieves the role of a given peer.
1140
+
1141
+ Args:
1142
+ peer_id: The peer ID to query.
1143
+
1144
+ Returns:
1145
+ The integer role of the peer.
1146
+ """
1147
+ role = self.peer_id_to_misc.get(peer_id, 0) # 0 means public
1148
+ assert role >= 0, "Expecting role to be >= 0"
1149
+ assert role & 1 != 0 or role == 0, "Expecting public role to be zero (all-zero-bits)"
1150
+ return role
1151
+
1152
+ def get_addrs(self, peer_id):
1153
+ """Retrieves the list of addresses for a given peer.
1154
+
1155
+ Args:
1156
+ peer_id: The peer ID to query.
1157
+
1158
+ Returns:
1159
+ A list of addresses for the peer.
1160
+ """
1161
+ return self.peer_id_to_addrs.get(peer_id)
1162
+
1163
+ def in_connection_queues(self, peer_id):
1164
+ """Checks if a peer ID exists in any connection pool.
1165
+
1166
+ Args:
1167
+ peer_id: The peer ID to check.
1168
+
1169
+ Returns:
1170
+ True if the peer is found in any pool, otherwise False.
1171
+ """
1172
+ return peer_id in self.peer_id_to_pool_name
1173
+
1174
+ def find_addrs_by_role(self, role, return_peer_ids_too: bool = False):
1175
+ """Finds all addresses of peers with a specific role.
1176
+
1177
+ Args:
1178
+ role: The integer role to search for.
1179
+ return_peer_ids_too: A boolean to also return the peer IDs.
1180
+
1181
+ Returns:
1182
+ A list of lists of addresses, and optionally a list of peer IDs.
1183
+ """
1184
+ if role in self.role_to_peer_ids:
1185
+ peer_ids = self.role_to_peer_ids[role]
1186
+ else:
1187
+ if not return_peer_ids_too:
1188
+ return []
1189
+ else:
1190
+ return [], []
1191
+ ret_addrs = []
1192
+ ret_peer_ids = []
1193
+ for peer_id in peer_ids:
1194
+ addrs = self.get_addrs(peer_id)
1195
+ if addrs is not None:
1196
+ ret_addrs.append(addrs)
1197
+ ret_peer_ids.append(peer_id)
1198
+ if not return_peer_ids_too:
1199
+ return ret_addrs
1200
+ else:
1201
+ return ret_addrs, ret_peer_ids
1202
+
1203
+ def count_by_role(self, role: int):
1204
+ """Counts the number of peers with a specific role.
1205
+
1206
+ Args:
1207
+ role: The integer role to count.
1208
+
1209
+ Returns:
1210
+ The number of peers with that role.
1211
+ """
1212
+ if role in self.role_to_peer_ids:
1213
+ return len(self.role_to_peer_ids[role])
1214
+ else:
1215
+ return 0
1216
+
1217
+ def get_all_connected_peer_infos(self, pool_names: list[str] | set[str]):
1218
+ """Retrieves a list of all peer info dictionaries for the specified pools.
1219
+
1220
+ Args:
1221
+ pool_names: A list or set of pool names to query.
1222
+
1223
+ Returns:
1224
+ A list of dictionaries containing peer information.
1225
+ """
1226
+ ret = []
1227
+ for p in pool_names:
1228
+ ret += super().get_all_connected_peer_infos(p)
1229
+ return ret
1230
+
1231
+ async def set_world_agents_and_world_masters_lists_from_rendezvous(self):
1232
+ """Updates the lists of world agents and masters using data from the rendezvous topic (async)."""
1233
+ rendezvous_state = self.p2p_world.get_rendezvous_peers_info()
1234
+
1235
+ if rendezvous_state is not None:
1236
+ tag = rendezvous_state.get('update_count', -1)
1237
+
1238
+ if tag > self.rendezvous_tag:
1239
+ self.rendezvous_tag = tag
1240
+ rendezvous_peer_infos = rendezvous_state.get('peers', [])
1241
+
1242
+ world_agents_peer_infos = []
1243
+ world_masters_peer_infos = []
1244
+
1245
+ if ConnectionPools.DEBUG:
1246
+ print(f"[DEBUG CONNECTIONS-POOL] Rendezvous peer infos (tag: {tag}, peers: "
1247
+ f"{len(rendezvous_peer_infos)} peers)")
1248
+
1249
+ for c in rendezvous_peer_infos:
1250
+ if c['addrs'] is None:
1251
+ print(f"[DEBUG CONNECTIONS-POOL] Skipping a peer with None addrs (unexpected)")
1252
+ continue
1253
+ # if len(c['addrs']) == 0:
1254
+ # print(f"[DEBUG CONNECTIONS-POOL] Skipping a peer with zero-length addrs-list (unexpected)")
1255
+ # continue
1256
+ if (c['misc'] & 1) == 1 and (c['misc'] & 2) == 0:
1257
+ world_agents_peer_infos.append(c)
1258
+ elif (c['misc'] & 1) == 1 and (c['misc'] & 2) == 2:
1259
+ world_masters_peer_infos.append(c)
1260
+ else:
1261
+ raise ValueError("Unexpected value of the 'misc' field: " + str(c))
1262
+
1263
+ # Updating lists
1264
+ self.set_world_agents_list(world_agents_peer_infos)
1265
+ self.set_world_masters_list(world_masters_peer_infos)
1266
+
1267
+ async def get_cv_hash_from_last_token(self, peer_id):
1268
+ """Retrieves the CV hash from the last token received from a peer (async).
1269
+
1270
+ Args:
1271
+ peer_id: The peer ID to query.
1272
+
1273
+ Returns:
1274
+ The CV hash string, or None if not found.
1275
+ """
1276
+ token = self.get_last_token(peer_id)
1277
+ if token is not None:
1278
+ _, cv_hash = await self.verify_token(token, peer_id)
1279
+ return cv_hash
1280
+ else:
1281
+ return None
1282
+
1283
+ async def remove(self, peer_id: str):
1284
+ """Removes a peer and its associated information from all lists and pools (async).
1285
+
1286
+ Args:
1287
+ peer_id: The peer ID to remove.
1288
+ """
1289
+ await super().remove(peer_id)
1290
+ #if peer_id in self.peer_id_to_addrs:
1291
+ # del self.peer_id_to_addrs[peer_id]
1292
+
1293
+ async def remove_all_world_agents(self):
1294
+ """Removes all connected world agents from the pools and role lists (async)."""
1295
+ peer_infos = self.get_all_connected_peer_infos(NodeConn.WORLD)
1296
+ for c in peer_infos:
1297
+ peer_id = c['id']
1298
+ await self.remove(peer_id)
1299
+ if peer_id in self.peer_id_to_addrs:
1300
+ del self.peer_id_to_addrs[peer_id]
1301
+ for role, peer_ids in self.role_to_peer_ids.items():
1302
+ if role & 1 == NodeConn.WORLD:
1303
+ peer_ids.remove(peer_id)
1304
+
1305
+ async def subscribe(self, peer_id: str, channel: str, default_p2p_name: str | None = None):
1306
+ """Subscribes to a channel, defaulting to the world P2P network if a network is not specified (async).
1307
+
1308
+ Args:
1309
+ peer_id: The peer ID associated with the channel.
1310
+ channel: The channel to subscribe to.
1311
+ default_p2p_name: An optional P2P name to use for the subscription.
1312
+
1313
+ Returns:
1314
+ True if successful, False otherwise.
1315
+ """
1316
+ return await super().subscribe(peer_id, channel,
1317
+ default_p2p_name=NodeConn.P2P_WORLD
1318
+ if default_p2p_name is None else default_p2p_name)
1319
+
1320
+ async def get_messages(self, p2p_name: str, allowed_not_connected_peers: set | None = None) -> list[Msg]:
1321
+ """Retrieves messages, allowing for messages from known world agents and masters even if not in a
1322
+ connection pool (async).
1323
+
1324
+ Args:
1325
+ p2p_name: The name of the P2P network to get messages from.
1326
+ allowed_not_connected_peers: This parameter is ignored in this implementation.
1327
+
1328
+ Returns:
1329
+ A list of verified and processed message objects.
1330
+ """
1331
+ assert allowed_not_connected_peers is None, "This param (allowed_not_connected_peers is ignored in NodeConn"
1332
+ return await super().get_messages(p2p_name, allowed_not_connected_peers=self.world_agents_and_world_masters_list)