unaiverse 0.1.6__cp312-cp312-macosx_10_13_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of unaiverse might be problematic. Click here for more details.
- unaiverse/__init__.py +19 -0
- unaiverse/agent.py +2008 -0
- unaiverse/agent_basics.py +1846 -0
- unaiverse/clock.py +191 -0
- unaiverse/dataprops.py +1209 -0
- unaiverse/hsm.py +1880 -0
- unaiverse/modules/__init__.py +18 -0
- unaiverse/modules/cnu/__init__.py +17 -0
- unaiverse/modules/cnu/cnus.py +536 -0
- unaiverse/modules/cnu/layers.py +261 -0
- unaiverse/modules/cnu/psi.py +60 -0
- unaiverse/modules/hl/__init__.py +15 -0
- unaiverse/modules/hl/hl_utils.py +411 -0
- unaiverse/modules/networks.py +1509 -0
- unaiverse/modules/utils.py +680 -0
- unaiverse/networking/__init__.py +16 -0
- unaiverse/networking/node/__init__.py +18 -0
- unaiverse/networking/node/connpool.py +1261 -0
- unaiverse/networking/node/node.py +2223 -0
- unaiverse/networking/node/profile.py +446 -0
- unaiverse/networking/node/tokens.py +79 -0
- unaiverse/networking/p2p/__init__.py +198 -0
- unaiverse/networking/p2p/go.mod +127 -0
- unaiverse/networking/p2p/go.sum +548 -0
- unaiverse/networking/p2p/golibp2p.py +18 -0
- unaiverse/networking/p2p/golibp2p.pyi +135 -0
- unaiverse/networking/p2p/lib.go +2714 -0
- unaiverse/networking/p2p/lib.go.sha256 +1 -0
- unaiverse/networking/p2p/lib_types.py +312 -0
- unaiverse/networking/p2p/message_pb2.py +63 -0
- unaiverse/networking/p2p/messages.py +265 -0
- unaiverse/networking/p2p/mylogger.py +77 -0
- unaiverse/networking/p2p/p2p.py +929 -0
- unaiverse/networking/p2p/proto-go/message.pb.go +616 -0
- unaiverse/networking/p2p/unailib.cpython-312-darwin.so +0 -0
- unaiverse/streamlib/__init__.py +15 -0
- unaiverse/streamlib/streamlib.py +210 -0
- unaiverse/streams.py +770 -0
- unaiverse/utils/__init__.py +16 -0
- unaiverse/utils/ask_lone_wolf.json +27 -0
- unaiverse/utils/lone_wolf.json +19 -0
- unaiverse/utils/misc.py +305 -0
- unaiverse/utils/sandbox.py +293 -0
- unaiverse/utils/server.py +435 -0
- unaiverse/world.py +175 -0
- unaiverse-0.1.6.dist-info/METADATA +365 -0
- unaiverse-0.1.6.dist-info/RECORD +50 -0
- unaiverse-0.1.6.dist-info/WHEEL +6 -0
- unaiverse-0.1.6.dist-info/licenses/LICENSE +43 -0
- unaiverse-0.1.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
█████ █████ ██████ █████ █████ █████ █████ ██████████ ███████████ █████████ ██████████
|
|
3
|
+
░░███ ░░███ ░░██████ ░░███ ░░███ ░░███ ░░███ ░░███░░░░░█░░███░░░░░███ ███░░░░░███░░███░░░░░█
|
|
4
|
+
░███ ░███ ░███░███ ░███ ██████ ░███ ░███ ░███ ░███ █ ░ ░███ ░███ ░███ ░░░ ░███ █ ░
|
|
5
|
+
░███ ░███ ░███░░███░███ ░░░░░███ ░███ ░███ ░███ ░██████ ░██████████ ░░█████████ ░██████
|
|
6
|
+
░███ ░███ ░███ ░░██████ ███████ ░███ ░░███ ███ ░███░░█ ░███░░░░░███ ░░░░░░░░███ ░███░░█
|
|
7
|
+
░███ ░███ ░███ ░░█████ ███░░███ ░███ ░░░█████░ ░███ ░ █ ░███ ░███ ███ ░███ ░███ ░ █
|
|
8
|
+
░░████████ █████ ░░█████░░████████ █████ ░░███ ██████████ █████ █████░░█████████ ██████████
|
|
9
|
+
░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░ ░░░░░ ░░░ ░░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░ ░░░░░░░░░░
|
|
10
|
+
A Collectionless AI Project (https://collectionless.ai)
|
|
11
|
+
Registration/Login: https://unaiverse.io
|
|
12
|
+
Code Repositories: https://github.com/collectionlessai/
|
|
13
|
+
Main Developers: Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
|
|
14
|
+
"""
|
|
15
|
+
from . import misc
|
|
16
|
+
from . import sandbox
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"initial_state": "ready",
|
|
3
|
+
"state": "ready",
|
|
4
|
+
"prev_state": null,
|
|
5
|
+
"limbo_state": null,
|
|
6
|
+
"state_actions": {
|
|
7
|
+
"found_lone_wolf": ["do_gen", {"samples": 1}, 0, false]
|
|
8
|
+
},
|
|
9
|
+
"transitions": {
|
|
10
|
+
"ready": {
|
|
11
|
+
"found_lone_wolf": [["find_agents", {"role": "public_agent", "engage": true}, true, 0]]
|
|
12
|
+
},
|
|
13
|
+
"found_lone_wolf": {
|
|
14
|
+
"asked": [["ask_gen", {"u_hashes": ["<agent>:processor"], "samples": 1}, true, 1]]
|
|
15
|
+
},
|
|
16
|
+
"asked": {
|
|
17
|
+
"lone_wolf_done": [
|
|
18
|
+
["done_gen", {}, false, 2],
|
|
19
|
+
["nop", {"delay": 30.0}, true, 3]
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
"lone_wolf_done": {
|
|
23
|
+
"ready": [["disconnect_by_role", {"role": "public_agent"}, true, 4]]
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"cur_action": null
|
|
27
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"initial_state": "ready",
|
|
3
|
+
"state": "ready",
|
|
4
|
+
"prev_state": null,
|
|
5
|
+
"limbo_state": null,
|
|
6
|
+
"state_actions": {
|
|
7
|
+
"ready": [null, null, 0, false],
|
|
8
|
+
"done_gen": [null, null, 1, false]
|
|
9
|
+
},
|
|
10
|
+
"transitions": {
|
|
11
|
+
"ready": {
|
|
12
|
+
"done_gen": [["do_gen", {"timeout": 7.5}, false, 0]]
|
|
13
|
+
},
|
|
14
|
+
"done_gen": {
|
|
15
|
+
"ready": [["nop", {}, true, 3]]
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"cur_action": null
|
|
19
|
+
}
|
unaiverse/utils/misc.py
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""
|
|
2
|
+
█████ █████ ██████ █████ █████ █████ █████ ██████████ ███████████ █████████ ██████████
|
|
3
|
+
░░███ ░░███ ░░██████ ░░███ ░░███ ░░███ ░░███ ░░███░░░░░█░░███░░░░░███ ███░░░░░███░░███░░░░░█
|
|
4
|
+
░███ ░███ ░███░███ ░███ ██████ ░███ ░███ ░███ ░███ █ ░ ░███ ░███ ░███ ░░░ ░███ █ ░
|
|
5
|
+
░███ ░███ ░███░░███░███ ░░░░░███ ░███ ░███ ░███ ░██████ ░██████████ ░░█████████ ░██████
|
|
6
|
+
░███ ░███ ░███ ░░██████ ███████ ░███ ░░███ ███ ░███░░█ ░███░░░░░███ ░░░░░░░░███ ░███░░█
|
|
7
|
+
░███ ░███ ░███ ░░█████ ███░░███ ░███ ░░░█████░ ░███ ░ █ ░███ ░███ ███ ░███ ░███ ░ █
|
|
8
|
+
░░████████ █████ ░░█████░░████████ █████ ░░███ ██████████ █████ █████░░█████████ ██████████
|
|
9
|
+
░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░ ░░░░░ ░░░ ░░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░ ░░░░░░░░░░
|
|
10
|
+
A Collectionless AI Project (https://collectionless.ai)
|
|
11
|
+
Registration/Login: https://unaiverse.io
|
|
12
|
+
Code Repositories: https://github.com/collectionlessai/
|
|
13
|
+
Main Developers: Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
|
|
14
|
+
"""
|
|
15
|
+
import os
|
|
16
|
+
import ast
|
|
17
|
+
import sys
|
|
18
|
+
import time
|
|
19
|
+
import json
|
|
20
|
+
import math
|
|
21
|
+
import threading
|
|
22
|
+
from tqdm import tqdm
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class GenException(Exception):
|
|
28
|
+
"""Base exception for this application (a simple wrapper around a generic Exception)."""
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def save_node_addresses_to_file(node, dir_path: str, public: bool,
|
|
33
|
+
filename: str = "addresses.txt", append: bool = False):
|
|
34
|
+
address_file = os.path.join(dir_path, filename)
|
|
35
|
+
with open(address_file, "w" if not append else "a") as file:
|
|
36
|
+
file.write(node.hosted.get_name() + ";" +
|
|
37
|
+
str(node.get_public_addresses() if public else node.get_world_addresses()) + "\n")
|
|
38
|
+
file.flush()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_node_addresses_from_file(dir_path: str, filename: str = "addresses.txt") -> dict[str, list[str]]:
|
|
42
|
+
ret = {}
|
|
43
|
+
with open(os.path.join(dir_path, filename)) as file:
|
|
44
|
+
lines = file.readlines()
|
|
45
|
+
|
|
46
|
+
# Old file format
|
|
47
|
+
if lines[0].strip() == "/":
|
|
48
|
+
addresses = []
|
|
49
|
+
for line in lines:
|
|
50
|
+
_line = line.strip()
|
|
51
|
+
if len(_line) > 0:
|
|
52
|
+
addresses.append(_line)
|
|
53
|
+
ret["unk"] = addresses
|
|
54
|
+
return ret
|
|
55
|
+
|
|
56
|
+
# New file format
|
|
57
|
+
for line in lines:
|
|
58
|
+
if line.strip().startswith("***"): # Header marker
|
|
59
|
+
continue
|
|
60
|
+
comma_separated_values = [v.strip() for v in line.split(';')]
|
|
61
|
+
node_name, addresses_str = comma_separated_values
|
|
62
|
+
ret[node_name] = ast.literal_eval(addresses_str) # Name appearing multiple times? the last entry is kept
|
|
63
|
+
|
|
64
|
+
return ret
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Silent:
|
|
68
|
+
def __init__(self, ignore: bool = False):
|
|
69
|
+
self.ignore = ignore
|
|
70
|
+
|
|
71
|
+
def __enter__(self):
|
|
72
|
+
if not self.ignore:
|
|
73
|
+
self._original_stdout = sys.stdout
|
|
74
|
+
sys.stdout = open(os.devnull, "w")
|
|
75
|
+
|
|
76
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
77
|
+
if not self.ignore:
|
|
78
|
+
sys.stdout.close()
|
|
79
|
+
sys.stdout = self._original_stdout
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# The countdown function
|
|
83
|
+
def countdown_start(seconds: int, msg: str):
|
|
84
|
+
class TqdmPrintRedirector:
|
|
85
|
+
def __init__(self, tqdm_instance):
|
|
86
|
+
self.tqdm_instance = tqdm_instance
|
|
87
|
+
self.original_stdout = sys.__stdout__
|
|
88
|
+
|
|
89
|
+
def write(self, s):
|
|
90
|
+
if s.strip(): # Ignore empty lines (needed for the way tqdm works)
|
|
91
|
+
self.tqdm_instance.write(s, file=self.original_stdout)
|
|
92
|
+
|
|
93
|
+
def flush(self):
|
|
94
|
+
pass # Tqdm handles flushing
|
|
95
|
+
|
|
96
|
+
def drawing(secs: int, message: str):
|
|
97
|
+
with tqdm(total=secs, desc=message, file=sys.__stdout__) as t:
|
|
98
|
+
sys.stdout = TqdmPrintRedirector(t) # Redirect prints to tqdm.write
|
|
99
|
+
for i in range(secs):
|
|
100
|
+
time.sleep(1)
|
|
101
|
+
t.update(1.)
|
|
102
|
+
sys.stdout = sys.__stdout__ # Restore original stdout
|
|
103
|
+
|
|
104
|
+
sys.stdout.flush()
|
|
105
|
+
handle = threading.Thread(target=drawing, args=(seconds, msg))
|
|
106
|
+
handle.start()
|
|
107
|
+
return handle
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def countdown_wait(handle):
|
|
111
|
+
handle.join()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def check_json_start(file: str, msg: str, delete_existing: bool = False):
|
|
115
|
+
from rich.json import JSON
|
|
116
|
+
from rich.console import Console
|
|
117
|
+
cons = Console(file=sys.__stdout__)
|
|
118
|
+
|
|
119
|
+
if delete_existing:
|
|
120
|
+
if os.path.exists(file):
|
|
121
|
+
os.remove(file)
|
|
122
|
+
|
|
123
|
+
def checking(file_path: str, console: Console):
|
|
124
|
+
print(msg)
|
|
125
|
+
prev_dict = {}
|
|
126
|
+
while True:
|
|
127
|
+
if os.path.exists(file_path):
|
|
128
|
+
try:
|
|
129
|
+
with open(file_path, "r") as f:
|
|
130
|
+
json_dict = json.load(f)
|
|
131
|
+
if json_dict != prev_dict:
|
|
132
|
+
now = datetime.now()
|
|
133
|
+
console.print("─" * 80)
|
|
134
|
+
console.print("Printing updated file "
|
|
135
|
+
"(print time: " + now.strftime("%Y-%m-%d %H:%M:%S") + ")")
|
|
136
|
+
console.print("─" * 80)
|
|
137
|
+
console.print(JSON.from_data(json_dict))
|
|
138
|
+
prev_dict = json_dict
|
|
139
|
+
except KeyboardInterrupt:
|
|
140
|
+
break
|
|
141
|
+
except Exception:
|
|
142
|
+
pass
|
|
143
|
+
time.sleep(1)
|
|
144
|
+
|
|
145
|
+
handle = threading.Thread(target=checking, args=(file, cons), daemon=True)
|
|
146
|
+
handle.start()
|
|
147
|
+
return handle
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def check_json_start_wait(handle):
|
|
151
|
+
handle.join()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def show_images_grid(image_paths, max_cols=3):
|
|
155
|
+
import matplotlib.pyplot as plt
|
|
156
|
+
import matplotlib.image as mpimg
|
|
157
|
+
|
|
158
|
+
n = len(image_paths)
|
|
159
|
+
cols = min(max_cols, n)
|
|
160
|
+
rows = math.ceil(n / cols)
|
|
161
|
+
|
|
162
|
+
# Load images
|
|
163
|
+
images = [mpimg.imread(p) for p in image_paths]
|
|
164
|
+
|
|
165
|
+
# Determine figure size based on image sizes
|
|
166
|
+
widths, heights = zip(*[(img.shape[1], img.shape[0]) for img in images])
|
|
167
|
+
|
|
168
|
+
# Use average width/height for scaling
|
|
169
|
+
avg_width = sum(widths) / len(widths)
|
|
170
|
+
avg_height = sum(heights) / len(heights)
|
|
171
|
+
|
|
172
|
+
fig_width = cols * avg_width / 100
|
|
173
|
+
fig_height = rows * avg_height / 100
|
|
174
|
+
|
|
175
|
+
fig, axes = plt.subplots(rows, cols, figsize=(fig_width, fig_height))
|
|
176
|
+
axes = axes.flatten() if n > 1 else [axes]
|
|
177
|
+
|
|
178
|
+
fig.canvas.manager.set_window_title("Image Grid")
|
|
179
|
+
|
|
180
|
+
# Hide unused axes
|
|
181
|
+
for ax in axes[n:]:
|
|
182
|
+
ax.axis('off')
|
|
183
|
+
|
|
184
|
+
for idx, (ax, img) in enumerate(zip(axes, images)):
|
|
185
|
+
ax.imshow(img)
|
|
186
|
+
ax.axis('off')
|
|
187
|
+
ax.set_title(str(idx), fontsize=12, fontweight='bold')
|
|
188
|
+
|
|
189
|
+
# Display images
|
|
190
|
+
for ax, img in zip(axes, images):
|
|
191
|
+
ax.imshow(img)
|
|
192
|
+
ax.axis('off')
|
|
193
|
+
|
|
194
|
+
plt.subplots_adjust(wspace=0, hspace=0)
|
|
195
|
+
|
|
196
|
+
# Turn on interactive mode
|
|
197
|
+
plt.ion()
|
|
198
|
+
plt.show()
|
|
199
|
+
|
|
200
|
+
fig.canvas.draw()
|
|
201
|
+
plt.pause(0.1)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class FileTracker:
|
|
205
|
+
def __init__(self, folder, ext=".json"):
|
|
206
|
+
self.folder = Path(folder)
|
|
207
|
+
self.ext = ext.lower()
|
|
208
|
+
self.last_state = self._scan_files()
|
|
209
|
+
|
|
210
|
+
def _scan_files(self):
|
|
211
|
+
state = {}
|
|
212
|
+
for file in self.folder.iterdir():
|
|
213
|
+
if file.is_file() and file.suffix.lower() == self.ext:
|
|
214
|
+
state[file.name] = os.path.getmtime(file)
|
|
215
|
+
return state
|
|
216
|
+
|
|
217
|
+
def something_changed(self):
|
|
218
|
+
new_state = self._scan_files()
|
|
219
|
+
created = [f for f in new_state if f not in self.last_state]
|
|
220
|
+
modified = [f for f in new_state
|
|
221
|
+
if f in self.last_state and new_state[f] != self.last_state[f]]
|
|
222
|
+
self.last_state = new_state
|
|
223
|
+
return created or modified
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def prepare_key_dir(app_name):
|
|
227
|
+
app_name = app_name.lower()
|
|
228
|
+
if os.name == "nt": # Windows
|
|
229
|
+
if os.getenv("APPDATA") is not None:
|
|
230
|
+
key_dir = os.path.join(os.getenv("APPDATA"), "Local", app_name) # Expected
|
|
231
|
+
else:
|
|
232
|
+
key_dir = os.path.join(str(Path.home()), f".{app_name}") # Fallback
|
|
233
|
+
else: # Linux/macOS
|
|
234
|
+
key_dir = os.path.join(str(Path.home()), f".{app_name}")
|
|
235
|
+
os.makedirs(key_dir, exist_ok=True)
|
|
236
|
+
return key_dir
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def get_key_considering_multiple_sources(key_variable: str | None) -> str:
|
|
240
|
+
|
|
241
|
+
# Creating folder (if needed) to store the key
|
|
242
|
+
try:
|
|
243
|
+
key_dir = prepare_key_dir(app_name="UNaIVERSE")
|
|
244
|
+
except Exception:
|
|
245
|
+
raise GenException("Cannot create folder to store the key file")
|
|
246
|
+
key_file = os.path.join(key_dir, "key")
|
|
247
|
+
|
|
248
|
+
# Getting from an existing file
|
|
249
|
+
key_from_file = None
|
|
250
|
+
if os.path.exists(key_file):
|
|
251
|
+
with open(key_file, "r") as f:
|
|
252
|
+
key_from_file = f.read().strip()
|
|
253
|
+
|
|
254
|
+
# Getting from env variable
|
|
255
|
+
key_from_env = os.getenv("NODE_KEY", None)
|
|
256
|
+
|
|
257
|
+
# Getting from code-specified option
|
|
258
|
+
if key_variable is not None and len(key_variable.strip()) > 0:
|
|
259
|
+
key_from_var = key_variable.strip()
|
|
260
|
+
if key_from_var.startswith("<") and key_from_var.endswith(">"): # Something like <UNAIVERSE_KEY_GOES_HERE>
|
|
261
|
+
key_from_var = None
|
|
262
|
+
else:
|
|
263
|
+
key_from_var = None
|
|
264
|
+
|
|
265
|
+
# Finding valid sources and checking if multiple keys were provided
|
|
266
|
+
_keys = [key_from_var, key_from_env, key_from_file]
|
|
267
|
+
_source_names = ["your code", "env variable 'NODE_KEY'", f"cache file {key_file}"]
|
|
268
|
+
source_names = []
|
|
269
|
+
mismatching = False
|
|
270
|
+
multiple_source = False
|
|
271
|
+
first_key = None
|
|
272
|
+
first_source = None
|
|
273
|
+
_prev_key = None
|
|
274
|
+
for i, (_key, _source_name) in enumerate(zip(_keys, _source_names)):
|
|
275
|
+
if _key is not None:
|
|
276
|
+
source_names.append(_source_name)
|
|
277
|
+
if _prev_key is not None:
|
|
278
|
+
if _key != _prev_key:
|
|
279
|
+
mismatching = True
|
|
280
|
+
multiple_source = True
|
|
281
|
+
else:
|
|
282
|
+
_prev_key = _key
|
|
283
|
+
first_key = _key
|
|
284
|
+
first_source = _source_name
|
|
285
|
+
|
|
286
|
+
if len(source_names) > 0:
|
|
287
|
+
msg = ""
|
|
288
|
+
if multiple_source and not mismatching:
|
|
289
|
+
msg = "UNaIVERSE key (the exact same key) present in multiple locations: " + ", ".join(source_names)
|
|
290
|
+
if multiple_source and mismatching:
|
|
291
|
+
msg = "UNaIVERSE keys (different keys) present in multiple locations: " + ", ".join(source_names)
|
|
292
|
+
msg += "\nLoaded the one stored in " + first_source
|
|
293
|
+
if not multiple_source:
|
|
294
|
+
msg = f"UNaIVERSE key loaded from {first_source}"
|
|
295
|
+
print(msg)
|
|
296
|
+
return first_key
|
|
297
|
+
else:
|
|
298
|
+
|
|
299
|
+
# If no key present, ask user and save to file
|
|
300
|
+
print("UNaIVERSE key not present in " + ", ".join(_source_names))
|
|
301
|
+
print("If you did not already do it, go to https://unaiverse.io, login, and generate a key")
|
|
302
|
+
key = input("Enter your UNaIVERSE key, that will be saved to the cache file: ").strip()
|
|
303
|
+
with open(key_file, "w") as f:
|
|
304
|
+
f.write(key)
|
|
305
|
+
return key
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""
|
|
2
|
+
█████ █████ ██████ █████ █████ █████ █████ ██████████ ███████████ █████████ ██████████
|
|
3
|
+
░░███ ░░███ ░░██████ ░░███ ░░███ ░░███ ░░███ ░░███░░░░░█░░███░░░░░███ ███░░░░░███░░███░░░░░█
|
|
4
|
+
░███ ░███ ░███░███ ░███ ██████ ░███ ░███ ░███ ░███ █ ░ ░███ ░███ ░███ ░░░ ░███ █ ░
|
|
5
|
+
░███ ░███ ░███░░███░███ ░░░░░███ ░███ ░███ ░███ ░██████ ░██████████ ░░█████████ ░██████
|
|
6
|
+
░███ ░███ ░███ ░░██████ ███████ ░███ ░░███ ███ ░███░░█ ░███░░░░░███ ░░░░░░░░███ ░███░░█
|
|
7
|
+
░███ ░███ ░███ ░░█████ ███░░███ ░███ ░░░█████░ ░███ ░ █ ░███ ░███ ███ ░███ ░███ ░ █
|
|
8
|
+
░░████████ █████ ░░█████░░████████ █████ ░░███ ██████████ █████ █████░░█████████ ██████████
|
|
9
|
+
░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░ ░░░░░ ░░░ ░░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░ ░░░░░░░░░░
|
|
10
|
+
A Collectionless AI Project (https://collectionless.ai)
|
|
11
|
+
Registration/Login: https://unaiverse.io
|
|
12
|
+
Code Repositories: https://github.com/collectionlessai/
|
|
13
|
+
Main Developers: Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
|
|
14
|
+
"""
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import uuid
|
|
18
|
+
import argparse
|
|
19
|
+
import subprocess
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from unaiverse.networking.p2p import P2P
|
|
22
|
+
|
|
23
|
+
# Configuration
|
|
24
|
+
DOCKER_IMAGE_NAME = "unaiverse-sandbox"
|
|
25
|
+
CONTAINER_NAME_BASE = "unaiverse-sandbox-container"
|
|
26
|
+
CONTAINER_NAME = f"{CONTAINER_NAME_BASE}-{uuid.uuid4().hex[:8]}" # Append a short unique ID
|
|
27
|
+
DOCKERFILE_CONTENT = """
|
|
28
|
+
|
|
29
|
+
# Debian image, automatically guessed architecture
|
|
30
|
+
FROM python:3.12-slim-bookworm
|
|
31
|
+
|
|
32
|
+
# Installing Go compiler
|
|
33
|
+
RUN apt-get update && apt-get install -y --no-install-recommends build-essential curl git
|
|
34
|
+
RUN rm -rf /var/lib/apt/lists/*
|
|
35
|
+
RUN ARCH=$(dpkg --print-architecture) && curl -LO https://go.dev/dl/go1.24.5.linux-${ARCH}.tar.gz
|
|
36
|
+
RUN ARCH=$(dpkg --print-architecture) && tar -C /usr/local -xzf go1.24.5.linux-${ARCH}.tar.gz
|
|
37
|
+
RUN ARCH=$(dpkg --print-architecture) && rm go1.24.5.linux-${ARCH}.tar.gz
|
|
38
|
+
|
|
39
|
+
# Set Go environment variables
|
|
40
|
+
ENV PATH="/usr/local/go/bin:${PATH}"
|
|
41
|
+
ENV GOPATH="/go"
|
|
42
|
+
RUN mkdir -p /go/bin /go/src /go/pkg
|
|
43
|
+
|
|
44
|
+
# Setting the working directory inside the container
|
|
45
|
+
WORKDIR /unaiverse
|
|
46
|
+
|
|
47
|
+
# Dependencies
|
|
48
|
+
RUN <create_requirements.txt>
|
|
49
|
+
RUN pip install --no-cache-dir -r requirements.txt --break-system-packages
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sandbox(file_to_run: str,
|
|
54
|
+
read_only_paths: tuple[str] | list[str] | None = None,
|
|
55
|
+
writable_paths: tuple[str] | list[str] | None = None) -> None:
|
|
56
|
+
|
|
57
|
+
# Path of this file
|
|
58
|
+
absolute_path_of_this_file = os.path.abspath(__file__)
|
|
59
|
+
|
|
60
|
+
# Folders composing the path (and file name at the end)
|
|
61
|
+
path_components = list(Path(absolute_path_of_this_file).parts)
|
|
62
|
+
|
|
63
|
+
# Ensuring the folder/file structure was not manipulated
|
|
64
|
+
assert path_components[-1] == 'sandbox.py', "Major security issue, stopping."
|
|
65
|
+
assert path_components[-2] == 'utils', "Major security issue, stopping."
|
|
66
|
+
assert path_components[-3] == 'unaiverse', "Major security issue, stopping."
|
|
67
|
+
|
|
68
|
+
# Main folder of UNaIVERSE
|
|
69
|
+
abspath_of_unaiverse_code = str(Path(*path_components[0:-3]))
|
|
70
|
+
|
|
71
|
+
# Clean up any remnants from previous runs first (safety)
|
|
72
|
+
cleanup_docker_artifacts(where=abspath_of_unaiverse_code)
|
|
73
|
+
|
|
74
|
+
# Requirements
|
|
75
|
+
echoed_contents_of_requirements = 'printf "'
|
|
76
|
+
with open(os.path.join(abspath_of_unaiverse_code, "requirements.txt"), 'r') as req_file:
|
|
77
|
+
req_lines = req_file.readlines()
|
|
78
|
+
for i, req_line in enumerate(req_lines):
|
|
79
|
+
if i != (len(req_lines) - 1) and len(req_line.strip()) > 0:
|
|
80
|
+
echoed_contents_of_requirements += req_line.strip() + "\\n"
|
|
81
|
+
else:
|
|
82
|
+
echoed_contents_of_requirements += req_line.strip() + "\\n\" > requirements.txt"
|
|
83
|
+
|
|
84
|
+
# Create Dockerfile
|
|
85
|
+
print("Creating Dockerfile...")
|
|
86
|
+
with open(os.path.join(abspath_of_unaiverse_code, "Dockerfile"), "w") as f:
|
|
87
|
+
f.write(DOCKERFILE_CONTENT.replace('<create_requirements.txt>', echoed_contents_of_requirements))
|
|
88
|
+
|
|
89
|
+
# Building Docker image
|
|
90
|
+
if not build_docker_image(where=abspath_of_unaiverse_code):
|
|
91
|
+
print("Exiting due to Docker image build failure")
|
|
92
|
+
cleanup_docker_artifacts(where=abspath_of_unaiverse_code) # Try to clean up what was created (if any)
|
|
93
|
+
sys.exit(1)
|
|
94
|
+
|
|
95
|
+
# Read only folders from the host machine
|
|
96
|
+
read_only_mount_paths = ([abspath_of_unaiverse_code] +
|
|
97
|
+
(list(read_only_paths) if read_only_paths is not None else []))
|
|
98
|
+
|
|
99
|
+
# Writable folders in host machine
|
|
100
|
+
writable_mount_paths = ([os.path.join(abspath_of_unaiverse_code, 'runners'),
|
|
101
|
+
os.path.join(abspath_of_unaiverse_code, 'unaiverse', 'library'),
|
|
102
|
+
os.path.join(abspath_of_unaiverse_code, 'unaiverse', 'networking', 'p2p')] +
|
|
103
|
+
(list(writable_paths) if writable_paths is not None else []))
|
|
104
|
+
|
|
105
|
+
# Running
|
|
106
|
+
if not run_in_docker(file_to_run=os.path.abspath(file_to_run),
|
|
107
|
+
read_only_host_paths=read_only_mount_paths,
|
|
108
|
+
writable_host_paths=writable_mount_paths):
|
|
109
|
+
print("Exiting due to Docker container run failure")
|
|
110
|
+
sys.exit(1)
|
|
111
|
+
|
|
112
|
+
# Final cleanup
|
|
113
|
+
cleanup_docker_artifacts(where=abspath_of_unaiverse_code)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def build_docker_image(where: str):
|
|
117
|
+
"""Builds the Docker image."""
|
|
118
|
+
print(f"Building Docker image '{DOCKER_IMAGE_NAME}'...")
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
|
|
122
|
+
# The '.' at the end means build from the current directory
|
|
123
|
+
subprocess.run(["docker", "build", "-t", DOCKER_IMAGE_NAME, where], check=True)
|
|
124
|
+
print(f"Docker image '{DOCKER_IMAGE_NAME}' built successfully.")
|
|
125
|
+
return True
|
|
126
|
+
except subprocess.CalledProcessError as e:
|
|
127
|
+
print(f"Error building Docker image: {e}")
|
|
128
|
+
return False
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def cleanup_docker_artifacts(where: str):
|
|
132
|
+
"""Cleans up the generated files and Docker image."""
|
|
133
|
+
print("Cleaning...")
|
|
134
|
+
|
|
135
|
+
# Stop and remove container if it's still running (e.g., if previous run failed)
|
|
136
|
+
try:
|
|
137
|
+
print(f"Attempting to stop and remove container '{CONTAINER_NAME}' (if running)...")
|
|
138
|
+
subprocess.run(["docker", "stop", CONTAINER_NAME],
|
|
139
|
+
check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
140
|
+
subprocess.run(["docker", "rm", CONTAINER_NAME],
|
|
141
|
+
check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
142
|
+
except Exception as e:
|
|
143
|
+
print(f"Error during preliminary container cleanup: {e}")
|
|
144
|
+
|
|
145
|
+
# Remove the Docker image
|
|
146
|
+
try:
|
|
147
|
+
print(f"Removing Docker image '{DOCKER_IMAGE_NAME}'...")
|
|
148
|
+
subprocess.run(["docker", "rmi", DOCKER_IMAGE_NAME], check=True)
|
|
149
|
+
print("Docker image removed.")
|
|
150
|
+
except subprocess.CalledProcessError as e:
|
|
151
|
+
print(f"Error removing Docker image '{DOCKER_IMAGE_NAME}': {e}")
|
|
152
|
+
|
|
153
|
+
# Remove the generated Dockerfile
|
|
154
|
+
if os.path.exists(os.path.join(where, "Dockerfile")):
|
|
155
|
+
os.remove(os.path.join(where, "Dockerfile"))
|
|
156
|
+
print("Removed Dockerfile.")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def run_in_docker(file_to_run: str, read_only_host_paths: list[str] = None, writable_host_paths: list[str] = None):
|
|
160
|
+
"""Runs the code in a Docker container with optional mounts."""
|
|
161
|
+
print(f"\nRunning code in Docker container '{CONTAINER_NAME}'...")
|
|
162
|
+
|
|
163
|
+
# Building command (it will continue below...)
|
|
164
|
+
command = ["docker", "run",
|
|
165
|
+
"--rm", # Automatically remove the container when it exits
|
|
166
|
+
"-e", "PYTHONUNBUFFERED=1", # Ensure Python output is unbuffered
|
|
167
|
+
"-e", "NODE_STARTING_PORT",
|
|
168
|
+
"--name", CONTAINER_NAME]
|
|
169
|
+
|
|
170
|
+
if sys.platform.startswith('linux'):
|
|
171
|
+
|
|
172
|
+
# Linux
|
|
173
|
+
command.extend(["--net", "host"]), # Expose the host network (in macOS and Windows it is still a virtual host)
|
|
174
|
+
else:
|
|
175
|
+
|
|
176
|
+
# Not-linux: check ports (adding -p port:port)
|
|
177
|
+
port_int = int(os.getenv("NODE_STARTING_PORT", "0"))
|
|
178
|
+
if port_int > 0:
|
|
179
|
+
command.extend(["-p", str(port_int + 0) + ":" + str(port_int + 0)])
|
|
180
|
+
command.extend(["-p", str(port_int + 1) + ":" + str(port_int + 1) + "/udp"])
|
|
181
|
+
command.extend(["-p", str(port_int + 2) + ":" + str(port_int + 2)])
|
|
182
|
+
command.extend(["-p", str(port_int + 3) + ":" + str(port_int + 3) + "/udp"])
|
|
183
|
+
|
|
184
|
+
# Add read-only mount if path is provided
|
|
185
|
+
if read_only_host_paths is not None and len(read_only_host_paths) > 0:
|
|
186
|
+
for path in read_only_host_paths:
|
|
187
|
+
|
|
188
|
+
# Ensure the host path exists and is a directory
|
|
189
|
+
if not os.path.isdir(path):
|
|
190
|
+
print(
|
|
191
|
+
f"Error: Read-only host path '{path}' does not exist or is not a directory. Cannot mount.")
|
|
192
|
+
return False
|
|
193
|
+
else:
|
|
194
|
+
|
|
195
|
+
# Augmenting command
|
|
196
|
+
path = os.path.abspath(path)
|
|
197
|
+
command.extend(["-v", f"{path}:{path}:ro"])
|
|
198
|
+
print(f"Mounted host '{path}' as read-only to container")
|
|
199
|
+
|
|
200
|
+
# Add writable mount if path is provided
|
|
201
|
+
if writable_host_paths is not None and len(writable_host_paths) > 0:
|
|
202
|
+
for path in writable_host_paths:
|
|
203
|
+
|
|
204
|
+
# Ensure the host path exists and is a directory
|
|
205
|
+
if not os.path.isdir(path):
|
|
206
|
+
print(
|
|
207
|
+
f"Error: Writable host path '{path}' does not exist or is not a directory. Cannot mount.")
|
|
208
|
+
return False
|
|
209
|
+
else:
|
|
210
|
+
|
|
211
|
+
# Augmenting command
|
|
212
|
+
path = os.path.abspath(path)
|
|
213
|
+
command.extend(["-v", f"{path}:{path}"])
|
|
214
|
+
print(f"Mounted host '{path}' as writable to container")
|
|
215
|
+
|
|
216
|
+
# Completing command
|
|
217
|
+
command.append(DOCKER_IMAGE_NAME)
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
|
|
221
|
+
# Running the prepared command... (using Popen to stream output in real-time)
|
|
222
|
+
try:
|
|
223
|
+
command.extend(["python3", file_to_run])
|
|
224
|
+
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
225
|
+
for line in iter(process.stdout.readline, ''):
|
|
226
|
+
sys.stdout.write(line)
|
|
227
|
+
process.wait() # Wait for the process to finish
|
|
228
|
+
if process.returncode != 0:
|
|
229
|
+
print(f"Container exited with non-zero status code: {process.returncode}")
|
|
230
|
+
except KeyboardInterrupt:
|
|
231
|
+
pass
|
|
232
|
+
|
|
233
|
+
print(f"\nContainer '{CONTAINER_NAME}' finished execution.")
|
|
234
|
+
return True
|
|
235
|
+
except FileNotFoundError:
|
|
236
|
+
print("Error: Docker command not found. Is Docker installed and in your PATH?")
|
|
237
|
+
print("Please ensure Docker is installed and running.")
|
|
238
|
+
return False
|
|
239
|
+
except subprocess.CalledProcessError as e:
|
|
240
|
+
print(f"Error running Docker container: {e}")
|
|
241
|
+
return False
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
# Entry point
|
|
245
|
+
if __name__ == "__main__":
|
|
246
|
+
parser = argparse.ArgumentParser(
|
|
247
|
+
description="Run a Python script adding customizable read-only and writable paths.",
|
|
248
|
+
formatter_class=argparse.RawTextHelpFormatter,
|
|
249
|
+
epilog="""
|
|
250
|
+
Examples:
|
|
251
|
+
python utils/sandbox.py my_script.py -r /home/user/data:/opt/app/data -p 1234
|
|
252
|
+
python utils/sandbox.py another_script.py -w /tmp/output:/mnt/results
|
|
253
|
+
python utils/sandbox.py script_with_both.py -r /input:/app/in -w /output:/app/out -p 8082
|
|
254
|
+
""")
|
|
255
|
+
parser.add_argument(help="Path to the Python script to execute.", dest="script_to_run",
|
|
256
|
+
type=str)
|
|
257
|
+
parser.add_argument("-p", "--port", dest="port",
|
|
258
|
+
help="The starting port of the node(s) (each node uses 4 ports, consecutive port numbers)",
|
|
259
|
+
type=str, required=True)
|
|
260
|
+
parser.add_argument("-r", "--read-only", dest="read_only_folders",
|
|
261
|
+
help="One or multiple paths to mount as read-only. "
|
|
262
|
+
"Use a colon to separate multiple paths (e.g., /path/a:/path/b).",
|
|
263
|
+
type=str, default=None)
|
|
264
|
+
parser.add_argument("-w", "--writable", dest="writable_folders",
|
|
265
|
+
help="One or multiple paths to mount as writable. "
|
|
266
|
+
"Use a colon to separate multiple paths (e.g., /path/c:/path/d).",
|
|
267
|
+
type=str, default=None)
|
|
268
|
+
args = parser.parse_args()
|
|
269
|
+
|
|
270
|
+
if not args.script_to_run.endswith(".py"):
|
|
271
|
+
parser.error(f"The script '{args.script_to_run}' must be a Python file (e.g., ending with .py)")
|
|
272
|
+
script_to_run = args.script_to_run
|
|
273
|
+
if not int(args.port) > 0:
|
|
274
|
+
parser.error(f"Invalid port")
|
|
275
|
+
|
|
276
|
+
read_only_folders = None
|
|
277
|
+
if args.read_only_folders:
|
|
278
|
+
read_only_folders = args.read_only_folders.split(':')
|
|
279
|
+
writable_folders = None
|
|
280
|
+
if args.writable_folders:
|
|
281
|
+
writable_folders = args.writable_folders.split(':')
|
|
282
|
+
|
|
283
|
+
print("\n Running in sandbox...")
|
|
284
|
+
print(f"- Script to run: {script_to_run}")
|
|
285
|
+
print(f"- Starting port (+0, +1, +2, +3): {args.port}")
|
|
286
|
+
print(f"- Read only paths to mount (the UNaIVERSE code folder will be automatically mounted): {read_only_folders}")
|
|
287
|
+
print(f"- Writable paths to mount: {writable_folders}\n")
|
|
288
|
+
|
|
289
|
+
# Marking
|
|
290
|
+
os.environ["NODE_STARTING_PORT"] = args.port
|
|
291
|
+
|
|
292
|
+
# Running the sandbox and the script
|
|
293
|
+
sandbox(script_to_run, read_only_paths=read_only_folders, writable_paths=writable_folders)
|