agi-cluster 2025.12.19__tar.gz → 2026.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agi_cluster-2025.12.19/src/agi_cluster.egg-info → agi_cluster-2026.2.6}/PKG-INFO +3 -3
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/README.md +1 -1
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/pyproject.toml +3 -2
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster/agi_distributor/agi_distributor.py +176 -58
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6/src/agi_cluster.egg-info}/PKG-INFO +3 -3
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster.egg-info/requires.txt +1 -1
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/LICENSE +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/MANIFEST.in +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/setup.cfg +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/__init__.py +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster/agi_distributor/__init__.py +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster/agi_distributor/cli.py +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster.egg-info/SOURCES.txt +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster.egg-info/dependency_links.txt +0 -0
- {agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agi-cluster
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2026.2.6
|
|
4
4
|
Summary: agi-cluster a framework for AGI
|
|
5
5
|
Author-email: Jean-Pierre Morard <focus@thalesgroup.com>
|
|
6
6
|
Project-URL: Documentation, https://thalesgroup.github.io/agilab
|
|
@@ -28,7 +28,7 @@ Requires-Dist: ipython
|
|
|
28
28
|
Requires-Dist: jupyter
|
|
29
29
|
Requires-Dist: msgpack
|
|
30
30
|
Requires-Dist: mypy
|
|
31
|
-
Requires-Dist: numba
|
|
31
|
+
Requires-Dist: numba>=0.61.0
|
|
32
32
|
Requires-Dist: parso
|
|
33
33
|
Requires-Dist: pathspec
|
|
34
34
|
Requires-Dist: psutil
|
|
@@ -47,7 +47,7 @@ Requires-Dist: wheel
|
|
|
47
47
|
Requires-Dist: cmake>=3.29
|
|
48
48
|
Dynamic: license-file
|
|
49
49
|
|
|
50
|
-
[](https://pypi.org/project/agi-cluster)
|
|
51
51
|
[](https://pypi.org/project/agilab/)
|
|
52
52
|
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
53
53
|
[]()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
[](https://pypi.org/project/agi-cluster)
|
|
2
2
|
[](https://pypi.org/project/agilab/)
|
|
3
3
|
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
4
4
|
[]()
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[project]
|
|
2
|
-
version = "
|
|
2
|
+
version = "2026.02.06"
|
|
3
3
|
name = "agi-cluster"
|
|
4
4
|
description = "agi-cluster a framework for AGI"
|
|
5
5
|
requires-python = ">=3.11"
|
|
@@ -41,7 +41,7 @@ dependencies = [
|
|
|
41
41
|
"jupyter",
|
|
42
42
|
"msgpack",
|
|
43
43
|
"mypy",
|
|
44
|
-
"numba",
|
|
44
|
+
"numba>=0.61.0",
|
|
45
45
|
"parso",
|
|
46
46
|
"pathspec",
|
|
47
47
|
"psutil",
|
|
@@ -119,6 +119,7 @@ Tracker = "https://github.com/ThalesGroup/agilab/issues"
|
|
|
119
119
|
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
|
|
122
123
|
|
|
123
124
|
|
|
124
125
|
[dependency-groups]
|
{agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster/agi_distributor/agi_distributor.py
RENAMED
|
@@ -32,7 +32,7 @@ import warnings
|
|
|
32
32
|
from copy import deepcopy
|
|
33
33
|
from datetime import timedelta
|
|
34
34
|
from ipaddress import ip_address as is_ip
|
|
35
|
-
from pathlib import Path
|
|
35
|
+
from pathlib import Path, PurePosixPath
|
|
36
36
|
from tempfile import gettempdir
|
|
37
37
|
|
|
38
38
|
from agi_cluster.agi_distributor import cli as distributor_cli
|
|
@@ -45,6 +45,61 @@ logger = logging.getLogger(__name__)
|
|
|
45
45
|
# uv path-source rewriting helpers
|
|
46
46
|
# ---------------------------------------------------------------------------
|
|
47
47
|
|
|
48
|
+
def _envar_truthy(envars: dict, key: str) -> bool:
|
|
49
|
+
"""Return True when an env var value is truthy.
|
|
50
|
+
|
|
51
|
+
Accepts common boolean-ish representations and defaults to False when unset
|
|
52
|
+
or unparsable.
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
raw = envars.get(key)
|
|
56
|
+
except Exception:
|
|
57
|
+
return False
|
|
58
|
+
if raw is None:
|
|
59
|
+
return False
|
|
60
|
+
if isinstance(raw, bool):
|
|
61
|
+
return raw
|
|
62
|
+
if isinstance(raw, (int, float)):
|
|
63
|
+
try:
|
|
64
|
+
return int(raw) == 1
|
|
65
|
+
except (TypeError, ValueError):
|
|
66
|
+
return False
|
|
67
|
+
value = str(raw).strip().lower()
|
|
68
|
+
return value in {"1", "true", "yes", "on"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _ensure_optional_extras(pyproject_file: Path, extras: Set[str]) -> None:
|
|
72
|
+
"""Ensure ``[project.optional-dependencies]`` contains the requested extras.
|
|
73
|
+
|
|
74
|
+
Some worker environments are bootstrapped from a manager ``pyproject.toml`` that
|
|
75
|
+
doesn't declare worker-only extras (e.g. ``polars-worker``). ``uv sync --extra``
|
|
76
|
+
fails hard when the extra doesn't exist, even if it would be empty.
|
|
77
|
+
"""
|
|
78
|
+
if not extras:
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
doc = tomlkit.parse(pyproject_file.read_text())
|
|
83
|
+
except FileNotFoundError:
|
|
84
|
+
doc = tomlkit.document()
|
|
85
|
+
|
|
86
|
+
project_tbl = doc.get("project")
|
|
87
|
+
if project_tbl is None:
|
|
88
|
+
project_tbl = tomlkit.table()
|
|
89
|
+
|
|
90
|
+
optional_tbl = project_tbl.get("optional-dependencies")
|
|
91
|
+
if optional_tbl is None or not isinstance(optional_tbl, tomlkit.items.Table):
|
|
92
|
+
optional_tbl = tomlkit.table()
|
|
93
|
+
|
|
94
|
+
for extra in sorted({e for e in extras if isinstance(e, str) and e.strip()}):
|
|
95
|
+
if extra not in optional_tbl:
|
|
96
|
+
optional_tbl[extra] = tomlkit.array()
|
|
97
|
+
|
|
98
|
+
project_tbl["optional-dependencies"] = optional_tbl
|
|
99
|
+
doc["project"] = project_tbl
|
|
100
|
+
pyproject_file.write_text(tomlkit.dumps(doc))
|
|
101
|
+
|
|
102
|
+
|
|
48
103
|
def _rewrite_uv_sources_paths_for_copied_pyproject(
|
|
49
104
|
*,
|
|
50
105
|
src_pyproject: Path,
|
|
@@ -301,6 +356,7 @@ class AGI:
|
|
|
301
356
|
verbose: Optional[int] = None
|
|
302
357
|
_worker_init_error: bool = False
|
|
303
358
|
_workers: Optional[Dict[str, int]] = None
|
|
359
|
+
_workers_data_path: Optional[str] = None
|
|
304
360
|
_capacity: Optional[Dict[str, float]] = None
|
|
305
361
|
_capacity_data_file: Optional[Path] = None
|
|
306
362
|
_capacity_model_file: Optional[Path] = None
|
|
@@ -348,6 +404,7 @@ class AGI:
|
|
|
348
404
|
env: AgiEnv, # some_default_value must be defined
|
|
349
405
|
scheduler: Optional[str] = None,
|
|
350
406
|
workers: Optional[Dict[str, int]] = None,
|
|
407
|
+
workers_data_path: Optional[str] = None,
|
|
351
408
|
verbose: int = 0,
|
|
352
409
|
mode: Optional[Union[int, List[int], str]] = None,
|
|
353
410
|
rapids_enabled: bool = False,
|
|
@@ -414,6 +471,7 @@ class AGI:
|
|
|
414
471
|
AGI._args = args
|
|
415
472
|
AGI.verbose = verbose
|
|
416
473
|
AGI._workers = workers
|
|
474
|
+
AGI._workers_data_path = workers_data_path
|
|
417
475
|
AGI._run_time = {}
|
|
418
476
|
|
|
419
477
|
AGI._capacity_data_file = env.resources_path / "balancer_df.csv"
|
|
@@ -1116,7 +1174,6 @@ class AGI:
|
|
|
1116
1174
|
cli_abs = env.wenv_abs.parent / cli_rel.name
|
|
1117
1175
|
cmd_prefix = env.envars.get(f"{ip}_CMD_PREFIX", "")
|
|
1118
1176
|
kill_prefix = f'{cmd_prefix}{uv} run --no-sync python'
|
|
1119
|
-
|
|
1120
1177
|
if env.is_local(ip):
|
|
1121
1178
|
if not (cli_abs).exists():
|
|
1122
1179
|
shutil.copy(env.cluster_pck / "agi_distributor/cli.py", cli_abs)
|
|
@@ -1126,7 +1183,7 @@ class AGI:
|
|
|
1126
1183
|
cmds.append(cmd)
|
|
1127
1184
|
else:
|
|
1128
1185
|
if force:
|
|
1129
|
-
cmd = f"{kill_prefix} '{cli_rel}' kill"
|
|
1186
|
+
cmd = f"{kill_prefix} '{cli_rel.as_posix()}' kill"
|
|
1130
1187
|
cmds.append(cmd)
|
|
1131
1188
|
|
|
1132
1189
|
last_res = None
|
|
@@ -1140,7 +1197,6 @@ class AGI:
|
|
|
1140
1197
|
else:
|
|
1141
1198
|
await AgiEnv.run(cmd, cwd)
|
|
1142
1199
|
else:
|
|
1143
|
-
cli = env.wenv_rel.parent / "cli.py"
|
|
1144
1200
|
last_res = await AGI.exec_ssh(ip, cmd)
|
|
1145
1201
|
|
|
1146
1202
|
# handle tuple or dict result
|
|
@@ -1226,7 +1282,7 @@ class AGI:
|
|
|
1226
1282
|
cmd_prefix = env.envars.get(f"{ip}_CMD_PREFIX", "")
|
|
1227
1283
|
wenv = env.wenv_rel
|
|
1228
1284
|
cli = wenv.parent / 'cli.py'
|
|
1229
|
-
cmd = (f"{cmd_prefix}{uv} run --no-sync -p {env.python_version} python {cli} clean {wenv}")
|
|
1285
|
+
cmd = (f"{cmd_prefix}{uv} run --no-sync -p {env.python_version} python {cli.as_posix()} clean {wenv}")
|
|
1230
1286
|
await AGI.exec_ssh(ip, cmd)
|
|
1231
1287
|
|
|
1232
1288
|
@staticmethod
|
|
@@ -1299,41 +1355,44 @@ class AGI:
|
|
|
1299
1355
|
logger.info(f"mkdir {wenv_abs}")
|
|
1300
1356
|
wenv_abs.mkdir(parents=True, exist_ok=True)
|
|
1301
1357
|
|
|
1302
|
-
if
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
uv_parts
|
|
1308
|
-
|
|
1358
|
+
if _envar_truthy(env.envars, "AGI_INTERNET_ON"):
|
|
1359
|
+
if os.name == "nt":
|
|
1360
|
+
standalone_uv = Path.home() / ".local" / "bin" / "uv.exe"
|
|
1361
|
+
if standalone_uv.exists():
|
|
1362
|
+
uv_parts = shlex.split(env.uv)
|
|
1363
|
+
if uv_parts:
|
|
1364
|
+
uv_parts[0] = str(standalone_uv)
|
|
1365
|
+
windows_uv = cmd_prefix + " ".join(shlex.quote(part) for part in uv_parts)
|
|
1366
|
+
else:
|
|
1367
|
+
windows_uv = cmd_prefix + shlex.quote(str(standalone_uv))
|
|
1368
|
+
try:
|
|
1369
|
+
await AgiEnv.run(f"{windows_uv} self update", wenv_abs.parent)
|
|
1370
|
+
except RuntimeError as exc:
|
|
1371
|
+
logger.warning(
|
|
1372
|
+
"Failed to update standalone uv at %s (skipping self update): %s",
|
|
1373
|
+
standalone_uv,
|
|
1374
|
+
exc,
|
|
1375
|
+
)
|
|
1309
1376
|
else:
|
|
1310
|
-
windows_uv = cmd_prefix + shlex.quote(str(standalone_uv))
|
|
1311
|
-
try:
|
|
1312
|
-
await AgiEnv.run(f"{windows_uv} self update", wenv_abs.parent)
|
|
1313
|
-
except RuntimeError as exc:
|
|
1314
1377
|
logger.warning(
|
|
1315
|
-
"
|
|
1378
|
+
"Standalone uv not found at %s; skipping 'uv self update' on Windows",
|
|
1316
1379
|
standalone_uv,
|
|
1317
|
-
exc,
|
|
1318
1380
|
)
|
|
1319
1381
|
else:
|
|
1320
|
-
|
|
1321
|
-
"Standalone uv not found at %s; skipping 'uv self update' on Windows",
|
|
1322
|
-
standalone_uv,
|
|
1323
|
-
)
|
|
1324
|
-
else:
|
|
1325
|
-
await AgiEnv.run(f"{uv} self update", wenv_abs.parent)
|
|
1382
|
+
await AgiEnv.run(f"{uv} self update", wenv_abs.parent)
|
|
1326
1383
|
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1384
|
+
try:
|
|
1385
|
+
await AgiEnv.run(f"{uv} python install {pyvers}", wenv_abs.parent)
|
|
1386
|
+
except RuntimeError as exc:
|
|
1387
|
+
if "No download found for request" in str(exc):
|
|
1388
|
+
logger.warning(
|
|
1389
|
+
"uv could not download interpreter '%s'; assuming a system interpreter is available",
|
|
1390
|
+
pyvers,
|
|
1391
|
+
)
|
|
1392
|
+
else:
|
|
1393
|
+
raise
|
|
1394
|
+
else:
|
|
1395
|
+
logger.warning("No internet connection detected; skipping uv update and assuming a system interpreter is available")
|
|
1337
1396
|
|
|
1338
1397
|
res = distributor_cli.python_version() or ""
|
|
1339
1398
|
pyvers = res.strip()
|
|
@@ -1387,13 +1446,21 @@ class AGI:
|
|
|
1387
1446
|
uv_is_installed = True
|
|
1388
1447
|
|
|
1389
1448
|
# 2) Check uv
|
|
1449
|
+
agi_internet_on = 1 if _envar_truthy(env.envars, "AGI_INTERNET_ON") else 0
|
|
1390
1450
|
try:
|
|
1391
1451
|
await AGI.exec_ssh(ip, f"{cmd_prefix}{env.uv} --version")
|
|
1392
|
-
|
|
1452
|
+
if agi_internet_on == 1:
|
|
1453
|
+
await AGI.exec_ssh(ip, f"{cmd_prefix}{env.uv} self update")
|
|
1454
|
+
else:
|
|
1455
|
+
logger.warning("You appears to be on a local network. Please be sure to have uv latest release.")
|
|
1393
1456
|
except ConnectionError:
|
|
1394
1457
|
raise
|
|
1395
1458
|
except Exception:
|
|
1396
1459
|
uv_is_installed = False
|
|
1460
|
+
if agi_internet_on == 0:
|
|
1461
|
+
logger.error("Uv binary is not installed, please install it manually on the workers.")
|
|
1462
|
+
raise EnvironmentError("Uv binary is not installed, please install it manually on the workers.")
|
|
1463
|
+
|
|
1397
1464
|
# Try Windows installer
|
|
1398
1465
|
try:
|
|
1399
1466
|
await AGI.exec_ssh(ip,
|
|
@@ -1410,7 +1477,7 @@ class AGI:
|
|
|
1410
1477
|
# await AGI.exec_ssh(ip, 'source ~/.local/bin/env')
|
|
1411
1478
|
uv_is_installed = True
|
|
1412
1479
|
|
|
1413
|
-
if not uv_is_installed
|
|
1480
|
+
if not uv_is_installed:
|
|
1414
1481
|
logger.error("Failed to install uv")
|
|
1415
1482
|
raise EnvironmentError("Failed to install uv")
|
|
1416
1483
|
|
|
@@ -1421,7 +1488,6 @@ class AGI:
|
|
|
1421
1488
|
cmd = f"{uv} run python -c \"import os; os.makedirs('{dist_rel.parents[1]}', exist_ok=True)\""
|
|
1422
1489
|
await AGI.exec_ssh(ip, cmd)
|
|
1423
1490
|
|
|
1424
|
-
await AGI.exec_ssh(ip, f"{uv} self update")
|
|
1425
1491
|
try:
|
|
1426
1492
|
await AGI.exec_ssh(ip, f"{uv} python install {pyvers_worker}")
|
|
1427
1493
|
except ProcessError as exc:
|
|
@@ -1443,8 +1509,25 @@ class AGI:
|
|
|
1443
1509
|
cmd = f"{uv} run python -c \"import os; os.makedirs('{dist_rel}', exist_ok=True)\""
|
|
1444
1510
|
await AGI.exec_ssh(ip, cmd)
|
|
1445
1511
|
|
|
1446
|
-
|
|
1447
|
-
|
|
1512
|
+
files_to_send: list[Path] = []
|
|
1513
|
+
pyproject_src = env.worker_pyproject if env.worker_pyproject.exists() else env.manager_pyproject
|
|
1514
|
+
if pyproject_src.exists():
|
|
1515
|
+
# Ensure the receiving worker can install with `uv sync --extra ...` even when
|
|
1516
|
+
# the source pyproject doesn't declare worker extras (common for built-in apps).
|
|
1517
|
+
extras_to_seed = set(getattr(AGI, "agi_workers", {}).values())
|
|
1518
|
+
try:
|
|
1519
|
+
tmp_dir = Path(gettempdir()) / f"agilab_{env.target_worker}_pyproject"
|
|
1520
|
+
tmp_dir.mkdir(parents=True, exist_ok=True)
|
|
1521
|
+
tmp_pyproject = tmp_dir / "pyproject.toml"
|
|
1522
|
+
shutil.copy(pyproject_src, tmp_pyproject)
|
|
1523
|
+
_ensure_optional_extras(tmp_pyproject, extras_to_seed)
|
|
1524
|
+
files_to_send.append(tmp_pyproject)
|
|
1525
|
+
except Exception:
|
|
1526
|
+
# Fall back to the original file if anything goes wrong.
|
|
1527
|
+
files_to_send.append(pyproject_src)
|
|
1528
|
+
if env.uvproject.exists():
|
|
1529
|
+
files_to_send.append(env.uvproject)
|
|
1530
|
+
await AGI.send_files(env, ip, files_to_send, wenv_rel)
|
|
1448
1531
|
|
|
1449
1532
|
@staticmethod
|
|
1450
1533
|
async def _deploy_application(scheduler_addr: Optional[str]) -> None:
|
|
@@ -1528,6 +1611,27 @@ class AGI:
|
|
|
1528
1611
|
dep_versions: dict[str, str] = {}
|
|
1529
1612
|
worker_pyprojects: set[str] = set()
|
|
1530
1613
|
|
|
1614
|
+
def _force_remove(path: Path) -> None:
|
|
1615
|
+
"""Suppression robuste : tente shutil, puis bascule sur rmdir /s /q en cas d'échec."""
|
|
1616
|
+
if not path.exists():
|
|
1617
|
+
return
|
|
1618
|
+
|
|
1619
|
+
def _on_err(func, p, exc):
|
|
1620
|
+
os.chmod(p, stat.S_IWRITE)
|
|
1621
|
+
try:
|
|
1622
|
+
func(p)
|
|
1623
|
+
except Exception:
|
|
1624
|
+
pass
|
|
1625
|
+
|
|
1626
|
+
try:
|
|
1627
|
+
shutil.rmtree(path, onerror=_on_err)
|
|
1628
|
+
except Exception:
|
|
1629
|
+
pass
|
|
1630
|
+
|
|
1631
|
+
if path.exists():
|
|
1632
|
+
AGI.env.logger.warn("Path {} still exists, using subprocess cmd to delete it.".format(path))
|
|
1633
|
+
subprocess.run(["cmd", "/c", "rmdir", "/s", "/q", str(path)], shell=True, check=False)
|
|
1634
|
+
|
|
1531
1635
|
def _cleanup_editable(site_packages: Path) -> None:
|
|
1532
1636
|
patterns = (
|
|
1533
1637
|
'__editable__.agi_env*.pth',
|
|
@@ -1762,8 +1866,9 @@ class AGI:
|
|
|
1762
1866
|
else:
|
|
1763
1867
|
cmd_manager = f"{extra_indexes}{uv} {run_type} --project '{app_path}'"
|
|
1764
1868
|
|
|
1765
|
-
#
|
|
1766
|
-
|
|
1869
|
+
# USE ROBUST REMOVE
|
|
1870
|
+
_force_remove(app_path / ".venv")
|
|
1871
|
+
|
|
1767
1872
|
try:
|
|
1768
1873
|
(app_path / "uv.lock").unlink()
|
|
1769
1874
|
except FileNotFoundError:
|
|
@@ -1800,7 +1905,7 @@ class AGI:
|
|
|
1800
1905
|
logger.info(f"mkdir {manager_resources.parent}")
|
|
1801
1906
|
manager_resources.parent.mkdir(parents=True, exist_ok=True)
|
|
1802
1907
|
if manager_resources.exists():
|
|
1803
|
-
|
|
1908
|
+
_force_remove(manager_resources)
|
|
1804
1909
|
shutil.copytree(resources_src, manager_resources, dirs_exist_ok=True)
|
|
1805
1910
|
|
|
1806
1911
|
site_packages_manager = env.env_pck.parent
|
|
@@ -1848,7 +1953,7 @@ class AGI:
|
|
|
1848
1953
|
filter_to_worker=True,
|
|
1849
1954
|
)
|
|
1850
1955
|
|
|
1851
|
-
|
|
1956
|
+
_force_remove(wenv_abs / ".venv")
|
|
1852
1957
|
|
|
1853
1958
|
if env.is_source_env:
|
|
1854
1959
|
# add missing agi-anv and agi-node as there are not in pyproject.toml as wished
|
|
@@ -1894,7 +1999,7 @@ class AGI:
|
|
|
1894
1999
|
logger.info(f"mkdir {resources_dest.parent}")
|
|
1895
2000
|
resources_dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1896
2001
|
if resources_dest.exists():
|
|
1897
|
-
|
|
2002
|
+
_force_remove(resources_dest)
|
|
1898
2003
|
if worker_resources_src.exists():
|
|
1899
2004
|
shutil.copytree(worker_resources_src, resources_dest, dirs_exist_ok=True)
|
|
1900
2005
|
|
|
@@ -2035,9 +2140,10 @@ class AGI:
|
|
|
2035
2140
|
f"--python {pyvers_worker} python -m {env.post_install_rel} "
|
|
2036
2141
|
f"{wenv_rel.stem}"
|
|
2037
2142
|
)
|
|
2143
|
+
|
|
2038
2144
|
if env.user and env.user != getpass.getuser():
|
|
2039
2145
|
try:
|
|
2040
|
-
await AGI.exec_ssh("127.0.0.1", post_install_cmd)
|
|
2146
|
+
await AGI.exec_ssh("127.0.0.1", post_install_cmd) #workaround for certain usecase (dont know which one)
|
|
2041
2147
|
except ConnectionError as exc:
|
|
2042
2148
|
logger.warning("SSH execution failed on localhost (%s), falling back to local run.", exc)
|
|
2043
2149
|
await AgiEnv.run(post_install_cmd, wenv_abs)
|
|
@@ -2070,6 +2176,11 @@ class AGI:
|
|
|
2070
2176
|
cmd_prefix = env.envars.get(f"{ip}_CMD_PREFIX", "")
|
|
2071
2177
|
uv = cmd_prefix + env.uv_worker
|
|
2072
2178
|
|
|
2179
|
+
# 1) set AGI_CLUSTER_SHARE on workers
|
|
2180
|
+
if AGI._workers_data_path:
|
|
2181
|
+
await AGI.exec_ssh(ip, "mkdir -p .agilab")
|
|
2182
|
+
await AGI.exec_ssh(ip, f"echo 'AGI_CLUSTER_SHARE=\"{Path(AGI._workers_data_path).expanduser().as_posix()}\"' > .agilab/.env")
|
|
2183
|
+
|
|
2073
2184
|
if env.is_source_env:
|
|
2074
2185
|
# Then send the files to the remote directory
|
|
2075
2186
|
egg_file = next(iter(dist_abs.glob(f"{env.target_worker}*.egg")), None)
|
|
@@ -2127,14 +2238,14 @@ class AGI:
|
|
|
2127
2238
|
|
|
2128
2239
|
# unzip egg to get src/
|
|
2129
2240
|
cli = env.wenv_rel.parent / "cli.py"
|
|
2130
|
-
cmd = f"{uv} run -p {pyvers} python {cli} unzip {wenv_rel}"
|
|
2241
|
+
cmd = f"{uv} run -p {pyvers} python {cli.as_posix()} unzip {wenv_rel.as_posix()}"
|
|
2131
2242
|
await AGI.exec_ssh(ip, cmd)
|
|
2132
2243
|
|
|
2133
2244
|
#############
|
|
2134
2245
|
# install env
|
|
2135
2246
|
#############
|
|
2136
2247
|
|
|
2137
|
-
cmd = f"{uv} --project {wenv_rel} run -p {pyvers} python -m ensurepip"
|
|
2248
|
+
cmd = f"{uv} --project {wenv_rel.as_posix()} run -p {pyvers} python -m ensurepip"
|
|
2138
2249
|
await AGI.exec_ssh(ip, cmd)
|
|
2139
2250
|
|
|
2140
2251
|
if env.is_source_env:
|
|
@@ -2145,21 +2256,21 @@ class AGI:
|
|
|
2145
2256
|
node_pck = "agi-node"
|
|
2146
2257
|
|
|
2147
2258
|
# install env
|
|
2148
|
-
cmd = f"{uv} --project {wenv_rel} add -p {pyvers} --upgrade {env_pck}"
|
|
2259
|
+
cmd = f"{uv} --project {wenv_rel.as_posix()} add -p {pyvers} --upgrade {env_pck.as_posix()}"
|
|
2149
2260
|
await AGI.exec_ssh(ip, cmd)
|
|
2150
2261
|
|
|
2151
2262
|
# install node
|
|
2152
|
-
cmd = f"{uv} --project {wenv_rel} add -p {pyvers} --upgrade {node_pck}"
|
|
2263
|
+
cmd = f"{uv} --project {wenv_rel.as_posix()} add -p {pyvers} --upgrade {node_pck.as_posix()}"
|
|
2153
2264
|
await AGI.exec_ssh(ip, cmd)
|
|
2154
2265
|
|
|
2155
2266
|
# unzip egg to get src/
|
|
2156
2267
|
cli = env.wenv_rel.parent / "cli.py"
|
|
2157
|
-
cmd = f"{uv} --project {wenv_rel} run --no-sync -p {pyvers} python {cli} unzip {wenv_rel}"
|
|
2268
|
+
cmd = f"{uv} --project {wenv_rel.as_posix()} run --no-sync -p {pyvers} python {cli.as_posix()} unzip {wenv_rel.as_posix()}"
|
|
2158
2269
|
await AGI.exec_ssh(ip, cmd)
|
|
2159
2270
|
|
|
2160
2271
|
# Post-install script
|
|
2161
2272
|
cmd = (
|
|
2162
|
-
f"{uv} --project {wenv_rel} run --no-sync -p {pyvers} python -m "
|
|
2273
|
+
f"{uv} --project {wenv_rel.as_posix()} run --no-sync -p {pyvers} python -m "
|
|
2163
2274
|
f"{env.post_install_rel} {wenv_rel.stem}"
|
|
2164
2275
|
)
|
|
2165
2276
|
await AGI.exec_ssh(ip, cmd)
|
|
@@ -2167,13 +2278,13 @@ class AGI:
|
|
|
2167
2278
|
# build target_worker lib from src/
|
|
2168
2279
|
if env.verbose > 1:
|
|
2169
2280
|
cmd = (
|
|
2170
|
-
f"{uv} --project '{wenv_rel}' run --no-sync -p {pyvers} python -m "
|
|
2171
|
-
f"agi_node.agi_dispatcher.build --app-path '{wenv_rel}' build_ext -b '{wenv_rel}'"
|
|
2281
|
+
f"{uv} --project '{wenv_rel.as_posix()}' run --no-sync -p {pyvers} python -m "
|
|
2282
|
+
f"agi_node.agi_dispatcher.build --app-path '{wenv_rel.as_posix()}' build_ext -b '{wenv_rel.as_posix()}'"
|
|
2172
2283
|
)
|
|
2173
2284
|
else:
|
|
2174
2285
|
cmd = (
|
|
2175
|
-
f"{uv} --project '{wenv_rel}' run --no-sync -p {pyvers} python -m "
|
|
2176
|
-
f"agi_node.agi_dispatcher.build --app-path '{wenv_rel}' -q build_ext -b '{wenv_rel}'"
|
|
2286
|
+
f"{uv} --project '{wenv_rel.as_posix()}' run --no-sync -p {pyvers} python -m "
|
|
2287
|
+
f"agi_node.agi_dispatcher.build --app-path '{wenv_rel.as_posix()}' -q build_ext -b '{wenv_rel.as_posix()}'"
|
|
2177
2288
|
)
|
|
2178
2289
|
await AGI.exec_ssh(ip, cmd)
|
|
2179
2290
|
|
|
@@ -2227,6 +2338,7 @@ class AGI:
|
|
|
2227
2338
|
env: AgiEnv,
|
|
2228
2339
|
scheduler: Optional[str] = None,
|
|
2229
2340
|
workers: Optional[Dict[str, int]] = None,
|
|
2341
|
+
workers_data_path: Optional[str] = None,
|
|
2230
2342
|
modes_enabled: int = _RUN_MASK,
|
|
2231
2343
|
verbose: Optional[int] = None,
|
|
2232
2344
|
**args: Any,
|
|
@@ -2263,6 +2375,7 @@ class AGI:
|
|
|
2263
2375
|
env=env,
|
|
2264
2376
|
scheduler=scheduler,
|
|
2265
2377
|
workers=workers,
|
|
2378
|
+
workers_data_path=workers_data_path,
|
|
2266
2379
|
mode=mode,
|
|
2267
2380
|
rapids_enabled=AGI._INSTALL_MODE & modes_enabled,
|
|
2268
2381
|
verbose=verbose, **args
|
|
@@ -2680,10 +2793,15 @@ class AGI:
|
|
|
2680
2793
|
wenv_arg = f"\"{wenv_abs}\"" # shlex.quote(str(wenv_abs))
|
|
2681
2794
|
|
|
2682
2795
|
worker_pyproject_dest = env.wenv_abs / env.worker_pyproject.name
|
|
2683
|
-
|
|
2684
|
-
|
|
2796
|
+
worker_pyproject_src = env.worker_pyproject if env.worker_pyproject.exists() else env.manager_pyproject
|
|
2797
|
+
if not worker_pyproject_src.exists():
|
|
2798
|
+
raise FileNotFoundError(f"Missing pyproject.toml for worker environment: {worker_pyproject_src}")
|
|
2799
|
+
shutil.copy(worker_pyproject_src, worker_pyproject_dest)
|
|
2800
|
+
_ensure_optional_extras(worker_pyproject_dest, set(getattr(AGI, "agi_workers", {}).values()))
|
|
2801
|
+
if env.uvproject.exists():
|
|
2802
|
+
shutil.copy(env.uvproject, env.wenv_abs)
|
|
2685
2803
|
_rewrite_uv_sources_paths_for_copied_pyproject(
|
|
2686
|
-
src_pyproject=
|
|
2804
|
+
src_pyproject=worker_pyproject_src,
|
|
2687
2805
|
dest_pyproject=worker_pyproject_dest,
|
|
2688
2806
|
log_rewrites=bool(env.verbose),
|
|
2689
2807
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agi-cluster
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2026.2.6
|
|
4
4
|
Summary: agi-cluster a framework for AGI
|
|
5
5
|
Author-email: Jean-Pierre Morard <focus@thalesgroup.com>
|
|
6
6
|
Project-URL: Documentation, https://thalesgroup.github.io/agilab
|
|
@@ -28,7 +28,7 @@ Requires-Dist: ipython
|
|
|
28
28
|
Requires-Dist: jupyter
|
|
29
29
|
Requires-Dist: msgpack
|
|
30
30
|
Requires-Dist: mypy
|
|
31
|
-
Requires-Dist: numba
|
|
31
|
+
Requires-Dist: numba>=0.61.0
|
|
32
32
|
Requires-Dist: parso
|
|
33
33
|
Requires-Dist: pathspec
|
|
34
34
|
Requires-Dist: psutil
|
|
@@ -47,7 +47,7 @@ Requires-Dist: wheel
|
|
|
47
47
|
Requires-Dist: cmake>=3.29
|
|
48
48
|
Dynamic: license-file
|
|
49
49
|
|
|
50
|
-
[](https://pypi.org/project/agi-cluster)
|
|
51
51
|
[](https://pypi.org/project/agilab/)
|
|
52
52
|
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
53
53
|
[]()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{agi_cluster-2025.12.19 → agi_cluster-2026.2.6}/src/agi_cluster.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|