eray 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eray/__init__.py +192 -0
- eray/cli/__init__.py +36 -0
- eray/cli/autoscale.py +418 -0
- eray/cli/dashboard.py +437 -0
- eray/cli/fleet.py +422 -0
- eray/cli/hostjob.py +506 -0
- eray/cli/jobs.py +1350 -0
- eray/cli/main.py +802 -0
- eray/cli/qr.py +249 -0
- eray/cli/tpu.py +763 -0
- eray/cli/utils.py +707 -0
- eray/core/__init__.py +63 -0
- eray/core/cluster.py +106 -0
- eray/core/exceptions.py +207 -0
- eray/core/monitoring.py +383 -0
- eray/core/sentinels.py +97 -0
- eray/core/status.py +105 -0
- eray/docker/__init__.py +48 -0
- eray/docker/config.py +131 -0
- eray/docker/runner.py +351 -0
- eray/execution/__init__.py +43 -0
- eray/execution/decorators.py +281 -0
- eray/execution/executor.py +741 -0
- eray/execution/remote.py +655 -0
- eray/gcp.py +177 -0
- eray/pool/__init__.py +40 -0
- eray/pool/base.py +284 -0
- eray/pool/device_host.py +717 -0
- eray/pool/slice.py +1195 -0
- eray/provision/__init__.py +66 -0
- eray/provision/fleet.py +332 -0
- eray/provision/launcher.py +497 -0
- eray/provision/qr.py +442 -0
- eray/provision/registry.py +510 -0
- eray/provision/templates/cluster-template.yaml +63 -0
- eray/provision/tunnel.py +424 -0
- eray/provision/watcher.py +777 -0
- eray/py.typed +0 -0
- eray/resources/__init__.py +50 -0
- eray/resources/configs.py +676 -0
- eray/resources/hardware.py +82 -0
- eray/resources/ray_resources.py +387 -0
- eray/resources/topology.py +344 -0
- eray/swarm.py +845 -0
- eray-0.1.0.dist-info/METADATA +561 -0
- eray-0.1.0.dist-info/RECORD +48 -0
- eray-0.1.0.dist-info/WHEEL +4 -0
- eray-0.1.0.dist-info/entry_points.txt +3 -0
eray/__init__.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
"""eray — Ray-based distributed execution and scaling framework.
|
|
17
|
+
|
|
18
|
+
A standalone extraction of the Ray executor stack from eFormer/EasyDeL.
|
|
19
|
+
Provides distributed execution of machine learning workloads using Ray,
|
|
20
|
+
with support for CPUs, GPUs, TPUs, Docker containerization, resource
|
|
21
|
+
pooling, multi-slice coordination, and fault-tolerant execution.
|
|
22
|
+
|
|
23
|
+
Subpackages:
|
|
24
|
+
core: Job status types, exceptions, sentinels, monitoring, cluster info
|
|
25
|
+
resources: Hardware constants, Ray resource specs, accelerator configs
|
|
26
|
+
pool: Actor pool management, device hosts, multi-slice coordination
|
|
27
|
+
execution: RayExecutor engine, decorators, device-remote machinery
|
|
28
|
+
docker: Docker container config and execution
|
|
29
|
+
|
|
30
|
+
Example:
|
|
31
|
+
Basic distributed execution:
|
|
32
|
+
|
|
33
|
+
>>> from eray import execute, GpuAcceleratorConfig
|
|
34
|
+
>>>
|
|
35
|
+
>>> @execute(GpuAcceleratorConfig(count=1, gpu_model="A100"))
|
|
36
|
+
... def train_step(batch):
|
|
37
|
+
... ...
|
|
38
|
+
... return loss
|
|
39
|
+
|
|
40
|
+
Multi-slice TPU execution with Docker:
|
|
41
|
+
|
|
42
|
+
>>> from eray import run_docker_multislice, DockerConfig, TpuAcceleratorConfig
|
|
43
|
+
>>>
|
|
44
|
+
>>> docker_config = DockerConfig(
|
|
45
|
+
... image="my-ml-image:latest",
|
|
46
|
+
... command="python train.py",
|
|
47
|
+
... )
|
|
48
|
+
>>> run_docker_multislice(
|
|
49
|
+
... docker_config,
|
|
50
|
+
... TpuAcceleratorConfig(tpu_version="v4-32", pod_count=4),
|
|
51
|
+
... )
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
from ray import init
|
|
55
|
+
from ray.runtime_env import RuntimeEnv
|
|
56
|
+
|
|
57
|
+
from .core import (
|
|
58
|
+
DONE,
|
|
59
|
+
DoneSentinel,
|
|
60
|
+
ExceptionInfo,
|
|
61
|
+
HostInfo,
|
|
62
|
+
JobError,
|
|
63
|
+
JobFailed,
|
|
64
|
+
JobInfo,
|
|
65
|
+
JobPreempted,
|
|
66
|
+
JobStatus,
|
|
67
|
+
JobSucceeded,
|
|
68
|
+
MultisliceInfo,
|
|
69
|
+
RefBox,
|
|
70
|
+
SliceInfo,
|
|
71
|
+
SnitchRecipient,
|
|
72
|
+
StopwatchActor,
|
|
73
|
+
current_actor_handle,
|
|
74
|
+
handle_ray_error,
|
|
75
|
+
log_failures_to,
|
|
76
|
+
print_remote_raise,
|
|
77
|
+
start_raylet_log_guard,
|
|
78
|
+
sweep_raylet_logs,
|
|
79
|
+
)
|
|
80
|
+
from .docker import (
|
|
81
|
+
DockerConfig,
|
|
82
|
+
build_and_push_docker_image,
|
|
83
|
+
make_docker_run_command,
|
|
84
|
+
run_docker_async,
|
|
85
|
+
run_docker_multislice,
|
|
86
|
+
run_docker_on_pod,
|
|
87
|
+
)
|
|
88
|
+
from .execution import (
|
|
89
|
+
ENV_CALL_INDEX,
|
|
90
|
+
ENV_CALL_SLICE,
|
|
91
|
+
MEGASCALE_DEFAULT_PORT,
|
|
92
|
+
RayExecutor,
|
|
93
|
+
autoscale_execute,
|
|
94
|
+
autoscale_execute_resumable,
|
|
95
|
+
device_remote,
|
|
96
|
+
execute,
|
|
97
|
+
execute_resumable,
|
|
98
|
+
resolve_maybe_refs,
|
|
99
|
+
)
|
|
100
|
+
from .pool import (
|
|
101
|
+
ActorPoolMember,
|
|
102
|
+
DeviceHostActor,
|
|
103
|
+
InsufficientSlicesError,
|
|
104
|
+
ResourcePoolManager,
|
|
105
|
+
SliceActor,
|
|
106
|
+
SlicePoolManager,
|
|
107
|
+
)
|
|
108
|
+
from .provision import ensure_tpu, watch_and_reconnect
|
|
109
|
+
from .resources import (
|
|
110
|
+
AcceleratorConfigType,
|
|
111
|
+
ComputeResourceConfig,
|
|
112
|
+
CpuAcceleratorConfig,
|
|
113
|
+
GpuAcceleratorConfig,
|
|
114
|
+
HardwareType,
|
|
115
|
+
HostPartitionPlan,
|
|
116
|
+
HostSplitPlan,
|
|
117
|
+
RayResources,
|
|
118
|
+
TpuAcceleratorConfig,
|
|
119
|
+
available_cpu_cores,
|
|
120
|
+
plan_host_partition,
|
|
121
|
+
plan_host_split,
|
|
122
|
+
)
|
|
123
|
+
from .swarm import GpuSwarmConfig, SwarmConfig, SwarmRun, shutdown_swarm, swarm_execute, swarmed
|
|
124
|
+
|
|
125
|
+
__version__ = "0.1.0"
|
|
126
|
+
__all__ = (
|
|
127
|
+
"DONE",
|
|
128
|
+
"ENV_CALL_INDEX",
|
|
129
|
+
"ENV_CALL_SLICE",
|
|
130
|
+
"MEGASCALE_DEFAULT_PORT",
|
|
131
|
+
"AcceleratorConfigType",
|
|
132
|
+
"ActorPoolMember",
|
|
133
|
+
"ComputeResourceConfig",
|
|
134
|
+
"CpuAcceleratorConfig",
|
|
135
|
+
"DeviceHostActor",
|
|
136
|
+
"DockerConfig",
|
|
137
|
+
"DoneSentinel",
|
|
138
|
+
"ExceptionInfo",
|
|
139
|
+
"GpuAcceleratorConfig",
|
|
140
|
+
"GpuSwarmConfig",
|
|
141
|
+
"HardwareType",
|
|
142
|
+
"HostInfo",
|
|
143
|
+
"HostPartitionPlan",
|
|
144
|
+
"HostSplitPlan",
|
|
145
|
+
"InsufficientSlicesError",
|
|
146
|
+
"JobError",
|
|
147
|
+
"JobFailed",
|
|
148
|
+
"JobInfo",
|
|
149
|
+
"JobPreempted",
|
|
150
|
+
"JobStatus",
|
|
151
|
+
"JobSucceeded",
|
|
152
|
+
"MultisliceInfo",
|
|
153
|
+
"RayExecutor",
|
|
154
|
+
"RayResources",
|
|
155
|
+
"RefBox",
|
|
156
|
+
"ResourcePoolManager",
|
|
157
|
+
"RuntimeEnv",
|
|
158
|
+
"SliceActor",
|
|
159
|
+
"SliceInfo",
|
|
160
|
+
"SlicePoolManager",
|
|
161
|
+
"SnitchRecipient",
|
|
162
|
+
"StopwatchActor",
|
|
163
|
+
"SwarmConfig",
|
|
164
|
+
"SwarmRun",
|
|
165
|
+
"TpuAcceleratorConfig",
|
|
166
|
+
"autoscale_execute",
|
|
167
|
+
"autoscale_execute_resumable",
|
|
168
|
+
"available_cpu_cores",
|
|
169
|
+
"build_and_push_docker_image",
|
|
170
|
+
"current_actor_handle",
|
|
171
|
+
"device_remote",
|
|
172
|
+
"ensure_tpu",
|
|
173
|
+
"execute",
|
|
174
|
+
"execute_resumable",
|
|
175
|
+
"handle_ray_error",
|
|
176
|
+
"init",
|
|
177
|
+
"log_failures_to",
|
|
178
|
+
"make_docker_run_command",
|
|
179
|
+
"plan_host_partition",
|
|
180
|
+
"plan_host_split",
|
|
181
|
+
"print_remote_raise",
|
|
182
|
+
"resolve_maybe_refs",
|
|
183
|
+
"run_docker_async",
|
|
184
|
+
"run_docker_multislice",
|
|
185
|
+
"run_docker_on_pod",
|
|
186
|
+
"shutdown_swarm",
|
|
187
|
+
"start_raylet_log_guard",
|
|
188
|
+
"swarm_execute",
|
|
189
|
+
"swarmed",
|
|
190
|
+
"sweep_raylet_logs",
|
|
191
|
+
"watch_and_reconnect",
|
|
192
|
+
)
|
eray/cli/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Command-line interface for eray.
|
|
16
|
+
|
|
17
|
+
Provides commands for connecting TPU hosts into a Ray cluster and
|
|
18
|
+
managing cluster lifecycle.
|
|
19
|
+
|
|
20
|
+
Example:
|
|
21
|
+
# Connect a TPU slice into a Ray cluster
|
|
22
|
+
$ eray tpu connect --tpu-name my-tpu --project my-project --zone us-central2-b
|
|
23
|
+
|
|
24
|
+
# Check cluster status
|
|
25
|
+
$ eray tpu status --address 10.0.0.1:6379
|
|
26
|
+
|
|
27
|
+
# Run a health check
|
|
28
|
+
$ eray tpu health --address 10.0.0.1:6379
|
|
29
|
+
|
|
30
|
+
# Disconnect (stop Ray on all hosts)
|
|
31
|
+
$ eray tpu disconnect --tpu-name my-tpu --project my-project --zone us-central2-b
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from .main import cli
|
|
35
|
+
|
|
36
|
+
__all__ = ("cli",)
|
eray/cli/autoscale.py
ADDED
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
# Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""`eray autoscale` — Ray cluster-launcher configs for elastic TPU clusters.
|
|
16
|
+
|
|
17
|
+
Generates per-zone launcher YAMLs (all-spot by default; every TPU node type
|
|
18
|
+
carries the eray-canonical resource labels so the pool scheduler works) and
|
|
19
|
+
wraps `ray up/down/get-head-ip`. For reliable large slices, prefer the
|
|
20
|
+
QR-based `eray fleet` path; this one is for elastic many-small-slices use.
|
|
21
|
+
|
|
22
|
+
`up`/`down`/`status` never require a YAML path: with no CONFIG they discover
|
|
23
|
+
generated profiles under `--dir` and either auto-select the only one or offer
|
|
24
|
+
a numbered picker. Every `up`/`down` also upserts a ``kind="launcher"``
|
|
25
|
+
record into the same fleet registry `eray fleet` uses, which is what lets
|
|
26
|
+
`eray autoscale status` (no CONFIG) and `eray dashboard` show every launcher
|
|
27
|
+
cluster's live state without the operator tracking YAML paths by hand.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import json as json_lib
|
|
33
|
+
import time
|
|
34
|
+
|
|
35
|
+
import click
|
|
36
|
+
|
|
37
|
+
from ..provision.registry import ClusterRecord, ClusterRegistry
|
|
38
|
+
from .utils import detect_project, error, info, print_table, success, warning
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _pick_profile(profiles: list, *, verb: str):
|
|
42
|
+
"""Resolve which profile to act on when no CONFIG was given.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
profiles: Discovered profiles (see `provision.launcher.list_profiles`).
|
|
46
|
+
verb: The action being taken, for the prompt copy (e.g. "bring up").
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
The chosen `AutoscaleProfile`.
|
|
50
|
+
|
|
51
|
+
Raises:
|
|
52
|
+
click.ClickException: No profiles exist, or no selection was made
|
|
53
|
+
(e.g. stdin has nothing left to read — a non-interactive call).
|
|
54
|
+
"""
|
|
55
|
+
if not profiles:
|
|
56
|
+
raise click.ClickException("no generated profiles found — run `eray autoscale generate` first.")
|
|
57
|
+
if len(profiles) == 1:
|
|
58
|
+
profile = profiles[0]
|
|
59
|
+
info(f"one profile found: {profile.name} — using it.")
|
|
60
|
+
return profile
|
|
61
|
+
click.echo(f"Multiple profiles found — choose one to {verb}:")
|
|
62
|
+
for idx, profile in enumerate(profiles, start=1):
|
|
63
|
+
click.echo(f" {idx}) {profile.name} ({profile.zone or '?'}, {profile.project or '?'})")
|
|
64
|
+
try:
|
|
65
|
+
choice = click.prompt("select", type=click.IntRange(1, len(profiles)))
|
|
66
|
+
except click.Abort:
|
|
67
|
+
raise click.ClickException(
|
|
68
|
+
"no profile selected — pass a profile name explicitly (non-interactive session?)."
|
|
69
|
+
) from None
|
|
70
|
+
return profiles[choice - 1]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _upsert_launcher_record(profile, *, state: str, stamp: str, head_ip: str | None = None) -> None:
|
|
74
|
+
"""Persist a launcher-kind cluster's state after `up`/`down`.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
profile: The `AutoscaleProfile` that was acted on.
|
|
78
|
+
state: New `ClusterRecord.state` (``"HEALTHY"`` or ``"DOWN"``).
|
|
79
|
+
stamp: Which timestamp field to stamp with now — ``"last_up_ts"`` or
|
|
80
|
+
``"last_down_ts"``.
|
|
81
|
+
head_ip: Best-effort head IP (`up` only; `down` leaves it as-is).
|
|
82
|
+
"""
|
|
83
|
+
registry = ClusterRegistry.from_config()
|
|
84
|
+
if registry.get(profile.name) is None:
|
|
85
|
+
registry.upsert(
|
|
86
|
+
ClusterRecord(
|
|
87
|
+
name=profile.name,
|
|
88
|
+
kind="launcher",
|
|
89
|
+
project=profile.project,
|
|
90
|
+
zone=profile.zone,
|
|
91
|
+
config_path=str(profile.path),
|
|
92
|
+
state=state,
|
|
93
|
+
head_ip=head_ip,
|
|
94
|
+
**{stamp: time.time()},
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
def mutate(r: ClusterRecord) -> None:
|
|
100
|
+
r.kind = "launcher"
|
|
101
|
+
r.project = profile.project or r.project
|
|
102
|
+
r.zone = profile.zone or r.zone
|
|
103
|
+
r.config_path = str(profile.path)
|
|
104
|
+
r.state = state
|
|
105
|
+
if head_ip is not None:
|
|
106
|
+
r.head_ip = head_ip
|
|
107
|
+
if stamp == "last_up_ts":
|
|
108
|
+
# A fresh bring-up invalidates any earlier "died at T" history —
|
|
109
|
+
# otherwise a live cluster can flash DOWN/"died ... ago" if the
|
|
110
|
+
# GCE probe runs before the new head instance is discoverable.
|
|
111
|
+
r.last_down_ts = None
|
|
112
|
+
setattr(r, stamp, time.time())
|
|
113
|
+
|
|
114
|
+
registry.mutate_record(profile.name, mutate)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def launcher_row_status(
|
|
118
|
+
cluster_name: str, *, project: str | None, zone: str | None, state: str, last_down_ts: float | None
|
|
119
|
+
) -> tuple[str, str]:
|
|
120
|
+
"""Live (status, since) for one launcher-kind cluster.
|
|
121
|
+
|
|
122
|
+
Shared by `eray autoscale status` (no CONFIG) and `eray dashboard` so
|
|
123
|
+
the two views can never disagree about what "running" means.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
cluster_name: The launcher config's ``cluster_name``.
|
|
127
|
+
project: GCP project id, or None (skips the live probe).
|
|
128
|
+
zone: GCP zone, or None (skips the live probe).
|
|
129
|
+
state: Fallback status when no live head instance is found and
|
|
130
|
+
there's no `last_down_ts` to report against (e.g. registered
|
|
131
|
+
but never brought up).
|
|
132
|
+
last_down_ts: Unix timestamp of the last `eray autoscale down`, or
|
|
133
|
+
None.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
``(status, since)`` display strings — status is either a live GCE
|
|
137
|
+
instance status (``RUNNING``, ``STAGING``, ...), ``"DOWN"``, or the
|
|
138
|
+
fallback `state`; since is "up <duration>", "died <duration> ago",
|
|
139
|
+
"?" (status known, age unknown), or "-".
|
|
140
|
+
"""
|
|
141
|
+
from ..provision.launcher import gce_instances_for_cluster
|
|
142
|
+
from .utils import format_ago, format_duration, parse_gcp_timestamp
|
|
143
|
+
|
|
144
|
+
instances = gce_instances_for_cluster(cluster_name, project=project, zone=zone) if project and zone else []
|
|
145
|
+
head = next((i for i in instances if (i.get("labels") or {}).get("ray-node-type") == "head"), None)
|
|
146
|
+
if head is not None:
|
|
147
|
+
status = str(head.get("status", "UNKNOWN"))
|
|
148
|
+
created = parse_gcp_timestamp(head.get("creationTimestamp"))
|
|
149
|
+
since = f"up {format_duration(time.time() - created)}" if created else "?"
|
|
150
|
+
elif last_down_ts is not None:
|
|
151
|
+
status = "DOWN"
|
|
152
|
+
since = f"died {format_ago(last_down_ts)}"
|
|
153
|
+
else:
|
|
154
|
+
status = state
|
|
155
|
+
since = "-"
|
|
156
|
+
return status, since
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _launcher_status_rows(profiles_dir) -> list[dict]:
|
|
160
|
+
"""Merge on-disk profiles and registered launcher clusters into rows.
|
|
161
|
+
|
|
162
|
+
Each row's live GCE probe is an independent gcloud subprocess; a thread
|
|
163
|
+
pool runs them concurrently so `eray autoscale status` doesn't scale
|
|
164
|
+
linearly with the number of profiles (mirrors `run_on_all_hosts` in
|
|
165
|
+
`cli/utils.py`).
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
profiles_dir: Directory profiles are discovered in.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
One row per known profile (on disk, registered, or both): name,
|
|
172
|
+
zone, status (a live GCE probe when project/zone are known), head,
|
|
173
|
+
since (uptime, or "died N ago", or "-").
|
|
174
|
+
"""
|
|
175
|
+
import concurrent.futures
|
|
176
|
+
|
|
177
|
+
from ..provision.launcher import list_profiles
|
|
178
|
+
|
|
179
|
+
registry = ClusterRegistry.from_config()
|
|
180
|
+
records = {name: r for name, r in registry.load().items() if r.kind == "launcher"}
|
|
181
|
+
profiles = {p.name: p for p in list_profiles(profiles_dir)}
|
|
182
|
+
names = sorted(set(profiles) | set(records))
|
|
183
|
+
if not names:
|
|
184
|
+
return []
|
|
185
|
+
|
|
186
|
+
def build_row(name: str) -> dict:
|
|
187
|
+
profile = profiles.get(name)
|
|
188
|
+
record = records.get(name)
|
|
189
|
+
project = (profile.project if profile else None) or (record.project if record else None)
|
|
190
|
+
zone = (profile.zone if profile else None) or (record.zone if record else None)
|
|
191
|
+
cluster_name = profile.cluster_name if profile else name
|
|
192
|
+
|
|
193
|
+
status, since = launcher_row_status(
|
|
194
|
+
cluster_name,
|
|
195
|
+
project=project,
|
|
196
|
+
zone=zone,
|
|
197
|
+
state=record.state if record else "UNREGISTERED",
|
|
198
|
+
last_down_ts=record.last_down_ts if record else None,
|
|
199
|
+
)
|
|
200
|
+
return {
|
|
201
|
+
"name": name,
|
|
202
|
+
"zone": zone or "?",
|
|
203
|
+
"status": status,
|
|
204
|
+
"head": (record.head_ip if record and status == "RUNNING" else None) or "-",
|
|
205
|
+
"since": since,
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(names), 8)) as pool:
|
|
209
|
+
return list(pool.map(build_row, names))
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def register(cli: click.Group) -> None:
|
|
213
|
+
"""Register the `autoscale` command group on the root CLI.
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
cli: The root click group.
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
@cli.group()
|
|
220
|
+
def autoscale() -> None:
|
|
221
|
+
"""Ray cluster-launcher configs for elastic (autoscaled) TPU clusters."""
|
|
222
|
+
|
|
223
|
+
@autoscale.command()
|
|
224
|
+
@click.option("--project", "-p", default=None, help="GCP project (auto-detected).")
|
|
225
|
+
@click.option("--zones", "-z", default=None, help="Comma-separated zones (default: every TPU location).")
|
|
226
|
+
@click.option("--families", default="v4,v5e,v5p,v6e", show_default=True, help="TPU families to include.")
|
|
227
|
+
@click.option("--image", default=None, help="Docker image for head/workers (default: easydel latest-tpu).")
|
|
228
|
+
@click.option(
|
|
229
|
+
"--spot/--on-demand", "spot", default=True, show_default=True, help="Capacity tier for TPU node types."
|
|
230
|
+
)
|
|
231
|
+
@click.option(
|
|
232
|
+
"--output-dir",
|
|
233
|
+
"-o",
|
|
234
|
+
"output_dir",
|
|
235
|
+
default=None,
|
|
236
|
+
help="Where to write the YAMLs (default: ~/.eray/autoscale).",
|
|
237
|
+
)
|
|
238
|
+
def generate(project, zones, families, image, spot, output_dir):
|
|
239
|
+
"""Generate per-zone cluster-launcher YAMLs.
|
|
240
|
+
|
|
241
|
+
\b
|
|
242
|
+
Examples:
|
|
243
|
+
eray autoscale generate --zones us-central1-a
|
|
244
|
+
eray autoscale generate --families v5p --on-demand
|
|
245
|
+
"""
|
|
246
|
+
from ..provision.launcher import DEFAULT_IMAGE, DEFAULT_OUTPUT_DIR, generate_configs
|
|
247
|
+
|
|
248
|
+
project = project or detect_project()
|
|
249
|
+
if not project:
|
|
250
|
+
raise click.ClickException("Could not resolve a GCP project; pass --project.")
|
|
251
|
+
zone_list = [z.strip() for z in zones.split(",") if z.strip()] if zones else None
|
|
252
|
+
family_tuple = tuple(f.strip() for f in families.split(",") if f.strip())
|
|
253
|
+
try:
|
|
254
|
+
written = generate_configs(
|
|
255
|
+
project,
|
|
256
|
+
zones=zone_list,
|
|
257
|
+
families=family_tuple,
|
|
258
|
+
image=image or DEFAULT_IMAGE,
|
|
259
|
+
spot=spot,
|
|
260
|
+
output_dir=output_dir or DEFAULT_OUTPUT_DIR,
|
|
261
|
+
)
|
|
262
|
+
except Exception as exc:
|
|
263
|
+
error(str(exc))
|
|
264
|
+
raise click.ClickException(str(exc)) from exc
|
|
265
|
+
if not written:
|
|
266
|
+
info("no zones offered the requested TPU families; nothing written.")
|
|
267
|
+
return
|
|
268
|
+
for path in written:
|
|
269
|
+
success(f"wrote {path}")
|
|
270
|
+
|
|
271
|
+
_dir_opt = click.option(
|
|
272
|
+
"--dir",
|
|
273
|
+
"profiles_dir",
|
|
274
|
+
default=None,
|
|
275
|
+
help="Where generated profiles are discovered from (default: ~/.eray/autoscale).",
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
@autoscale.command()
|
|
279
|
+
@click.argument("config", required=False)
|
|
280
|
+
@_dir_opt
|
|
281
|
+
@click.option("--yes", "-y", is_flag=True, default=False, help="Skip the ray up confirmation.")
|
|
282
|
+
def up(config, profiles_dir, yes):
|
|
283
|
+
"""Bring a launcher cluster up (`ray up`).
|
|
284
|
+
|
|
285
|
+
\b
|
|
286
|
+
With no CONFIG: picks from generated profiles (auto-selects the
|
|
287
|
+
only one, prompts when there's more than one). CONFIG may also be a
|
|
288
|
+
YAML path or a bare profile name (its file stem) to skip the picker.
|
|
289
|
+
|
|
290
|
+
\b
|
|
291
|
+
Examples:
|
|
292
|
+
eray autoscale up # picker
|
|
293
|
+
eray autoscale up easydel-us-east5-a # by profile name
|
|
294
|
+
eray autoscale up ./my-cluster.yaml # by path
|
|
295
|
+
"""
|
|
296
|
+
from ..provision.launcher import (
|
|
297
|
+
DEFAULT_OUTPUT_DIR,
|
|
298
|
+
launcher_head_ip,
|
|
299
|
+
list_profiles,
|
|
300
|
+
load_profile,
|
|
301
|
+
ray_up,
|
|
302
|
+
resolve_profile,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
|
|
306
|
+
if config:
|
|
307
|
+
try:
|
|
308
|
+
path = resolve_profile(config, profiles_dir)
|
|
309
|
+
except FileNotFoundError as exc:
|
|
310
|
+
raise click.ClickException(str(exc)) from exc
|
|
311
|
+
else:
|
|
312
|
+
path = _pick_profile(list_profiles(profiles_dir), verb="bring up").path
|
|
313
|
+
|
|
314
|
+
if not yes:
|
|
315
|
+
click.confirm(f"ray up {path} — create/update this cluster?", abort=True)
|
|
316
|
+
try:
|
|
317
|
+
ray_up(path, yes=True)
|
|
318
|
+
except Exception as exc:
|
|
319
|
+
error(str(exc))
|
|
320
|
+
raise click.ClickException(str(exc)) from exc
|
|
321
|
+
success(f"cluster up: {path}")
|
|
322
|
+
|
|
323
|
+
profile = load_profile(path)
|
|
324
|
+
if profile is None:
|
|
325
|
+
warning("could not re-read cluster_name from CONFIG; skipping fleet registry update.")
|
|
326
|
+
return
|
|
327
|
+
head_ip = None
|
|
328
|
+
try:
|
|
329
|
+
head_ip = launcher_head_ip(path)
|
|
330
|
+
except Exception as exc:
|
|
331
|
+
warning(f"could not resolve head IP ({exc}); registering without it — `eray dashboard ls` will show '-'.")
|
|
332
|
+
_upsert_launcher_record(profile, state="HEALTHY", stamp="last_up_ts", head_ip=head_ip)
|
|
333
|
+
|
|
334
|
+
@autoscale.command()
|
|
335
|
+
@click.argument("config", required=False)
|
|
336
|
+
@_dir_opt
|
|
337
|
+
@click.option("--yes", "-y", is_flag=True, default=False, help="Skip the ray down confirmation.")
|
|
338
|
+
def down(config, profiles_dir, yes):
|
|
339
|
+
"""Tear a launcher cluster down (`ray down`).
|
|
340
|
+
|
|
341
|
+
\b
|
|
342
|
+
With no CONFIG: picks from generated/registered profiles the same
|
|
343
|
+
way `up` does. CONFIG may also be a YAML path or a bare profile name.
|
|
344
|
+
"""
|
|
345
|
+
from ..provision.launcher import (
|
|
346
|
+
DEFAULT_OUTPUT_DIR,
|
|
347
|
+
list_profiles,
|
|
348
|
+
load_profile,
|
|
349
|
+
ray_down,
|
|
350
|
+
resolve_profile,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
|
|
354
|
+
if config:
|
|
355
|
+
try:
|
|
356
|
+
path = resolve_profile(config, profiles_dir)
|
|
357
|
+
except FileNotFoundError as exc:
|
|
358
|
+
raise click.ClickException(str(exc)) from exc
|
|
359
|
+
else:
|
|
360
|
+
path = _pick_profile(list_profiles(profiles_dir), verb="tear down").path
|
|
361
|
+
|
|
362
|
+
if not yes:
|
|
363
|
+
click.confirm(f"ray down {path} — terminate this cluster's nodes?", abort=True)
|
|
364
|
+
try:
|
|
365
|
+
ray_down(path, yes=True)
|
|
366
|
+
except Exception as exc:
|
|
367
|
+
error(str(exc))
|
|
368
|
+
raise click.ClickException(str(exc)) from exc
|
|
369
|
+
success(f"cluster down: {path}")
|
|
370
|
+
|
|
371
|
+
profile = load_profile(path)
|
|
372
|
+
if profile is None:
|
|
373
|
+
warning("could not re-read cluster_name from CONFIG; skipping fleet registry update.")
|
|
374
|
+
return
|
|
375
|
+
_upsert_launcher_record(profile, state="DOWN", stamp="last_down_ts")
|
|
376
|
+
|
|
377
|
+
@autoscale.command()
|
|
378
|
+
@click.argument("config", required=False)
|
|
379
|
+
@_dir_opt
|
|
380
|
+
@click.option("--json", "as_json", is_flag=True, default=False, help="Output the no-CONFIG table as JSON.")
|
|
381
|
+
def status(config, profiles_dir, as_json):
|
|
382
|
+
"""Show launcher clusters: every known profile, or one cluster's detail.
|
|
383
|
+
|
|
384
|
+
\b
|
|
385
|
+
With no CONFIG: a table of every generated/registered profile —
|
|
386
|
+
zone, live status (a GCE instance-label probe), head, and how long
|
|
387
|
+
it's been up (or when it went down). With CONFIG: one cluster's
|
|
388
|
+
head + resource-usage detail (today's behavior, unchanged).
|
|
389
|
+
"""
|
|
390
|
+
from ..provision.launcher import DEFAULT_OUTPUT_DIR
|
|
391
|
+
|
|
392
|
+
profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
|
|
393
|
+
if config:
|
|
394
|
+
from ..provision.fleet import head_reachable
|
|
395
|
+
from ..provision.launcher import launcher_head_ip, resolve_profile
|
|
396
|
+
|
|
397
|
+
try:
|
|
398
|
+
path = resolve_profile(config, profiles_dir)
|
|
399
|
+
except FileNotFoundError as exc:
|
|
400
|
+
raise click.ClickException(str(exc)) from exc
|
|
401
|
+
try:
|
|
402
|
+
head_ip = launcher_head_ip(path)
|
|
403
|
+
except Exception as exc:
|
|
404
|
+
error(f"cluster not reachable via ray get-head-ip: {exc}")
|
|
405
|
+
raise click.ClickException(str(exc)) from exc
|
|
406
|
+
info(f"head: {head_ip}")
|
|
407
|
+
info(f"ray head port: {'up' if head_reachable(head_ip) else 'down'}")
|
|
408
|
+
info(f"resources: eray resources -a {head_ip}:6379")
|
|
409
|
+
return
|
|
410
|
+
|
|
411
|
+
rows = _launcher_status_rows(profiles_dir)
|
|
412
|
+
if as_json:
|
|
413
|
+
click.echo(json_lib.dumps(rows, indent=2))
|
|
414
|
+
return
|
|
415
|
+
if not rows:
|
|
416
|
+
info("no profiles found — run `eray autoscale generate` first.")
|
|
417
|
+
return
|
|
418
|
+
print_table(rows)
|