eray 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. eray/__init__.py +192 -0
  2. eray/cli/__init__.py +36 -0
  3. eray/cli/autoscale.py +418 -0
  4. eray/cli/dashboard.py +437 -0
  5. eray/cli/fleet.py +422 -0
  6. eray/cli/hostjob.py +506 -0
  7. eray/cli/jobs.py +1350 -0
  8. eray/cli/main.py +802 -0
  9. eray/cli/qr.py +249 -0
  10. eray/cli/tpu.py +763 -0
  11. eray/cli/utils.py +707 -0
  12. eray/core/__init__.py +63 -0
  13. eray/core/cluster.py +106 -0
  14. eray/core/exceptions.py +207 -0
  15. eray/core/monitoring.py +383 -0
  16. eray/core/sentinels.py +97 -0
  17. eray/core/status.py +105 -0
  18. eray/docker/__init__.py +48 -0
  19. eray/docker/config.py +131 -0
  20. eray/docker/runner.py +351 -0
  21. eray/execution/__init__.py +43 -0
  22. eray/execution/decorators.py +281 -0
  23. eray/execution/executor.py +741 -0
  24. eray/execution/remote.py +655 -0
  25. eray/gcp.py +177 -0
  26. eray/pool/__init__.py +40 -0
  27. eray/pool/base.py +284 -0
  28. eray/pool/device_host.py +717 -0
  29. eray/pool/slice.py +1195 -0
  30. eray/provision/__init__.py +66 -0
  31. eray/provision/fleet.py +332 -0
  32. eray/provision/launcher.py +497 -0
  33. eray/provision/qr.py +442 -0
  34. eray/provision/registry.py +510 -0
  35. eray/provision/templates/cluster-template.yaml +63 -0
  36. eray/provision/tunnel.py +424 -0
  37. eray/provision/watcher.py +777 -0
  38. eray/py.typed +0 -0
  39. eray/resources/__init__.py +50 -0
  40. eray/resources/configs.py +676 -0
  41. eray/resources/hardware.py +82 -0
  42. eray/resources/ray_resources.py +387 -0
  43. eray/resources/topology.py +344 -0
  44. eray/swarm.py +845 -0
  45. eray-0.1.0.dist-info/METADATA +561 -0
  46. eray-0.1.0.dist-info/RECORD +48 -0
  47. eray-0.1.0.dist-info/WHEEL +4 -0
  48. eray-0.1.0.dist-info/entry_points.txt +3 -0
eray/__init__.py ADDED
@@ -0,0 +1,192 @@
1
+ # Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ """eray — Ray-based distributed execution and scaling framework.
17
+
18
+ A standalone extraction of the Ray executor stack from eFormer/EasyDeL.
19
+ Provides distributed execution of machine learning workloads using Ray,
20
+ with support for CPUs, GPUs, TPUs, Docker containerization, resource
21
+ pooling, multi-slice coordination, and fault-tolerant execution.
22
+
23
+ Subpackages:
24
+ core: Job status types, exceptions, sentinels, monitoring, cluster info
25
+ resources: Hardware constants, Ray resource specs, accelerator configs
26
+ pool: Actor pool management, device hosts, multi-slice coordination
27
+ execution: RayExecutor engine, decorators, device-remote machinery
28
+ docker: Docker container config and execution
29
+
30
+ Example:
31
+ Basic distributed execution:
32
+
33
+ >>> from eray import execute, GpuAcceleratorConfig
34
+ >>>
35
+ >>> @execute(GpuAcceleratorConfig(count=1, gpu_model="A100"))
36
+ ... def train_step(batch):
37
+ ... ...
38
+ ... return loss
39
+
40
+ Multi-slice TPU execution with Docker:
41
+
42
+ >>> from eray import run_docker_multislice, DockerConfig, TpuAcceleratorConfig
43
+ >>>
44
+ >>> docker_config = DockerConfig(
45
+ ... image="my-ml-image:latest",
46
+ ... command="python train.py",
47
+ ... )
48
+ >>> run_docker_multislice(
49
+ ... docker_config,
50
+ ... TpuAcceleratorConfig(tpu_version="v4-32", pod_count=4),
51
+ ... )
52
+ """
53
+
54
+ from ray import init
55
+ from ray.runtime_env import RuntimeEnv
56
+
57
+ from .core import (
58
+ DONE,
59
+ DoneSentinel,
60
+ ExceptionInfo,
61
+ HostInfo,
62
+ JobError,
63
+ JobFailed,
64
+ JobInfo,
65
+ JobPreempted,
66
+ JobStatus,
67
+ JobSucceeded,
68
+ MultisliceInfo,
69
+ RefBox,
70
+ SliceInfo,
71
+ SnitchRecipient,
72
+ StopwatchActor,
73
+ current_actor_handle,
74
+ handle_ray_error,
75
+ log_failures_to,
76
+ print_remote_raise,
77
+ start_raylet_log_guard,
78
+ sweep_raylet_logs,
79
+ )
80
+ from .docker import (
81
+ DockerConfig,
82
+ build_and_push_docker_image,
83
+ make_docker_run_command,
84
+ run_docker_async,
85
+ run_docker_multislice,
86
+ run_docker_on_pod,
87
+ )
88
+ from .execution import (
89
+ ENV_CALL_INDEX,
90
+ ENV_CALL_SLICE,
91
+ MEGASCALE_DEFAULT_PORT,
92
+ RayExecutor,
93
+ autoscale_execute,
94
+ autoscale_execute_resumable,
95
+ device_remote,
96
+ execute,
97
+ execute_resumable,
98
+ resolve_maybe_refs,
99
+ )
100
+ from .pool import (
101
+ ActorPoolMember,
102
+ DeviceHostActor,
103
+ InsufficientSlicesError,
104
+ ResourcePoolManager,
105
+ SliceActor,
106
+ SlicePoolManager,
107
+ )
108
+ from .provision import ensure_tpu, watch_and_reconnect
109
+ from .resources import (
110
+ AcceleratorConfigType,
111
+ ComputeResourceConfig,
112
+ CpuAcceleratorConfig,
113
+ GpuAcceleratorConfig,
114
+ HardwareType,
115
+ HostPartitionPlan,
116
+ HostSplitPlan,
117
+ RayResources,
118
+ TpuAcceleratorConfig,
119
+ available_cpu_cores,
120
+ plan_host_partition,
121
+ plan_host_split,
122
+ )
123
+ from .swarm import GpuSwarmConfig, SwarmConfig, SwarmRun, shutdown_swarm, swarm_execute, swarmed
124
+
125
+ __version__ = "0.1.0"
126
+ __all__ = (
127
+ "DONE",
128
+ "ENV_CALL_INDEX",
129
+ "ENV_CALL_SLICE",
130
+ "MEGASCALE_DEFAULT_PORT",
131
+ "AcceleratorConfigType",
132
+ "ActorPoolMember",
133
+ "ComputeResourceConfig",
134
+ "CpuAcceleratorConfig",
135
+ "DeviceHostActor",
136
+ "DockerConfig",
137
+ "DoneSentinel",
138
+ "ExceptionInfo",
139
+ "GpuAcceleratorConfig",
140
+ "GpuSwarmConfig",
141
+ "HardwareType",
142
+ "HostInfo",
143
+ "HostPartitionPlan",
144
+ "HostSplitPlan",
145
+ "InsufficientSlicesError",
146
+ "JobError",
147
+ "JobFailed",
148
+ "JobInfo",
149
+ "JobPreempted",
150
+ "JobStatus",
151
+ "JobSucceeded",
152
+ "MultisliceInfo",
153
+ "RayExecutor",
154
+ "RayResources",
155
+ "RefBox",
156
+ "ResourcePoolManager",
157
+ "RuntimeEnv",
158
+ "SliceActor",
159
+ "SliceInfo",
160
+ "SlicePoolManager",
161
+ "SnitchRecipient",
162
+ "StopwatchActor",
163
+ "SwarmConfig",
164
+ "SwarmRun",
165
+ "TpuAcceleratorConfig",
166
+ "autoscale_execute",
167
+ "autoscale_execute_resumable",
168
+ "available_cpu_cores",
169
+ "build_and_push_docker_image",
170
+ "current_actor_handle",
171
+ "device_remote",
172
+ "ensure_tpu",
173
+ "execute",
174
+ "execute_resumable",
175
+ "handle_ray_error",
176
+ "init",
177
+ "log_failures_to",
178
+ "make_docker_run_command",
179
+ "plan_host_partition",
180
+ "plan_host_split",
181
+ "print_remote_raise",
182
+ "resolve_maybe_refs",
183
+ "run_docker_async",
184
+ "run_docker_multislice",
185
+ "run_docker_on_pod",
186
+ "shutdown_swarm",
187
+ "start_raylet_log_guard",
188
+ "swarm_execute",
189
+ "swarmed",
190
+ "sweep_raylet_logs",
191
+ "watch_and_reconnect",
192
+ )
eray/cli/__init__.py ADDED
@@ -0,0 +1,36 @@
1
+ # Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Command-line interface for eray.
16
+
17
+ Provides commands for connecting TPU hosts into a Ray cluster and
18
+ managing cluster lifecycle.
19
+
20
+ Example:
21
+ # Connect a TPU slice into a Ray cluster
22
+ $ eray tpu connect --tpu-name my-tpu --project my-project --zone us-central2-b
23
+
24
+ # Check cluster status
25
+ $ eray tpu status --address 10.0.0.1:6379
26
+
27
+ # Run a health check
28
+ $ eray tpu health --address 10.0.0.1:6379
29
+
30
+ # Disconnect (stop Ray on all hosts)
31
+ $ eray tpu disconnect --tpu-name my-tpu --project my-project --zone us-central2-b
32
+ """
33
+
34
+ from .main import cli
35
+
36
+ __all__ = ("cli",)
eray/cli/autoscale.py ADDED
@@ -0,0 +1,418 @@
1
+ # Copyright 2026 The EasyDeL/eray Author @erfanzar (Erfan Zare Chavoshi).
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """`eray autoscale` — Ray cluster-launcher configs for elastic TPU clusters.
16
+
17
+ Generates per-zone launcher YAMLs (all-spot by default; every TPU node type
18
+ carries the eray-canonical resource labels so the pool scheduler works) and
19
+ wraps `ray up/down/get-head-ip`. For reliable large slices, prefer the
20
+ QR-based `eray fleet` path; this one is for elastic many-small-slices use.
21
+
22
+ `up`/`down`/`status` never require a YAML path: with no CONFIG they discover
23
+ generated profiles under `--dir` and either auto-select the only one or offer
24
+ a numbered picker. Every `up`/`down` also upserts a ``kind="launcher"``
25
+ record into the same fleet registry `eray fleet` uses, which is what lets
26
+ `eray autoscale status` (no CONFIG) and `eray dashboard` show every launcher
27
+ cluster's live state without the operator tracking YAML paths by hand.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json as json_lib
33
+ import time
34
+
35
+ import click
36
+
37
+ from ..provision.registry import ClusterRecord, ClusterRegistry
38
+ from .utils import detect_project, error, info, print_table, success, warning
39
+
40
+
41
+ def _pick_profile(profiles: list, *, verb: str):
42
+ """Resolve which profile to act on when no CONFIG was given.
43
+
44
+ Args:
45
+ profiles: Discovered profiles (see `provision.launcher.list_profiles`).
46
+ verb: The action being taken, for the prompt copy (e.g. "bring up").
47
+
48
+ Returns:
49
+ The chosen `AutoscaleProfile`.
50
+
51
+ Raises:
52
+ click.ClickException: No profiles exist, or no selection was made
53
+ (e.g. stdin has nothing left to read — a non-interactive call).
54
+ """
55
+ if not profiles:
56
+ raise click.ClickException("no generated profiles found — run `eray autoscale generate` first.")
57
+ if len(profiles) == 1:
58
+ profile = profiles[0]
59
+ info(f"one profile found: {profile.name} — using it.")
60
+ return profile
61
+ click.echo(f"Multiple profiles found — choose one to {verb}:")
62
+ for idx, profile in enumerate(profiles, start=1):
63
+ click.echo(f" {idx}) {profile.name} ({profile.zone or '?'}, {profile.project or '?'})")
64
+ try:
65
+ choice = click.prompt("select", type=click.IntRange(1, len(profiles)))
66
+ except click.Abort:
67
+ raise click.ClickException(
68
+ "no profile selected — pass a profile name explicitly (non-interactive session?)."
69
+ ) from None
70
+ return profiles[choice - 1]
71
+
72
+
73
+ def _upsert_launcher_record(profile, *, state: str, stamp: str, head_ip: str | None = None) -> None:
74
+ """Persist a launcher-kind cluster's state after `up`/`down`.
75
+
76
+ Args:
77
+ profile: The `AutoscaleProfile` that was acted on.
78
+ state: New `ClusterRecord.state` (``"HEALTHY"`` or ``"DOWN"``).
79
+ stamp: Which timestamp field to stamp with now — ``"last_up_ts"`` or
80
+ ``"last_down_ts"``.
81
+ head_ip: Best-effort head IP (`up` only; `down` leaves it as-is).
82
+ """
83
+ registry = ClusterRegistry.from_config()
84
+ if registry.get(profile.name) is None:
85
+ registry.upsert(
86
+ ClusterRecord(
87
+ name=profile.name,
88
+ kind="launcher",
89
+ project=profile.project,
90
+ zone=profile.zone,
91
+ config_path=str(profile.path),
92
+ state=state,
93
+ head_ip=head_ip,
94
+ **{stamp: time.time()},
95
+ )
96
+ )
97
+ return
98
+
99
+ def mutate(r: ClusterRecord) -> None:
100
+ r.kind = "launcher"
101
+ r.project = profile.project or r.project
102
+ r.zone = profile.zone or r.zone
103
+ r.config_path = str(profile.path)
104
+ r.state = state
105
+ if head_ip is not None:
106
+ r.head_ip = head_ip
107
+ if stamp == "last_up_ts":
108
+ # A fresh bring-up invalidates any earlier "died at T" history —
109
+ # otherwise a live cluster can flash DOWN/"died ... ago" if the
110
+ # GCE probe runs before the new head instance is discoverable.
111
+ r.last_down_ts = None
112
+ setattr(r, stamp, time.time())
113
+
114
+ registry.mutate_record(profile.name, mutate)
115
+
116
+
117
+ def launcher_row_status(
118
+ cluster_name: str, *, project: str | None, zone: str | None, state: str, last_down_ts: float | None
119
+ ) -> tuple[str, str]:
120
+ """Live (status, since) for one launcher-kind cluster.
121
+
122
+ Shared by `eray autoscale status` (no CONFIG) and `eray dashboard` so
123
+ the two views can never disagree about what "running" means.
124
+
125
+ Args:
126
+ cluster_name: The launcher config's ``cluster_name``.
127
+ project: GCP project id, or None (skips the live probe).
128
+ zone: GCP zone, or None (skips the live probe).
129
+ state: Fallback status when no live head instance is found and
130
+ there's no `last_down_ts` to report against (e.g. registered
131
+ but never brought up).
132
+ last_down_ts: Unix timestamp of the last `eray autoscale down`, or
133
+ None.
134
+
135
+ Returns:
136
+ ``(status, since)`` display strings — status is either a live GCE
137
+ instance status (``RUNNING``, ``STAGING``, ...), ``"DOWN"``, or the
138
+ fallback `state`; since is "up <duration>", "died <duration> ago",
139
+ "?" (status known, age unknown), or "-".
140
+ """
141
+ from ..provision.launcher import gce_instances_for_cluster
142
+ from .utils import format_ago, format_duration, parse_gcp_timestamp
143
+
144
+ instances = gce_instances_for_cluster(cluster_name, project=project, zone=zone) if project and zone else []
145
+ head = next((i for i in instances if (i.get("labels") or {}).get("ray-node-type") == "head"), None)
146
+ if head is not None:
147
+ status = str(head.get("status", "UNKNOWN"))
148
+ created = parse_gcp_timestamp(head.get("creationTimestamp"))
149
+ since = f"up {format_duration(time.time() - created)}" if created else "?"
150
+ elif last_down_ts is not None:
151
+ status = "DOWN"
152
+ since = f"died {format_ago(last_down_ts)}"
153
+ else:
154
+ status = state
155
+ since = "-"
156
+ return status, since
157
+
158
+
159
+ def _launcher_status_rows(profiles_dir) -> list[dict]:
160
+ """Merge on-disk profiles and registered launcher clusters into rows.
161
+
162
+ Each row's live GCE probe is an independent gcloud subprocess; a thread
163
+ pool runs them concurrently so `eray autoscale status` doesn't scale
164
+ linearly with the number of profiles (mirrors `run_on_all_hosts` in
165
+ `cli/utils.py`).
166
+
167
+ Args:
168
+ profiles_dir: Directory profiles are discovered in.
169
+
170
+ Returns:
171
+ One row per known profile (on disk, registered, or both): name,
172
+ zone, status (a live GCE probe when project/zone are known), head,
173
+ since (uptime, or "died N ago", or "-").
174
+ """
175
+ import concurrent.futures
176
+
177
+ from ..provision.launcher import list_profiles
178
+
179
+ registry = ClusterRegistry.from_config()
180
+ records = {name: r for name, r in registry.load().items() if r.kind == "launcher"}
181
+ profiles = {p.name: p for p in list_profiles(profiles_dir)}
182
+ names = sorted(set(profiles) | set(records))
183
+ if not names:
184
+ return []
185
+
186
+ def build_row(name: str) -> dict:
187
+ profile = profiles.get(name)
188
+ record = records.get(name)
189
+ project = (profile.project if profile else None) or (record.project if record else None)
190
+ zone = (profile.zone if profile else None) or (record.zone if record else None)
191
+ cluster_name = profile.cluster_name if profile else name
192
+
193
+ status, since = launcher_row_status(
194
+ cluster_name,
195
+ project=project,
196
+ zone=zone,
197
+ state=record.state if record else "UNREGISTERED",
198
+ last_down_ts=record.last_down_ts if record else None,
199
+ )
200
+ return {
201
+ "name": name,
202
+ "zone": zone or "?",
203
+ "status": status,
204
+ "head": (record.head_ip if record and status == "RUNNING" else None) or "-",
205
+ "since": since,
206
+ }
207
+
208
+ with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(names), 8)) as pool:
209
+ return list(pool.map(build_row, names))
210
+
211
+
212
+ def register(cli: click.Group) -> None:
213
+ """Register the `autoscale` command group on the root CLI.
214
+
215
+ Args:
216
+ cli: The root click group.
217
+ """
218
+
219
+ @cli.group()
220
+ def autoscale() -> None:
221
+ """Ray cluster-launcher configs for elastic (autoscaled) TPU clusters."""
222
+
223
+ @autoscale.command()
224
+ @click.option("--project", "-p", default=None, help="GCP project (auto-detected).")
225
+ @click.option("--zones", "-z", default=None, help="Comma-separated zones (default: every TPU location).")
226
+ @click.option("--families", default="v4,v5e,v5p,v6e", show_default=True, help="TPU families to include.")
227
+ @click.option("--image", default=None, help="Docker image for head/workers (default: easydel latest-tpu).")
228
+ @click.option(
229
+ "--spot/--on-demand", "spot", default=True, show_default=True, help="Capacity tier for TPU node types."
230
+ )
231
+ @click.option(
232
+ "--output-dir",
233
+ "-o",
234
+ "output_dir",
235
+ default=None,
236
+ help="Where to write the YAMLs (default: ~/.eray/autoscale).",
237
+ )
238
+ def generate(project, zones, families, image, spot, output_dir):
239
+ """Generate per-zone cluster-launcher YAMLs.
240
+
241
+ \b
242
+ Examples:
243
+ eray autoscale generate --zones us-central1-a
244
+ eray autoscale generate --families v5p --on-demand
245
+ """
246
+ from ..provision.launcher import DEFAULT_IMAGE, DEFAULT_OUTPUT_DIR, generate_configs
247
+
248
+ project = project or detect_project()
249
+ if not project:
250
+ raise click.ClickException("Could not resolve a GCP project; pass --project.")
251
+ zone_list = [z.strip() for z in zones.split(",") if z.strip()] if zones else None
252
+ family_tuple = tuple(f.strip() for f in families.split(",") if f.strip())
253
+ try:
254
+ written = generate_configs(
255
+ project,
256
+ zones=zone_list,
257
+ families=family_tuple,
258
+ image=image or DEFAULT_IMAGE,
259
+ spot=spot,
260
+ output_dir=output_dir or DEFAULT_OUTPUT_DIR,
261
+ )
262
+ except Exception as exc:
263
+ error(str(exc))
264
+ raise click.ClickException(str(exc)) from exc
265
+ if not written:
266
+ info("no zones offered the requested TPU families; nothing written.")
267
+ return
268
+ for path in written:
269
+ success(f"wrote {path}")
270
+
271
+ _dir_opt = click.option(
272
+ "--dir",
273
+ "profiles_dir",
274
+ default=None,
275
+ help="Where generated profiles are discovered from (default: ~/.eray/autoscale).",
276
+ )
277
+
278
+ @autoscale.command()
279
+ @click.argument("config", required=False)
280
+ @_dir_opt
281
+ @click.option("--yes", "-y", is_flag=True, default=False, help="Skip the ray up confirmation.")
282
+ def up(config, profiles_dir, yes):
283
+ """Bring a launcher cluster up (`ray up`).
284
+
285
+ \b
286
+ With no CONFIG: picks from generated profiles (auto-selects the
287
+ only one, prompts when there's more than one). CONFIG may also be a
288
+ YAML path or a bare profile name (its file stem) to skip the picker.
289
+
290
+ \b
291
+ Examples:
292
+ eray autoscale up # picker
293
+ eray autoscale up easydel-us-east5-a # by profile name
294
+ eray autoscale up ./my-cluster.yaml # by path
295
+ """
296
+ from ..provision.launcher import (
297
+ DEFAULT_OUTPUT_DIR,
298
+ launcher_head_ip,
299
+ list_profiles,
300
+ load_profile,
301
+ ray_up,
302
+ resolve_profile,
303
+ )
304
+
305
+ profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
306
+ if config:
307
+ try:
308
+ path = resolve_profile(config, profiles_dir)
309
+ except FileNotFoundError as exc:
310
+ raise click.ClickException(str(exc)) from exc
311
+ else:
312
+ path = _pick_profile(list_profiles(profiles_dir), verb="bring up").path
313
+
314
+ if not yes:
315
+ click.confirm(f"ray up {path} — create/update this cluster?", abort=True)
316
+ try:
317
+ ray_up(path, yes=True)
318
+ except Exception as exc:
319
+ error(str(exc))
320
+ raise click.ClickException(str(exc)) from exc
321
+ success(f"cluster up: {path}")
322
+
323
+ profile = load_profile(path)
324
+ if profile is None:
325
+ warning("could not re-read cluster_name from CONFIG; skipping fleet registry update.")
326
+ return
327
+ head_ip = None
328
+ try:
329
+ head_ip = launcher_head_ip(path)
330
+ except Exception as exc:
331
+ warning(f"could not resolve head IP ({exc}); registering without it — `eray dashboard ls` will show '-'.")
332
+ _upsert_launcher_record(profile, state="HEALTHY", stamp="last_up_ts", head_ip=head_ip)
333
+
334
+ @autoscale.command()
335
+ @click.argument("config", required=False)
336
+ @_dir_opt
337
+ @click.option("--yes", "-y", is_flag=True, default=False, help="Skip the ray down confirmation.")
338
+ def down(config, profiles_dir, yes):
339
+ """Tear a launcher cluster down (`ray down`).
340
+
341
+ \b
342
+ With no CONFIG: picks from generated/registered profiles the same
343
+ way `up` does. CONFIG may also be a YAML path or a bare profile name.
344
+ """
345
+ from ..provision.launcher import (
346
+ DEFAULT_OUTPUT_DIR,
347
+ list_profiles,
348
+ load_profile,
349
+ ray_down,
350
+ resolve_profile,
351
+ )
352
+
353
+ profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
354
+ if config:
355
+ try:
356
+ path = resolve_profile(config, profiles_dir)
357
+ except FileNotFoundError as exc:
358
+ raise click.ClickException(str(exc)) from exc
359
+ else:
360
+ path = _pick_profile(list_profiles(profiles_dir), verb="tear down").path
361
+
362
+ if not yes:
363
+ click.confirm(f"ray down {path} — terminate this cluster's nodes?", abort=True)
364
+ try:
365
+ ray_down(path, yes=True)
366
+ except Exception as exc:
367
+ error(str(exc))
368
+ raise click.ClickException(str(exc)) from exc
369
+ success(f"cluster down: {path}")
370
+
371
+ profile = load_profile(path)
372
+ if profile is None:
373
+ warning("could not re-read cluster_name from CONFIG; skipping fleet registry update.")
374
+ return
375
+ _upsert_launcher_record(profile, state="DOWN", stamp="last_down_ts")
376
+
377
+ @autoscale.command()
378
+ @click.argument("config", required=False)
379
+ @_dir_opt
380
+ @click.option("--json", "as_json", is_flag=True, default=False, help="Output the no-CONFIG table as JSON.")
381
+ def status(config, profiles_dir, as_json):
382
+ """Show launcher clusters: every known profile, or one cluster's detail.
383
+
384
+ \b
385
+ With no CONFIG: a table of every generated/registered profile —
386
+ zone, live status (a GCE instance-label probe), head, and how long
387
+ it's been up (or when it went down). With CONFIG: one cluster's
388
+ head + resource-usage detail (today's behavior, unchanged).
389
+ """
390
+ from ..provision.launcher import DEFAULT_OUTPUT_DIR
391
+
392
+ profiles_dir = profiles_dir or DEFAULT_OUTPUT_DIR
393
+ if config:
394
+ from ..provision.fleet import head_reachable
395
+ from ..provision.launcher import launcher_head_ip, resolve_profile
396
+
397
+ try:
398
+ path = resolve_profile(config, profiles_dir)
399
+ except FileNotFoundError as exc:
400
+ raise click.ClickException(str(exc)) from exc
401
+ try:
402
+ head_ip = launcher_head_ip(path)
403
+ except Exception as exc:
404
+ error(f"cluster not reachable via ray get-head-ip: {exc}")
405
+ raise click.ClickException(str(exc)) from exc
406
+ info(f"head: {head_ip}")
407
+ info(f"ray head port: {'up' if head_reachable(head_ip) else 'down'}")
408
+ info(f"resources: eray resources -a {head_ip}:6379")
409
+ return
410
+
411
+ rows = _launcher_status_rows(profiles_dir)
412
+ if as_json:
413
+ click.echo(json_lib.dumps(rows, indent=2))
414
+ return
415
+ if not rows:
416
+ info("no profiles found — run `eray autoscale generate` first.")
417
+ return
418
+ print_table(rows)