cortexgrid 0.3.6__tar.gz → 0.3.7__tar.gz

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.6
+Version: 0.3.7
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/cortexgrid/__init__.py

@@ -36,12 +36,19 @@ from cortexgrid.experiment import (
     Experiment,
     delete_experiment,
     delete_run,
+    experiment_has_active_runs,
+    experiments_pending_deletion,
+    finish_experiment_deletion,
+    finish_run_deletion,
     list_experiments,
+    runs_pending_deletion,
 )
 from cortexgrid.infra import get_ray_job_server_uri
 from cortexgrid.jobs import (
     schedule_remote_job,
     list_experiment_run_jobs,
+    request_job_deletion,
+    request_run_jobs_deletion,
     stop_experiment_run_jobs,
     wait_for_job_result,
     JobFailed,
@@ -67,14 +74,17 @@ from cortexgrid.mlflow_util import (
     list_run_metrics,
     get_metric_history,
     list_run_params,
+    delete_run_artifacts,
     list_run_artifacts,
 )
 from cortexgrid.ray_util import (
     get_ray_status,
     get_ray_logs,
     get_ray_job_url,
+    delete_ray_job,
     stop_ray_job,
     submit_ray_job,
+    list_ray_job_statuses,
     list_ray_jobs_with_submission_id,
     get_ray_job_status,
     get_ray_job_id_for_cortexgrid_job,
@@ -231,6 +241,11 @@ __all__ = [
     "Experiment",
     "delete_experiment",
     "delete_run",
+    "experiment_has_active_runs",
+    "experiments_pending_deletion",
+    "finish_experiment_deletion",
+    "finish_run_deletion",
+    "runs_pending_deletion",
     # Ray / jobs
     "remote",
     "get_job_result",
@@ -238,6 +253,8 @@ __all__ = [
     "get_ray_job_status",
     "get_ray_job_id_for_cortexgrid_job",
     "list_experiment_run_jobs",
+    "request_job_deletion",
+    "request_run_jobs_deletion",
     "stop_experiment_run_jobs",
     "JobStatus",
     "JobFailed",
@@ -250,9 +267,11 @@ __all__ = [
     "get_ray_status",
     "get_ray_logs",
     "get_ray_job_url",
+    "delete_ray_job",
     "stop_ray_job",
     "submit_ray_job",
     "get_ray_job_server_uri",
+    "list_ray_job_statuses",
     "list_ray_jobs_with_submission_id",
     "ray_submission_id",
     "get_ray_job_attempt",
@@ -265,6 +284,7 @@ __all__ = [
     "list_run_metrics",
     "get_metric_history",
     "list_run_params",
+    "delete_run_artifacts",
     "list_run_artifacts",
     "list_experiments",
     # Checkpointing

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/cortexgrid/experiment.py

@@ -4,19 +4,23 @@ import logging
 from dataclasses import dataclass
 from pathlib import Path
 
-from cortexgrid import s3_util
 from cortexgrid.infra import get_mlflow_tracking_uri
-from cortexgrid.jobs import stop_experiment_run_jobs
-from cortexgrid.ray_util import list_ray_jobs_with_submission_id, stop_ray_job
+from cortexgrid.jobs import request_run_jobs_deletion
 from cortexgrid.model_storage import delete_models_for_run
 from haikunator import Haikunator  # type: ignore
 from mlflow.entities import Experiment as MlflowExperiment
+from mlflow.entities import Run as MlflowRun
 from mlflow.tracking import MlflowClient
 
 
 log = logging.getLogger(__name__)
 _SINGLETON_EXPERIMENT: Experiment | None = None
 
+# Written on a run or an experiment that has been asked to go away. It is
+# what keeps such a record out of every by-name lookup while the control
+# plane, the one thing that reads it on purpose, tears the record down.
+DELETE_REQUESTED_TAG = "cortexgrid.delete_requested"
+
 
 @dataclass
 class Experiment:
@@ -123,9 +127,19 @@ def _try_create_experiment_and_run(
         experiment = name_gen.haikunate(token_length=2, token_chars="0123456789")
 
     client = MlflowClient(tracking_uri=mlflow_tracking_uri)
+    # An experiment that asked to be deleted is invisible to the lookup
+    # below, so a new one would quietly take its name while the control
+    # plane is still tearing the old one down — and any run created here
+    # would be torn down with it. Refuse instead.
+    pending = get_experiment_by_name(experiment, include_deleting=True)
+    if pending is not None and _pending_deletion(pending.tags):
+        raise RuntimeError(
+            f"Experiment {experiment!r} is being deleted; wait for the "
+            "control plane to finish before creating it again"
+        )
     # get_experiment_by_name also returns deleted experiments. A deleted one
     # cannot take new runs, so a fresh experiment is created under its name.
-    experiment_obj = client.get_experiment_by_name(name=experiment)
+    experiment_obj = get_experiment_by_name(experiment)
     if experiment_obj is not None and experiment_obj.lifecycle_stage != "active":
         _release_deleted_experiment_name(client, experiment_obj)
         experiment_obj = None
@@ -168,53 +182,48 @@ def _release_deleted_experiment_name(
 
 
 def delete_run(run_id: str) -> None:
-    """Soft-delete a run in MLflow, cancel its Ray attempts, and wipe its
-    S3 job packages so it cannot be relaunched or re-read.
-
-    stop_experiment_run_jobs runs first so the control plane stops spawning
-    fresh Ray attempts for retry=True jobs before we tear the run down."""
-    log.info("delete_run(%s): start", run_id)
-    stop_experiment_run_jobs(run_id)
-    log.info("delete_run(%s): stop_experiment_run_jobs done", run_id)
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    job_ids = [
-        Path(f.path).name for f in client.list_artifacts(run_id, path="job") if f.is_dir
-    ]
-    log.info("delete_run(%s): %d job artifact(s) to clean", run_id, len(job_ids))
-    all_submissions = list_ray_jobs_with_submission_id()
-    for job_id in job_ids:
-        prefix = f"{run_id}-{job_id}-"
-        for sid in all_submissions:
-            if sid.startswith(prefix):
-                stop_ray_job(sid)
-        s3_util.delete_prefix(f"job/{job_id}/")
+    """Ask for a run, and everything under it, to be deleted.
+
+    Intent only: every job in the run is latched and the run itself is
+    tagged. The control plane is the only thing that acts on either — it
+    stops the Ray attempts, wipes the job packages and artifacts, and
+    removes the run once nothing is left under it.
+
+    The tag takes the run out of every by-name lookup here, so a caller
+    that deletes a run and immediately looks one up never gets the one on
+    its way out. Models are not jobs and nothing else touches them, so
+    they are deleted outright.
+    """
+    log.info("delete_run(%s): requesting deletion", run_id)
+    request_run_jobs_deletion(run_id)
     delete_models_for_run(run_id)
-    client.delete_run(run_id)
-    log.info("delete_run(%s): done", run_id)
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    client.set_tag(run_id, DELETE_REQUESTED_TAG, "true")
 
 
 def list_run_ids_in_experiment(name: str) -> list[str]:
     """Return the run IDs of every active run in the named experiment."""
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    exp = client.get_experiment_by_name(name)
+    exp = get_experiment_by_name(name)
     if exp is None:
         return []
-    return [
-        r.info.run_id for r in client.search_runs(experiment_ids=[exp.experiment_id])
-    ]
+    return [r.info.run_id for r in search_runs([exp.experiment_id])]
 
 
 def delete_experiment(name: str) -> None:
-    """Soft-delete every run in the experiment, then the experiment itself.
-
-    The experiment is renamed (`<name>__deleted__<id>`) before it is deleted:
-    MLflow keeps a deleted experiment's name reserved, and the rename frees it
-    so `Experiment.init(name)` can create a new experiment under it.
-
-    Idempotent: already-deleted experiments are treated as success."""
-    log.info("delete_experiment(%r): start", name)
+    """Ask for an experiment, its runs and their jobs to be deleted.
+
+    The experiment is renamed off `name` (`<name>__deleted__<id>`) as the
+    last thing this does, so by the time the call returns the name is free
+    and `Experiment.init(name)` creates a fresh experiment instead of
+    attaching to the one being torn down. Everything else is intent: the
+    control plane deletes each run once its jobs are gone, and the
+    experiment once its runs are.
+
+    Idempotent: an experiment that is absent, already deleted or already
+    on its way out is treated as success."""
+    log.info("delete_experiment(%r): requesting deletion", name)
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    exp = client.get_experiment_by_name(name)
+    exp = get_experiment_by_name(name)
     if exp is None:
         log.info("delete_experiment(%r): early-exit, experiment not found", name)
         return
@@ -223,41 +232,144 @@ def delete_experiment(name: str) -> None:
             "delete_experiment(%r): early-exit, lifecycle=%s", name, exp.lifecycle_stage
         )
         return
-    runs = list(client.search_runs(experiment_ids=[exp.experiment_id]))
-    log.info("delete_experiment(%r): %d active run(s) to delete", name, len(runs))
+    runs = search_runs([exp.experiment_id])
+    log.info("delete_experiment(%r): %d run(s) to request", name, len(runs))
     for run in runs:
         delete_run(run.info.run_id)
+    client.set_experiment_tag(exp.experiment_id, DELETE_REQUESTED_TAG, "true")
     client.rename_experiment(
         exp.experiment_id, _deleted_experiment_name(name, exp.experiment_id)
     )
-    client.delete_experiment(exp.experiment_id)
-    log.info("delete_experiment(%r): done", name)
+    log.info("delete_experiment(%r): requested", name)
 
 
-def list_experiments() -> list[Experiment]:
-    """Map MLflow experiment names to their run IDs."""
+def _pending_deletion(tags: dict[str, str] | None) -> bool:
+    return (tags or {}).get(DELETE_REQUESTED_TAG) == "true"
+
+
+def search_experiments(include_deleting: bool = False) -> list[MlflowExperiment]:
+    """Every experiment, minus the ones on their way out.
+
+    This is the gate: reading experiments straight off MlflowClient
+    bypasses the deletion rule and brings records back from the dead, so
+    every reader — this module, the UI streams, the control plane — comes
+    through here. Only the control plane passes `include_deleting=True`,
+    because removing them is its job.
+    """
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    return [
+        e
+        for e in client.search_experiments()
+        if include_deleting or not _pending_deletion(e.tags)
+    ]
+
+
+def get_experiment_by_name(
+    name: str, include_deleting: bool = False
+) -> MlflowExperiment | None:
+    """The experiment under `name`, unless it is on its way out.
+
+    The gate for by-name resolution. Like MlflowClient's own call it still
+    returns an experiment MLflow has deleted — `Experiment.init` has to see
+    one to release its name — but a record that asked to be deleted is,
+    to every caller but the control plane, already gone.
+    """
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    exp = client.get_experiment_by_name(name)
+    if exp is None:
+        return None
+    if not include_deleting and _pending_deletion(exp.tags):
+        return None
+    return exp
+
+
+def search_runs(
+    experiment_ids: list[str],
+    include_deleting: bool = False,
+    filter_string: str = "",
+) -> list[MlflowRun]:
+    """Every run of those experiments, minus the ones on their way out.
+
+    The same gate, one level down, shaped like the client call it stands
+    in front of so callers keep MLflow's server-side filtering. See
+    `search_experiments`.
+    """
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    return [
+        r
+        for r in client.search_runs(
+            experiment_ids=experiment_ids, filter_string=filter_string
+        )
+        if include_deleting or not _pending_deletion(r.data.tags)
+    ]
+
+
+def list_experiments(include_deleting: bool = False) -> list[Experiment]:
+    """Map MLflow experiment names to their run IDs.
+
+    Records that asked to be deleted are left out: they are on their way
+    to disappearing and must not be reachable any more. The control plane
+    passes `include_deleting=True`, because making them disappear is its
+    job and it cannot do it without seeing them."""
     result: list[Experiment] = []
-    for exp in client.search_experiments():
-        runs = client.search_runs(experiment_ids=[exp.experiment_id])
-        for run in runs:
+    for exp in search_experiments(include_deleting):
+        for run in search_runs([exp.experiment_id], include_deleting):
             result.append(Experiment(exp.name, run_id=run.info.run_id))
     return result
 
 
+def runs_pending_deletion() -> list[str]:
+    """Run ids tagged for deletion. The control plane is the only caller."""
+    return [
+        run.info.run_id
+        for exp in search_experiments(include_deleting=True)
+        for run in search_runs([exp.experiment_id], include_deleting=True)
+        if _pending_deletion(run.data.tags)
+    ]
+
+
+def experiments_pending_deletion() -> list[str]:
+    """Experiment ids tagged for deletion. The control plane is the only caller."""
+    return [
+        exp.experiment_id
+        for exp in search_experiments(include_deleting=True)
+        if _pending_deletion(exp.tags)
+    ]
+
+
+def experiment_has_active_runs(experiment_id: str) -> bool:
+    """Whether anything is still under the experiment. Control plane only.
+
+    Runs on their way out count: the experiment cannot go until they
+    have actually gone."""
+    return bool(search_runs([experiment_id], include_deleting=True))
+
+
+def finish_run_deletion(run_id: str) -> None:
+    """Remove a run whose jobs are gone. The control plane is the only caller."""
+    log.info("finish_run_deletion(%s)", run_id)
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    client.delete_run(run_id)
+
+
+def finish_experiment_deletion(experiment_id: str) -> None:
+    """Remove an experiment whose runs are gone. Control plane only."""
+    log.info("finish_experiment_deletion(%s)", experiment_id)
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    client.delete_experiment(experiment_id)
+
+
 def get_experiment_by_run_name(run_name: str) -> Experiment:
     """Resolve a run by its haikunator name back to its (experiment_name, run_id) pair."""
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    experiment_ids = [e.experiment_id for e in client.search_experiments()]
+    experiment_ids = [e.experiment_id for e in search_experiments()]
     if not experiment_ids:
         raise ValueError(f"No run named {run_name!r}")
-    runs = client.search_runs(
-        experiment_ids=experiment_ids,
-        filter_string=f"attributes.run_name = '{run_name}'",
-        max_results=1,
+    runs = search_runs(
+        experiment_ids, filter_string=f"attributes.run_name = '{run_name}'"
     )
     if not runs:
         raise ValueError(f"No run named {run_name!r}")
     run = runs[0]
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
     exp = client.get_experiment(run.info.experiment_id)
     return Experiment(experiment_name=exp.name, run_id=run.info.run_id)

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/cortexgrid/jobs.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import cloudpickle  # type: ignore
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict, dataclass, field, fields
 import inspect
 import io
 import json
@@ -67,6 +67,16 @@ class LifecycleEvent:
     error: str | None = None
 
 
+def _known_fields_only(cls: type, data: dict[str, Any]) -> dict[str, Any]:
+    """`data` minus the keys `cls` has no field for.
+
+    Records outlive the code that wrote them and the code that reads them:
+    a control plane one release ahead writes fields an older client has
+    never seen, and that client has to keep reading its own jobs."""
+    known = {f.name for f in fields(cls)}
+    return {k: v for k, v in data.items() if k in known}
+
+
 @dataclass
 class JobLifecycle:
     """Static identity and latches for a job.
@@ -81,6 +91,7 @@ class JobLifecycle:
     run_id: str
     job_id: str
     stop_requested: bool = False  # latch: False -> True, never cleared
+    delete_requested: bool = False  # latch: False -> True, never cleared
     retry: bool = False  # static flag set at job creation
     num_gpus: int = 0
     num_cpus: int = 1
@@ -90,13 +101,30 @@ class JobLifecycle:
     history: list[LifecycleEvent] = field(default_factory=list)
 
     def to_json(self) -> str:
-        return json.dumps(asdict(self))
+        """The record as it goes to MLflow.
+
+        A latch that is still False is left out. Every reader of this file
+        pins the fields it knows — a released client parses it with an
+        exact constructor call — so a latch a reader has never heard of
+        must not appear until something actually set it.
+        """
+        data = asdict(self)
+        if not self.delete_requested:
+            data.pop("delete_requested")
+        return json.dumps(data)
 
     @classmethod
     def from_json(cls, text: str) -> "JobLifecycle":
-        data = json.loads(text)
-        data.pop("error", None)
-        data["history"] = [LifecycleEvent(**e) for e in data.get("history", [])]
+        """Parse a record, including one a newer writer produced.
+
+        Fields this version does not know are dropped rather than raising,
+        so a client keeps working while the cluster runs ahead of it.
+        """
+        data = _known_fields_only(cls, json.loads(text))
+        data["history"] = [
+            LifecycleEvent(**_known_fields_only(LifecycleEvent, e))
+            for e in data.get("history", [])
+        ]
         return cls(**data)
 
     def get_ray_job_id(
@@ -147,17 +175,23 @@ class JobLifecycle:
 
     @classmethod
     def load_from_mlflow(cls, run_id: str, job_id: str) -> "JobLifecycle":
+        """Read a job's record.
+
+        One round trip: the fetch is the existence check. Listing the
+        directory first doubled the cost of every read, and a sweep over a
+        cluster's jobs pays it per job against a tracking server that
+        answers these one at a time.
+        """
         client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
         lifecycle_rel = f"job/{job_id}/lifecycle.json"
-        if not any(
-            a.path == lifecycle_rel
-            for a in client.list_artifacts(run_id, f"job/{job_id}")
-        ):
+        try:
+            local_path = client.download_artifacts(run_id, lifecycle_rel)
+            text = Path(local_path).read_text()
+        except Exception as err:
             raise FileNotFoundError(
                 f"artifact {lifecycle_rel} not found in run {run_id}"
-            )
-        local_path = client.download_artifacts(run_id, lifecycle_rel)
-        return cls.from_json(Path(local_path).read_text())
+            ) from err
+        return cls.from_json(text)
 
 
 class Payload(BaseModel):
@@ -496,22 +530,37 @@ def schedule_remote_job(
     return JobFuture(experiment_name=experiment_name, run_id=run_id, job_id=job_id)
 
 
+def list_experiment_run_job_ids(run_id: str) -> list[str]:
+    """The ids of every job of this run — one listing, no records read.
+
+    Reading a job's record costs a download each; a caller that wants many
+    of them can take this list and fetch them as it pleases."""
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    return [
+        Path(entry.path).name
+        for entry in client.list_artifacts(run_id, path="job")
+        if entry.is_dir
+    ]
+
+
+def load_job(run_id: str, job_id: str) -> JobLifecycle | None:
+    """The job's record, or None when it has not landed yet.
+
+    A job whose lifecycle is still being written is a normal state, not an
+    error: the caller lists it again on the next poll."""
+    try:
+        return JobLifecycle.load_from_mlflow(run_id, job_id)
+    except Exception:
+        logging.getLogger(__name__).warning(
+            "Skipping job %s: missing lifecycle", job_id
+        )
+        return None
+
+
 def list_experiment_run_jobs(run_id: str) -> list[JobLifecycle]:
     """Return all jobs and their lifecycle states for this experiment+run."""
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    entries = client.list_artifacts(run_id, path="job")
-    result: list[JobLifecycle] = []
-    for entry in entries:
-        if not entry.is_dir:
-            continue
-        job_id = Path(entry.path).name
-        try:
-            result.append(JobLifecycle.load_from_mlflow(run_id, job_id))
-        except Exception:
-            logging.getLogger(__name__).warning(
-                "Skipping job %s: missing lifecycle", job_id
-            )
-    return result
+    jobs = (load_job(run_id, job_id) for job_id in list_experiment_run_job_ids(run_id))
+    return [job for job in jobs if job is not None]
 
 
 def stop_experiment_run_jobs(run_id: str) -> None:
@@ -530,3 +579,30 @@ def stop_experiment_run_jobs(run_id: str) -> None:
             continue
         job.stop_requested = True
         job.save_to_mlflow()
+
+
+def request_job_deletion(run_id: str, job_id: str) -> None:
+    """Ask for one job to be deleted by flipping the delete_requested latch.
+
+    This function never touches Ray, S3 or the job's artifacts: it
+    records the intent on the job's own record and returns. The control
+    plane observes the latch on its next poll and is the only thing that
+    tears the job down, so a job created and deleted between two polls
+    is simply never submitted.
+
+    Idempotent: an already-requested job is left alone.
+    """
+    lifecycle = JobLifecycle.load_from_mlflow(run_id, job_id)
+    if lifecycle.delete_requested:
+        return
+    lifecycle.delete_requested = True
+    lifecycle.save_to_mlflow()
+
+
+def request_run_jobs_deletion(run_id: str) -> None:
+    """Ask for every job in the run to be deleted. See `request_job_deletion`."""
+    for job in list_experiment_run_jobs(run_id):
+        if job.delete_requested:
+            continue
+        job.delete_requested = True
+        job.save_to_mlflow()

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/cortexgrid/mlflow_util.py

@@ -14,6 +14,7 @@ import requests
 from cortexgrid.experiment import Experiment
 from cortexgrid.infra import get_mlflow_tracking_uri
 from mlflow.entities import Metric
+from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
 from mlflow.tracking import MlflowClient
 
 
@@ -108,3 +109,16 @@ def list_run_artifacts(run_id: str, path: str = "") -> list[str]:
     """Return artifact paths for a run."""
     client = get_mlflow_client()
     return [a.path for a in client.list_artifacts(run_id, path=path)]
+
+
+def delete_run_artifacts(run_id: str, path: str) -> None:
+    """Delete everything a run stores under `path`, directory and all.
+
+    MlflowClient has no delete: it goes through the run's artifact
+    repository, which the tracking server proxies to object storage."""
+    client = get_mlflow_client()
+    repo = get_artifact_repository(
+        client.get_run(run_id).info.artifact_uri,
+        tracking_uri=get_mlflow_tracking_uri(),
+    )
+    repo.delete_artifacts(path)

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/cortexgrid/ray_util.py

@@ -58,23 +58,43 @@ def get_ray_status(ray_job_id: str | None) -> str | None:
     return client.get_job_status(ray_job_id).value
 
 
+def to_job_status(ray_status: str | None) -> JobStatus:
+    """Map one of Ray's words onto ours. `None` means never submitted."""
+    if ray_status is None:
+        return JobStatus.PENDING
+    if ray_status == "SUCCEEDED":
+        return JobStatus.FINISHED
+    if ray_status == "FAILED":
+        return JobStatus.FAILED
+    if ray_status == "STOPPED":
+        return JobStatus.STOPPED
+    return JobStatus.RUNNING
+
+
 def get_ray_job_status(ray_job_id: str | None) -> JobStatus:
     """Derive a job's observable status from a live Ray query.
 
     Returns ``PENDING`` both when ``ray_job_id is None`` (never submitted)
     and when Ray itself reports ``PENDING`` (queued). Callers that need
     to distinguish those two must check ``ray_job_id is None`` first.
+
+    One query per job. A caller with many jobs wants
+    ``list_ray_job_statuses`` instead, which answers for all of them in
+    a single call.
     """
-    ray_status = get_ray_status(ray_job_id)
     if ray_job_id is None:
         return JobStatus.PENDING
-    if ray_status == "SUCCEEDED":
-        return JobStatus.FINISHED
-    if ray_status == "FAILED":
-        return JobStatus.FAILED
-    if ray_status == "STOPPED":
-        return JobStatus.STOPPED
-    return JobStatus.RUNNING
+    return to_job_status(get_ray_status(ray_job_id))
+
+
+def list_ray_job_statuses() -> dict[str, JobStatus]:
+    """Every ray job's status, by submission id, in one call."""
+    client = get_ray_job_submission_client()
+    return {
+        job.submission_id: to_job_status(job.status.value)
+        for job in client.list_jobs()
+        if job.submission_id is not None
+    }
 
 
 def get_ray_logs(ray_job_id: str | None) -> str | None:
@@ -101,6 +121,15 @@ def stop_ray_job(ray_job_id: str) -> None:
     client.stop_job(ray_job_id)
 
 
+def delete_ray_job(ray_job_id: str) -> None:
+    """Drop a ray job and its data from Ray's job store.
+
+    Ray rejects the call for a job that has not settled, so callers stop
+    the job and let it reach a terminal state first."""
+    client = get_ray_job_submission_client()
+    client.delete_job(ray_job_id)
+
+
 def list_ray_jobs_with_submission_id() -> list[str]:
     """List all ray jobs, the ones that received submission id."""
     client = get_ray_job_submission_client()

{cortexgrid-0.3.6 → cortexgrid-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.3.6"
+version = "0.3.7"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"