cortexgrid 0.3.7__tar.gz → 0.3.8__tar.gz

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.7
+Version: 0.3.8
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/cortexgrid/__init__.py

@@ -36,19 +36,12 @@ from cortexgrid.experiment import (
     Experiment,
     delete_experiment,
     delete_run,
-    experiment_has_active_runs,
-    experiments_pending_deletion,
-    finish_experiment_deletion,
-    finish_run_deletion,
     list_experiments,
-    runs_pending_deletion,
 )
 from cortexgrid.infra import get_ray_job_server_uri
 from cortexgrid.jobs import (
     schedule_remote_job,
     list_experiment_run_jobs,
-    request_job_deletion,
-    request_run_jobs_deletion,
     stop_experiment_run_jobs,
     wait_for_job_result,
     JobFailed,
@@ -74,17 +67,14 @@ from cortexgrid.mlflow_util import (
     list_run_metrics,
     get_metric_history,
     list_run_params,
-    delete_run_artifacts,
     list_run_artifacts,
 )
 from cortexgrid.ray_util import (
     get_ray_status,
     get_ray_logs,
     get_ray_job_url,
-    delete_ray_job,
     stop_ray_job,
     submit_ray_job,
-    list_ray_job_statuses,
     list_ray_jobs_with_submission_id,
     get_ray_job_status,
     get_ray_job_id_for_cortexgrid_job,
@@ -241,11 +231,6 @@ __all__ = [
     "Experiment",
     "delete_experiment",
     "delete_run",
-    "experiment_has_active_runs",
-    "experiments_pending_deletion",
-    "finish_experiment_deletion",
-    "finish_run_deletion",
-    "runs_pending_deletion",
     # Ray / jobs
     "remote",
     "get_job_result",
@@ -253,8 +238,6 @@ __all__ = [
     "get_ray_job_status",
     "get_ray_job_id_for_cortexgrid_job",
     "list_experiment_run_jobs",
-    "request_job_deletion",
-    "request_run_jobs_deletion",
     "stop_experiment_run_jobs",
     "JobStatus",
     "JobFailed",
@@ -267,11 +250,9 @@ __all__ = [
     "get_ray_status",
     "get_ray_logs",
     "get_ray_job_url",
-    "delete_ray_job",
     "stop_ray_job",
     "submit_ray_job",
     "get_ray_job_server_uri",
-    "list_ray_job_statuses",
     "list_ray_jobs_with_submission_id",
     "ray_submission_id",
     "get_ray_job_attempt",
@@ -284,7 +265,6 @@ __all__ = [
     "list_run_metrics",
     "get_metric_history",
     "list_run_params",
-    "delete_run_artifacts",
     "list_run_artifacts",
     "list_experiments",
     # Checkpointing

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/cortexgrid/experiment.py

@@ -4,23 +4,19 @@ import logging
 from dataclasses import dataclass
 from pathlib import Path
 
+from cortexgrid import s3_util
 from cortexgrid.infra import get_mlflow_tracking_uri
-from cortexgrid.jobs import request_run_jobs_deletion
+from cortexgrid.jobs import stop_experiment_run_jobs
+from cortexgrid.ray_util import list_ray_jobs_with_submission_id, stop_ray_job
 from cortexgrid.model_storage import delete_models_for_run
 from haikunator import Haikunator  # type: ignore
 from mlflow.entities import Experiment as MlflowExperiment
-from mlflow.entities import Run as MlflowRun
 from mlflow.tracking import MlflowClient
 
 
 log = logging.getLogger(__name__)
 _SINGLETON_EXPERIMENT: Experiment | None = None
 
-# Written on a run or an experiment that has been asked to go away. It is
-# what keeps such a record out of every by-name lookup while the control
-# plane, the one thing that reads it on purpose, tears the record down.
-DELETE_REQUESTED_TAG = "cortexgrid.delete_requested"
-
 
 @dataclass
 class Experiment:
@@ -127,19 +123,9 @@ def _try_create_experiment_and_run(
         experiment = name_gen.haikunate(token_length=2, token_chars="0123456789")
 
     client = MlflowClient(tracking_uri=mlflow_tracking_uri)
-    # An experiment that asked to be deleted is invisible to the lookup
-    # below, so a new one would quietly take its name while the control
-    # plane is still tearing the old one down — and any run created here
-    # would be torn down with it. Refuse instead.
-    pending = get_experiment_by_name(experiment, include_deleting=True)
-    if pending is not None and _pending_deletion(pending.tags):
-        raise RuntimeError(
-            f"Experiment {experiment!r} is being deleted; wait for the "
-            "control plane to finish before creating it again"
-        )
     # get_experiment_by_name also returns deleted experiments. A deleted one
     # cannot take new runs, so a fresh experiment is created under its name.
-    experiment_obj = get_experiment_by_name(experiment)
+    experiment_obj = client.get_experiment_by_name(name=experiment)
     if experiment_obj is not None and experiment_obj.lifecycle_stage != "active":
         _release_deleted_experiment_name(client, experiment_obj)
         experiment_obj = None
@@ -182,48 +168,53 @@ def _release_deleted_experiment_name(
 
 
 def delete_run(run_id: str) -> None:
-    """Ask for a run, and everything under it, to be deleted.
-
-    Intent only: every job in the run is latched and the run itself is
-    tagged. The control plane is the only thing that acts on either — it
-    stops the Ray attempts, wipes the job packages and artifacts, and
-    removes the run once nothing is left under it.
-
-    The tag takes the run out of every by-name lookup here, so a caller
-    that deletes a run and immediately looks one up never gets the one on
-    its way out. Models are not jobs and nothing else touches them, so
-    they are deleted outright.
-    """
-    log.info("delete_run(%s): requesting deletion", run_id)
-    request_run_jobs_deletion(run_id)
-    delete_models_for_run(run_id)
+    """Soft-delete a run in MLflow, cancel its Ray attempts, and wipe its
+    S3 job packages so it cannot be relaunched or re-read.
+
+    stop_experiment_run_jobs runs first so the control plane stops spawning
+    fresh Ray attempts for retry=True jobs before we tear the run down."""
+    log.info("delete_run(%s): start", run_id)
+    stop_experiment_run_jobs(run_id)
+    log.info("delete_run(%s): stop_experiment_run_jobs done", run_id)
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    client.set_tag(run_id, DELETE_REQUESTED_TAG, "true")
+    job_ids = [
+        Path(f.path).name for f in client.list_artifacts(run_id, path="job") if f.is_dir
+    ]
+    log.info("delete_run(%s): %d job artifact(s) to clean", run_id, len(job_ids))
+    all_submissions = list_ray_jobs_with_submission_id()
+    for job_id in job_ids:
+        prefix = f"{run_id}-{job_id}-"
+        for sid in all_submissions:
+            if sid.startswith(prefix):
+                stop_ray_job(sid)
+        s3_util.delete_prefix(f"job/{job_id}/")
+    delete_models_for_run(run_id)
+    client.delete_run(run_id)
+    log.info("delete_run(%s): done", run_id)
 
 
 def list_run_ids_in_experiment(name: str) -> list[str]:
     """Return the run IDs of every active run in the named experiment."""
-    exp = get_experiment_by_name(name)
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    exp = client.get_experiment_by_name(name)
     if exp is None:
         return []
-    return [r.info.run_id for r in search_runs([exp.experiment_id])]
+    return [
+        r.info.run_id for r in client.search_runs(experiment_ids=[exp.experiment_id])
+    ]
 
 
 def delete_experiment(name: str) -> None:
-    """Ask for an experiment, its runs and their jobs to be deleted.
-
-    The experiment is renamed off `name` (`<name>__deleted__<id>`) as the
-    last thing this does, so by the time the call returns the name is free
-    and `Experiment.init(name)` creates a fresh experiment instead of
-    attaching to the one being torn down. Everything else is intent: the
-    control plane deletes each run once its jobs are gone, and the
-    experiment once its runs are.
-
-    Idempotent: an experiment that is absent, already deleted or already
-    on its way out is treated as success."""
-    log.info("delete_experiment(%r): requesting deletion", name)
+    """Soft-delete every run in the experiment, then the experiment itself.
+
+    The experiment is renamed (`<name>__deleted__<id>`) before it is deleted:
+    MLflow keeps a deleted experiment's name reserved, and the rename frees it
+    so `Experiment.init(name)` can create a new experiment under it.
+
+    Idempotent: already-deleted experiments are treated as success."""
+    log.info("delete_experiment(%r): start", name)
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    exp = get_experiment_by_name(name)
+    exp = client.get_experiment_by_name(name)
     if exp is None:
         log.info("delete_experiment(%r): early-exit, experiment not found", name)
         return
@@ -232,144 +223,41 @@ def delete_experiment(name: str) -> None:
             "delete_experiment(%r): early-exit, lifecycle=%s", name, exp.lifecycle_stage
         )
         return
-    runs = search_runs([exp.experiment_id])
-    log.info("delete_experiment(%r): %d run(s) to request", name, len(runs))
+    runs = list(client.search_runs(experiment_ids=[exp.experiment_id]))
+    log.info("delete_experiment(%r): %d active run(s) to delete", name, len(runs))
     for run in runs:
         delete_run(run.info.run_id)
-    client.set_experiment_tag(exp.experiment_id, DELETE_REQUESTED_TAG, "true")
     client.rename_experiment(
         exp.experiment_id, _deleted_experiment_name(name, exp.experiment_id)
     )
-    log.info("delete_experiment(%r): requested", name)
+    client.delete_experiment(exp.experiment_id)
+    log.info("delete_experiment(%r): done", name)
 
 
-def _pending_deletion(tags: dict[str, str] | None) -> bool:
-    return (tags or {}).get(DELETE_REQUESTED_TAG) == "true"
-
-
-def search_experiments(include_deleting: bool = False) -> list[MlflowExperiment]:
-    """Every experiment, minus the ones on their way out.
-
-    This is the gate: reading experiments straight off MlflowClient
-    bypasses the deletion rule and brings records back from the dead, so
-    every reader — this module, the UI streams, the control plane — comes
-    through here. Only the control plane passes `include_deleting=True`,
-    because removing them is its job.
-    """
+def list_experiments() -> list[Experiment]:
+    """Map MLflow experiment names to their run IDs."""
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    return [
-        e
-        for e in client.search_experiments()
-        if include_deleting or not _pending_deletion(e.tags)
-    ]
-
-
-def get_experiment_by_name(
-    name: str, include_deleting: bool = False
-) -> MlflowExperiment | None:
-    """The experiment under `name`, unless it is on its way out.
-
-    The gate for by-name resolution. Like MlflowClient's own call it still
-    returns an experiment MLflow has deleted — `Experiment.init` has to see
-    one to release its name — but a record that asked to be deleted is,
-    to every caller but the control plane, already gone.
-    """
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    exp = client.get_experiment_by_name(name)
-    if exp is None:
-        return None
-    if not include_deleting and _pending_deletion(exp.tags):
-        return None
-    return exp
-
-
-def search_runs(
-    experiment_ids: list[str],
-    include_deleting: bool = False,
-    filter_string: str = "",
-) -> list[MlflowRun]:
-    """Every run of those experiments, minus the ones on their way out.
-
-    The same gate, one level down, shaped like the client call it stands
-    in front of so callers keep MLflow's server-side filtering. See
-    `search_experiments`.
-    """
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    return [
-        r
-        for r in client.search_runs(
-            experiment_ids=experiment_ids, filter_string=filter_string
-        )
-        if include_deleting or not _pending_deletion(r.data.tags)
-    ]
-
-
-def list_experiments(include_deleting: bool = False) -> list[Experiment]:
-    """Map MLflow experiment names to their run IDs.
-
-    Records that asked to be deleted are left out: they are on their way
-    to disappearing and must not be reachable any more. The control plane
-    passes `include_deleting=True`, because making them disappear is its
-    job and it cannot do it without seeing them."""
     result: list[Experiment] = []
-    for exp in search_experiments(include_deleting):
-        for run in search_runs([exp.experiment_id], include_deleting):
+    for exp in client.search_experiments():
+        runs = client.search_runs(experiment_ids=[exp.experiment_id])
+        for run in runs:
             result.append(Experiment(exp.name, run_id=run.info.run_id))
     return result
 
 
-def runs_pending_deletion() -> list[str]:
-    """Run ids tagged for deletion. The control plane is the only caller."""
-    return [
-        run.info.run_id
-        for exp in search_experiments(include_deleting=True)
-        for run in search_runs([exp.experiment_id], include_deleting=True)
-        if _pending_deletion(run.data.tags)
-    ]
-
-
-def experiments_pending_deletion() -> list[str]:
-    """Experiment ids tagged for deletion. The control plane is the only caller."""
-    return [
-        exp.experiment_id
-        for exp in search_experiments(include_deleting=True)
-        if _pending_deletion(exp.tags)
-    ]
-
-
-def experiment_has_active_runs(experiment_id: str) -> bool:
-    """Whether anything is still under the experiment. Control plane only.
-
-    Runs on their way out count: the experiment cannot go until they
-    have actually gone."""
-    return bool(search_runs([experiment_id], include_deleting=True))
-
-
-def finish_run_deletion(run_id: str) -> None:
-    """Remove a run whose jobs are gone. The control plane is the only caller."""
-    log.info("finish_run_deletion(%s)", run_id)
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    client.delete_run(run_id)
-
-
-def finish_experiment_deletion(experiment_id: str) -> None:
-    """Remove an experiment whose runs are gone. Control plane only."""
-    log.info("finish_experiment_deletion(%s)", experiment_id)
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    client.delete_experiment(experiment_id)
-
-
 def get_experiment_by_run_name(run_name: str) -> Experiment:
     """Resolve a run by its haikunator name back to its (experiment_name, run_id) pair."""
-    experiment_ids = [e.experiment_id for e in search_experiments()]
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    experiment_ids = [e.experiment_id for e in client.search_experiments()]
     if not experiment_ids:
         raise ValueError(f"No run named {run_name!r}")
-    runs = search_runs(
-        experiment_ids, filter_string=f"attributes.run_name = '{run_name}'"
+    runs = client.search_runs(
+        experiment_ids=experiment_ids,
+        filter_string=f"attributes.run_name = '{run_name}'",
+        max_results=1,
     )
     if not runs:
         raise ValueError(f"No run named {run_name!r}")
     run = runs[0]
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
     exp = client.get_experiment(run.info.experiment_id)
     return Experiment(experiment_name=exp.name, run_id=run.info.run_id)

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/cortexgrid/jobs.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import cloudpickle  # type: ignore
-from dataclasses import asdict, dataclass, field, fields
+from dataclasses import asdict, dataclass, field
 import inspect
 import io
 import json
@@ -67,16 +67,6 @@ class LifecycleEvent:
     error: str | None = None
 
 
-def _known_fields_only(cls: type, data: dict[str, Any]) -> dict[str, Any]:
-    """`data` minus the keys `cls` has no field for.
-
-    Records outlive the code that wrote them and the code that reads them:
-    a control plane one release ahead writes fields an older client has
-    never seen, and that client has to keep reading its own jobs."""
-    known = {f.name for f in fields(cls)}
-    return {k: v for k, v in data.items() if k in known}
-
-
 @dataclass
 class JobLifecycle:
     """Static identity and latches for a job.
@@ -91,7 +81,6 @@ class JobLifecycle:
     run_id: str
     job_id: str
     stop_requested: bool = False  # latch: False -> True, never cleared
-    delete_requested: bool = False  # latch: False -> True, never cleared
     retry: bool = False  # static flag set at job creation
     num_gpus: int = 0
     num_cpus: int = 1
@@ -101,30 +90,13 @@ class JobLifecycle:
     history: list[LifecycleEvent] = field(default_factory=list)
 
     def to_json(self) -> str:
-        """The record as it goes to MLflow.
-
-        A latch that is still False is left out. Every reader of this file
-        pins the fields it knows — a released client parses it with an
-        exact constructor call — so a latch a reader has never heard of
-        must not appear until something actually set it.
-        """
-        data = asdict(self)
-        if not self.delete_requested:
-            data.pop("delete_requested")
-        return json.dumps(data)
+        return json.dumps(asdict(self))
 
     @classmethod
     def from_json(cls, text: str) -> "JobLifecycle":
-        """Parse a record, including one a newer writer produced.
-
-        Fields this version does not know are dropped rather than raising,
-        so a client keeps working while the cluster runs ahead of it.
-        """
-        data = _known_fields_only(cls, json.loads(text))
-        data["history"] = [
-            LifecycleEvent(**_known_fields_only(LifecycleEvent, e))
-            for e in data.get("history", [])
-        ]
+        data = json.loads(text)
+        data.pop("error", None)
+        data["history"] = [LifecycleEvent(**e) for e in data.get("history", [])]
         return cls(**data)
 
     def get_ray_job_id(
@@ -175,23 +147,17 @@ class JobLifecycle:
 
     @classmethod
     def load_from_mlflow(cls, run_id: str, job_id: str) -> "JobLifecycle":
-        """Read a job's record.
-
-        One round trip: the fetch is the existence check. Listing the
-        directory first doubled the cost of every read, and a sweep over a
-        cluster's jobs pays it per job against a tracking server that
-        answers these one at a time.
-        """
         client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
         lifecycle_rel = f"job/{job_id}/lifecycle.json"
-        try:
-            local_path = client.download_artifacts(run_id, lifecycle_rel)
-            text = Path(local_path).read_text()
-        except Exception as err:
+        if not any(
+            a.path == lifecycle_rel
+            for a in client.list_artifacts(run_id, f"job/{job_id}")
+        ):
             raise FileNotFoundError(
                 f"artifact {lifecycle_rel} not found in run {run_id}"
-            ) from err
-        return cls.from_json(text)
+            )
+        local_path = client.download_artifacts(run_id, lifecycle_rel)
+        return cls.from_json(Path(local_path).read_text())
 
 
 class Payload(BaseModel):
@@ -530,37 +496,22 @@ def schedule_remote_job(
     return JobFuture(experiment_name=experiment_name, run_id=run_id, job_id=job_id)
 
 
-def list_experiment_run_job_ids(run_id: str) -> list[str]:
-    """The ids of every job of this run — one listing, no records read.
-
-    Reading a job's record costs a download each; a caller that wants many
-    of them can take this list and fetch them as it pleases."""
-    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
-    return [
-        Path(entry.path).name
-        for entry in client.list_artifacts(run_id, path="job")
-        if entry.is_dir
-    ]
-
-
-def load_job(run_id: str, job_id: str) -> JobLifecycle | None:
-    """The job's record, or None when it has not landed yet.
-
-    A job whose lifecycle is still being written is a normal state, not an
-    error: the caller lists it again on the next poll."""
-    try:
-        return JobLifecycle.load_from_mlflow(run_id, job_id)
-    except Exception:
-        logging.getLogger(__name__).warning(
-            "Skipping job %s: missing lifecycle", job_id
-        )
-        return None
-
-
 def list_experiment_run_jobs(run_id: str) -> list[JobLifecycle]:
     """Return all jobs and their lifecycle states for this experiment+run."""
-    jobs = (load_job(run_id, job_id) for job_id in list_experiment_run_job_ids(run_id))
-    return [job for job in jobs if job is not None]
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    entries = client.list_artifacts(run_id, path="job")
+    result: list[JobLifecycle] = []
+    for entry in entries:
+        if not entry.is_dir:
+            continue
+        job_id = Path(entry.path).name
+        try:
+            result.append(JobLifecycle.load_from_mlflow(run_id, job_id))
+        except Exception:
+            logging.getLogger(__name__).warning(
+                "Skipping job %s: missing lifecycle", job_id
+            )
+    return result
 
 
 def stop_experiment_run_jobs(run_id: str) -> None:
@@ -579,30 +530,3 @@ def stop_experiment_run_jobs(run_id: str) -> None:
             continue
         job.stop_requested = True
         job.save_to_mlflow()
-
-
-def request_job_deletion(run_id: str, job_id: str) -> None:
-    """Ask for one job to be deleted by flipping the delete_requested latch.
-
-    This function never touches Ray, S3 or the job's artifacts: it
-    records the intent on the job's own record and returns. The control
-    plane observes the latch on its next poll and is the only thing that
-    tears the job down, so a job created and deleted between two polls
-    is simply never submitted.
-
-    Idempotent: an already-requested job is left alone.
-    """
-    lifecycle = JobLifecycle.load_from_mlflow(run_id, job_id)
-    if lifecycle.delete_requested:
-        return
-    lifecycle.delete_requested = True
-    lifecycle.save_to_mlflow()
-
-
-def request_run_jobs_deletion(run_id: str) -> None:
-    """Ask for every job in the run to be deleted. See `request_job_deletion`."""
-    for job in list_experiment_run_jobs(run_id):
-        if job.delete_requested:
-            continue
-        job.delete_requested = True
-        job.save_to_mlflow()

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/cortexgrid/mlflow_util.py

@@ -14,7 +14,6 @@ import requests
 from cortexgrid.experiment import Experiment
 from cortexgrid.infra import get_mlflow_tracking_uri
 from mlflow.entities import Metric
-from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
 from mlflow.tracking import MlflowClient
 
 
@@ -109,16 +108,3 @@ def list_run_artifacts(run_id: str, path: str = "") -> list[str]:
     """Return artifact paths for a run."""
     client = get_mlflow_client()
     return [a.path for a in client.list_artifacts(run_id, path=path)]
-
-
-def delete_run_artifacts(run_id: str, path: str) -> None:
-    """Delete everything a run stores under `path`, directory and all.
-
-    MlflowClient has no delete: it goes through the run's artifact
-    repository, which the tracking server proxies to object storage."""
-    client = get_mlflow_client()
-    repo = get_artifact_repository(
-        client.get_run(run_id).info.artifact_uri,
-        tracking_uri=get_mlflow_tracking_uri(),
-    )
-    repo.delete_artifacts(path)

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/cortexgrid/ray_util.py

@@ -58,43 +58,23 @@ def get_ray_status(ray_job_id: str | None) -> str | None:
     return client.get_job_status(ray_job_id).value
 
 
-def to_job_status(ray_status: str | None) -> JobStatus:
-    """Map one of Ray's words onto ours. `None` means never submitted."""
-    if ray_status is None:
-        return JobStatus.PENDING
-    if ray_status == "SUCCEEDED":
-        return JobStatus.FINISHED
-    if ray_status == "FAILED":
-        return JobStatus.FAILED
-    if ray_status == "STOPPED":
-        return JobStatus.STOPPED
-    return JobStatus.RUNNING
-
-
 def get_ray_job_status(ray_job_id: str | None) -> JobStatus:
     """Derive a job's observable status from a live Ray query.
 
     Returns ``PENDING`` both when ``ray_job_id is None`` (never submitted)
     and when Ray itself reports ``PENDING`` (queued). Callers that need
     to distinguish those two must check ``ray_job_id is None`` first.
-
-    One query per job. A caller with many jobs wants
-    ``list_ray_job_statuses`` instead, which answers for all of them in
-    a single call.
     """
+    ray_status = get_ray_status(ray_job_id)
     if ray_job_id is None:
         return JobStatus.PENDING
-    return to_job_status(get_ray_status(ray_job_id))
-
-
-def list_ray_job_statuses() -> dict[str, JobStatus]:
-    """Every ray job's status, by submission id, in one call."""
-    client = get_ray_job_submission_client()
-    return {
-        job.submission_id: to_job_status(job.status.value)
-        for job in client.list_jobs()
-        if job.submission_id is not None
-    }
+    if ray_status == "SUCCEEDED":
+        return JobStatus.FINISHED
+    if ray_status == "FAILED":
+        return JobStatus.FAILED
+    if ray_status == "STOPPED":
+        return JobStatus.STOPPED
+    return JobStatus.RUNNING
 
 
 def get_ray_logs(ray_job_id: str | None) -> str | None:
@@ -121,15 +101,6 @@ def stop_ray_job(ray_job_id: str) -> None:
     client.stop_job(ray_job_id)
 
 
-def delete_ray_job(ray_job_id: str) -> None:
-    """Drop a ray job and its data from Ray's job store.
-
-    Ray rejects the call for a job that has not settled, so callers stop
-    the job and let it reach a terminal state first."""
-    client = get_ray_job_submission_client()
-    client.delete_job(ray_job_id)
-
-
 def list_ray_jobs_with_submission_id() -> list[str]:
     """List all ray jobs, the ones that received submission id."""
     client = get_ray_job_submission_client()

{cortexgrid-0.3.7 → cortexgrid-0.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.3.7"
+version = "0.3.8"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"