cortexgrid 0.2.96__tar.gz → 0.2.97__tar.gz

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.2.96
+Version: 0.2.97
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
@@ -165,19 +165,23 @@ app = FastAPI()
 
 @serve.ingress(app)
 class MyServeApp:
-    num_gpus = 1
-
     def __init__(self, family: str, suffix: str, run_name: str) -> None:
         self._weights_dir = cortexgrid.load_model(family, suffix, run_name)
 
     @app.post("/complete")
     async def complete(self, body: dict): ...
 
-saved = cortexgrid.save_model(weights_dir, MyServeApp, family="qwen", suffix="instruct")
+saved = cortexgrid.save_model(
+    weights_dir, MyServeApp, family="qwen", suffix="instruct",
+    # What one replica needs; the model is deployed only on a host that has it.
+    requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
+)
 deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
 print(deployed.url)
 ```
 
+The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.
+
 `save_model` saves a new copy under every run - meant for weights the run produced (e.g. a fine-tune). For a model produced elsewhere (e.g. a pretrained base model), `cortexgrid.import_model(source, MyServeApp, family, suffix)` uploads it once under `run_name=cortexgrid.IMPORTED` and on later runs only re-bundles `MyServeApp` if its code changed; deploy it with `deploy_model(family, suffix, cortexgrid.IMPORTED)`.
 
 `save_model` is synchronous (registry lifecycle: `uploading` -> `ready`); `deploy_model` schedules the serving lifecycle (`deploying` -> `running`). With `wait=True` a failed deploy raises `cortexgrid.ModelDeployFailed`; `cortexgrid.wait_for_model_serving(family, suffix, run_name, timeout=...)` waits on a deploy started elsewhere, and re-deploying a failed model retries it from scratch. See [model-serving.md](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md) for both lifecycles end to end - upload/deploy/undeploy/delete, status queries (`model_registry_status`, `model_serving_status`), and error handling.

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/cortexgrid/__init__.py

@@ -80,12 +80,14 @@ from cortexgrid.model_storage import (
     list_models,
     load_model,
     model_registry_status,
+    set_model_requirements,
 )
 from cortexgrid.model_storage import import_model as _import_model_storage
 from cortexgrid.model_storage import save_model as _save_model_storage
 from cortexgrid.model_serving import (
     Deployment,
     ModelDeployFailed,
+    ModelRequirements,
     ServingStatus,
     deploy_model,
     list_deployed_models,
@@ -118,10 +120,15 @@ def remote(
 
 
 def save_model(
-    weights_dir: str | Path, serve_app: type, family: str, suffix: str
+    weights_dir: str | Path,
+    serve_app: type,
+    family: str,
+    suffix: str,
+    requirements: ModelRequirements | None = None,
 ) -> SavedModel:
     """Persist a weights directory under the current Experiment's run, paired
-    with the serve-app class that will front it at deploy time.
+    with the serve-app class that will front it at deploy time and the
+    hardware one replica of it needs.
 
     Every run saves a new copy under its own run_name - meant for weights the
     run produced (e.g. a fine-tune). For a model produced elsewhere that should
@@ -134,6 +141,7 @@ def save_model(
         family,
         run_id=experiment.run_id,
         run_name=experiment.run_name(),
+        requirements=requirements,
     )
 
 
@@ -142,6 +150,7 @@ def import_model(
     serve_app: type,
     family: str,
     suffix: str,
+    requirements: ModelRequirements | None = None,
 ) -> SavedModel:
     """Register a model produced elsewhere once, reuse it on every later call,
     and record on the current Experiment's run which imported model it used.
@@ -151,7 +160,7 @@ def import_model(
     `imported_model/<family>/<suffix>` holds the version's `created_at`, set
     whether this call uploaded the model or reused it."""
     experiment = Experiment.get_instance()
-    model = _import_model_storage(source, serve_app, family, suffix)
+    model = _import_model_storage(source, serve_app, family, suffix, requirements)
     get_mlflow_client().set_tag(
         experiment.run_id, f"imported_model/{family}/{suffix}", model.created_at
     )
@@ -213,6 +222,8 @@ __all__ = [
     "load_model",
     "list_models",
     "model_registry_status",
+    "ModelRequirements",
+    "set_model_requirements",
     "delete_model",
     # Model serving
     "Deployment",

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/cortexgrid/_serve_entry.py

@@ -4,9 +4,9 @@ Ray Serve's REST `import_path` resolves to `cortexgrid._serve_entry:build`.
 On the cluster replica, `build` imports the serve-app class bundled at
 `save_model` time (its import path was stored as an MLflow tag), applies Ray's
 ingress with the app it was marked with by `cortexgrid.serve.ingress` (again on
-each replica, see `_IngressOnReplica`), reads its
-`num_gpus`/`num_replicas` class attributes for actor placement, wraps it as a
-Ray Serve deployment, and binds it with the (family, suffix, run_name)
+each replica, see `_IngressOnReplica`), wraps it as a Ray Serve deployment
+with the replica count and Ray resource requests `deploy_model` derived from
+the model's requirements, and binds it with the (family, suffix, run_name)
 identifiers.
 
 The serve-app owns everything about traffic: its own routes, request schemas,
@@ -14,11 +14,9 @@ streaming, and timeouts. cortexgrid does not interpose a request/response
 contract - it only schedules the app and hands it the identifiers it needs to
 fetch its own weights via `cortexgrid.load_model`.
 
-Design note: resource needs (`num_gpus`/`num_replicas`) are read from plain
-class attributes rather than a cortexgrid decorator or base class. This is a
-deliberate, provisional choice - kept minimal until we see how serve-apps
-declare resources in practice; revisit if plain class attributes prove too
-limited.
+The serve-app declares no resources: the hardware a replica needs belongs to
+the model and is stored in the registry (`cortexgrid.ModelRequirements`), and
+the replica count is chosen per `deploy_model`.
 """
 
 from __future__ import annotations
@@ -72,12 +70,13 @@ def build(args: dict[str, Any]) -> Application:
             {"_serve_app": serve_app},
         )
         serve_app = serve.ingress(app)(on_replica)
-    num_gpus = getattr(serve_app, "num_gpus", 0)
-    num_replicas = getattr(serve_app, "num_replicas", 1)
     return serve.deployment(serve_app).options(
-        num_replicas=num_replicas,
+        # This builder ships in the bundle, frozen at save time, while `args`
+        # come from the cortexgrid that deploys it; one older than the bundle
+        # sends neither key.
+        num_replicas=args.get("num_replicas", 1),
         # Ray 2.32 lowered the default from 100 to 5; keep what serve-apps
         # had on Ray 2.9.
         max_ongoing_requests=_MAX_ONGOING_REQUESTS,
-        ray_actor_options={"num_gpus": num_gpus},
+        ray_actor_options=args.get("ray_actor_options", {}),
     ).bind(args["family"], args["suffix"], args["run_name"])

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/cortexgrid/model_serving.py

@@ -8,7 +8,8 @@ uploaded to MinIO under
 referenced via `runtime_env.working_dir` so Ray workers fetch it from there.
 The bundle URL, class import path, pip list, and fingerprint are persisted as
 MLflow tags on the ModelVersion so `deploy_model` can find them later without
-the caller holding the class object.
+the caller holding the class object. So are the model's `ModelRequirements`,
+which `deploy_model` turns into the replica's Ray resource requests.
 
 Naming: the Ray Serve application is named "<family>__<suffix>__<run_name>".
 This relies on family/suffix/run_name not containing the literal "__".
@@ -206,10 +207,41 @@ def upload_bundle(
     )
 
 
+# Custom Ray resource each GPU worker advertises: the MiB of memory its GPUs
+# have (see the ray-worker DaemonSet). A replica requests its vram_gb of it in
+# MiB, so Ray places it only on a node with that much VRAM left. MiB because
+# nvidia-smi reports MiB and a GPU's memory is not a whole number of GiB.
+_VRAM_RESOURCE = "vram_mib"
+
+_MIB_PER_GIB = 1024
+
+_GIB = 1024**3
+
+
+def _ray_actor_options(requirements: ModelRequirements) -> dict[str, Any]:
+    """Translate ModelRequirements into a replica's Ray actor resource requests.
+    Ray places the replica only on a node with that much free and reserves it
+    there; a zero requirement requests nothing."""
+    options: dict[str, Any] = {"num_gpus": requirements.num_gpus}
+    if requirements.ram_gb > 0:
+        options["memory"] = int(requirements.ram_gb * _GIB)
+    if requirements.vram_gb > 0:
+        options["resources"] = {
+            _VRAM_RESOURCE: round(requirements.vram_gb * _MIB_PER_GIB)
+        }
+    return options
+
+
 def _build_application_spec(
-    family: str, suffix: str, run_name: str, meta: BundleMetadata
+    family: str,
+    suffix: str,
+    run_name: str,
+    meta: BundleMetadata,
+    requirements: ModelRequirements,
+    num_replicas: int,
 ) -> dict[str, Any]:
-    """Assemble a Ray Serve application schema from pre-bundled metadata."""
+    """Assemble a Ray Serve application schema from pre-bundled metadata and
+    the model's requirements."""
     # working_dir carries the serve-app's own source; Ray pip-installs the
     # third-party distributions the image lacks into a per-node cached
     # virtualenv layered on the image. No pip key when there are none, so Ray
@@ -230,6 +262,8 @@ def _build_application_spec(
             "family": family,
             "suffix": suffix,
             "run_name": run_name,
+            "num_replicas": num_replicas,
+            "ray_actor_options": _ray_actor_options(requirements),
         },
         "runtime_env": runtime_env,
     }
@@ -268,10 +302,66 @@ def metadata_from_tags(tags: dict[str, str]) -> BundleMetadata:
     )
 
 
-def _load_bundle_metadata(
+@dataclass
+class ModelRequirements:
+    """Hardware one replica of a model needs to be served, in GiB. Persisted as
+    tags on the ModelVersion next to the bundle metadata, so it is read without
+    touching the weights or importing the serve-app class.
+
+    Zero means no requirement: a model with no requirements is served on any
+    node, CPU-only included. Models saved before requirements existed carry
+    no tags and read as the defaults."""
+
+    num_gpus: int = 0
+    ram_gb: float = 0.0
+    # GPU memory across the replica's num_gpus GPUs, so it needs num_gpus >= 1.
+    vram_gb: float = 0.0
+
+    def __post_init__(self) -> None:
+        if self.num_gpus < 0 or self.ram_gb < 0 or self.vram_gb < 0:
+            raise ValueError(f"Model requirements cannot be negative: {self}")
+        if self.vram_gb > 0 and self.num_gpus == 0:
+            raise ValueError(
+                f"vram_gb={self.vram_gb} needs a GPU; set num_gpus >= 1"
+            )
+
+
+# MLflow tag keys for the ModelRequirements.
+_NUM_GPUS_TAG = "num_gpus"
+_RAM_GB_TAG = "ram_gb"
+_VRAM_GB_TAG = "vram_gb"
+
+
+def has_requirement_tags(tags: dict[str, str]) -> bool:
+    """Whether requirements were ever stored on the ModelVersion."""
+    return any(key in tags for key in (_NUM_GPUS_TAG, _RAM_GB_TAG, _VRAM_GB_TAG))
+
+
+def requirements_to_tags(requirements: ModelRequirements) -> dict[str, str]:
+    """Serialise ModelRequirements to MLflow tags. The inverse of
+    `requirements_from_tags`."""
+    return {
+        _NUM_GPUS_TAG: str(requirements.num_gpus),
+        _RAM_GB_TAG: str(requirements.ram_gb),
+        _VRAM_GB_TAG: str(requirements.vram_gb),
+    }
+
+
+def requirements_from_tags(tags: dict[str, str]) -> ModelRequirements:
+    """Deserialise ModelRequirements from a ModelVersion's MLflow tags; a
+    missing tag reads as no requirement."""
+    return ModelRequirements(
+        num_gpus=int(tags.get(_NUM_GPUS_TAG, "0")),
+        ram_gb=float(tags.get(_RAM_GB_TAG, "0")),
+        vram_gb=float(tags.get(_VRAM_GB_TAG, "0")),
+    )
+
+
+def _load_deploy_metadata(
     family: str, suffix: str, run_name: str
-) -> BundleMetadata:
-    """Read the bundle metadata `save_model` persisted on the ModelVersion."""
+) -> tuple[BundleMetadata, ModelRequirements]:
+    """Read the bundle metadata and requirements `save_model` persisted on the
+    ModelVersion."""
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
     name = f"{family}__{suffix}"
     versions = client.search_model_versions(
@@ -281,8 +371,9 @@ def _load_bundle_metadata(
         raise ValueError(
             f"No saved model for {family}/{suffix}/{run_name}; cannot deploy."
         )
+    tags = versions[0].tags or {}
     try:
-        return metadata_from_tags(versions[0].tags or {})
+        return metadata_from_tags(tags), requirements_from_tags(tags)
     except KeyError as exc:
         raise ValueError(
             f"Saved model {family}/{suffix}/{run_name} is missing the deployment "
@@ -398,6 +489,7 @@ def deploy_model(
     family: str,
     suffix: str,
     run_name: str,
+    num_replicas: int = 1,
     wait: bool = False,
     timeout: float | None = 300.0,
 ) -> Deployment:
@@ -408,6 +500,8 @@ def deploy_model(
 
     The serve-app class is pulled from the MLflow ModelVersion tags `save_model`
     wrote at save time; the caller does not need to hold the class object.
+    Each of the `num_replicas` replicas requests the model's `ModelRequirements`
+    from Ray, so it is placed only on a node that has them free.
 
     A DEPLOY_FAILED app left by an earlier attempt is undeployed first, and it,
     or an app still DELETING, is waited out before the new spec is PUT, so the
@@ -421,8 +515,10 @@ def deploy_model(
     stuck in DEPLOYING) will hang forever.
     """
     deadline = _deadline(timeout)
-    meta = _load_bundle_metadata(family, suffix, run_name)
-    spec = _build_application_spec(family, suffix, run_name, meta)
+    meta, requirements = _load_deploy_metadata(family, suffix, run_name)
+    spec = _build_application_spec(
+        family, suffix, run_name, meta, requirements, num_replicas
+    )
     _clear_failed_application(family, suffix, run_name, timeout, deadline)
     existing = [a for a in _current_application_specs() if a["name"] != spec["name"]]
     # The controller registers the app, sets it DEPLOYING and stamps

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/cortexgrid/model_storage.py

@@ -8,6 +8,7 @@ Mapping cortexgrid taxonomy <-> MLflow Registry:
                            "s3://<bucket>/models/<run_name>/<family>/<suffix>/weights/"
     run linkage         -> ModelVersion.run_id  (built-in MLflow field; unset
                            for imported models)
+    requirements        -> ModelVersion.tags["num_gpus"], ["ram_gb"], ["vram_gb"]
 
 Two ways in: `save_model` registers a fresh copy under the calling run's
 run_name every time it runs (fine-tuned output); `import_model` registers a
@@ -36,10 +37,14 @@ from mlflow.tracking import MlflowClient
 from cortexgrid import s3_util
 from cortexgrid.infra import get_mlflow_tracking_uri, get_s3_bucket
 from cortexgrid.model_serving import (
+    ModelRequirements,
     build_bundle,
     bundle_class,
+    has_requirement_tags,
     metadata_from_tags,
     metadata_to_tags,
+    requirements_from_tags,
+    requirements_to_tags,
     upload_bundle,
 )
 
@@ -81,6 +86,8 @@ class SavedModel:
     # _UPLOAD_DEADLINE (writer presumed dead). Versions written before this tag
     # existed report "ready".
     phase: str
+    # Hardware one replica needs; defaults for versions stored without it.
+    requirements: ModelRequirements
 
 
 def _phase_for(version: Any) -> str:
@@ -111,6 +118,7 @@ def _to_saved_model(version: Any) -> SavedModel:
         data_blob_path=version.source,
         size_bytes=int(version.tags.get("size_bytes", "0")),
         phase=_phase_for(version),
+        requirements=requirements_from_tags(version.tags),
     )
 
 
@@ -150,6 +158,7 @@ def save_model(
     family: str,
     run_id: str,
     run_name: str,
+    requirements: ModelRequirements | None = None,
 ) -> SavedModel:
     """Upload a weights directory to S3 and register a new MLflow ModelVersion
     paired with the serve-app that fronts it.
@@ -168,8 +177,13 @@ def save_model(
     weights.
     Its code is bundled and its import path, bundle URL, and pip list are
     stored as tags on the ModelVersion so `deploy_model` can bind it later
-    without the caller holding the class object."""
-    return _upload_model(weights_dir, serve_app, suffix, family, run_id, run_name)
+    without the caller holding the class object.
+
+    `requirements` is the hardware one replica needs; None stores none, which
+    reads as no requirement. Change it later with `set_model_requirements`."""
+    return _upload_model(
+        weights_dir, serve_app, suffix, family, run_id, run_name, requirements
+    )
 
 
 def import_model(
@@ -177,6 +191,7 @@ def import_model(
     serve_app: type,
     family: str,
     suffix: str,
+    requirements: ModelRequirements | None = None,
 ) -> SavedModel:
     """Register a model produced elsewhere (e.g. a pretrained base model) under
     the fixed key (family, suffix, IMPORTED), once.
@@ -190,7 +205,9 @@ def import_model(
     If a version is already registered under the key:
       - "ready": returns it without calling `source`. If `serve_app`'s code no
         longer matches the stored bundle, it is re-bundled first and the
-        weights are kept (see `_refresh_bundle`). To replace the weights,
+        weights are kept (see `_refresh_bundle`). `requirements` are stored
+        only if the version has none yet, so values changed since with
+        `set_model_requirements` are kept. To replace the weights,
         `delete_model` it first.
       - "uploading": raises RuntimeError - another process is importing it.
       - "upload_failed" / "broken": deleted and imported again.
@@ -202,6 +219,10 @@ def import_model(
     if existing is not None:
         if existing.phase == _PHASE_READY:
             _refresh_bundle(serve_app, family, suffix)
+            if requirements is not None:
+                existing.requirements = _set_missing_requirements(
+                    family, suffix, requirements
+                )
             return existing
         if existing.phase == _PHASE_UPLOADING:
             raise RuntimeError(
@@ -209,7 +230,9 @@ def import_model(
                 "another process"
             )
         delete_model(family, suffix, IMPORTED)
-    return _upload_model(source, serve_app, suffix, family, None, IMPORTED)
+    return _upload_model(
+        source, serve_app, suffix, family, None, IMPORTED, requirements
+    )
 
 
 def _refresh_bundle(serve_app: type, family: str, suffix: str) -> None:
@@ -233,6 +256,23 @@ def _refresh_bundle(serve_app: type, family: str, suffix: str) -> None:
         client.set_model_version_tag(name, version.version, key, value)
 
 
+def _set_missing_requirements(
+    family: str, suffix: str, requirements: ModelRequirements
+) -> ModelRequirements:
+    """Store `requirements` on an imported model whose version has none yet.
+    Returns the requirements the version holds afterwards."""
+    name = f"{family}__{suffix}"
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    version = client.search_model_versions(
+        f"name='{name}' and tags.run_name='{IMPORTED}'"
+    )[0]
+    if has_requirement_tags(version.tags):
+        return requirements_from_tags(version.tags)
+    for key, value in requirements_to_tags(requirements).items():
+        client.set_model_version_tag(name, version.version, key, value)
+    return requirements
+
+
 def _upload_model(
     weights: str | Path | Callable[[], str | Path],
     serve_app: type,
@@ -240,6 +280,7 @@ def _upload_model(
     family: str,
     run_id: str | None,
     run_name: str,
+    requirements: ModelRequirements | None,
 ) -> SavedModel:
     """Register a ModelVersion in "uploading", resolve `weights` to a directory
     (calling it when it is a callable), upload the weights and the serve-app
@@ -255,7 +296,8 @@ def _upload_model(
     # fetched and uploaded, and a concurrent `import_model` sees the import in
     # flight for the whole download instead of starting one of its own. The
     # size is stamped once the directory exists; the bundle tags and the flip
-    # to "ready" happen only after the upload lands.
+    # to "ready" happen only after the upload lands. No requirements leaves
+    # their tags unset, so a later `import_model` can still store them.
     version = client.create_model_version(
         name=name,
         source=source,
@@ -265,6 +307,7 @@ def _upload_model(
             "suffix": suffix,
             "run_name": run_name,
             _LIFECYCLE_TAG: _PHASE_UPLOADING,
+            **(requirements_to_tags(requirements) if requirements is not None else {}),
         },
     )
     try:
@@ -333,6 +376,23 @@ def model_registry_status(
     return _to_saved_model(versions[0]) if versions else None
 
 
+def set_model_requirements(
+    family: str, suffix: str, run_name: str, requirements: ModelRequirements
+) -> None:
+    """Replace the hardware requirements stored on a model. Takes effect on
+    its next `deploy_model`; a replica already running keeps its placement.
+    Raises ValueError if the model was never registered."""
+    client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())
+    name = f"{family}__{suffix}"
+    versions = client.search_model_versions(
+        f"name='{name}' and tags.run_name='{run_name}'"
+    )
+    if not versions:
+        raise ValueError(f"No model {family}/{suffix}/{run_name}")
+    for key, value in requirements_to_tags(requirements).items():
+        client.set_model_version_tag(name, versions[0].version, key, value)
+
+
 def delete_model(family: str, suffix: str, run_name: str) -> None:
     """Delete the ModelVersion in MLflow, its weights blob, and its serve bundle."""
     client = MlflowClient(tracking_uri=get_mlflow_tracking_uri())

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/docs/cortexgrid/README.md

@@ -137,19 +137,23 @@ app = FastAPI()
 
 @serve.ingress(app)
 class MyServeApp:
-    num_gpus = 1
-
     def __init__(self, family: str, suffix: str, run_name: str) -> None:
         self._weights_dir = cortexgrid.load_model(family, suffix, run_name)
 
     @app.post("/complete")
     async def complete(self, body: dict): ...
 
-saved = cortexgrid.save_model(weights_dir, MyServeApp, family="qwen", suffix="instruct")
+saved = cortexgrid.save_model(
+    weights_dir, MyServeApp, family="qwen", suffix="instruct",
+    # What one replica needs; the model is deployed only on a host that has it.
+    requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
+)
 deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
 print(deployed.url)
 ```
 
+The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.
+
 `save_model` saves a new copy under every run - meant for weights the run produced (e.g. a fine-tune). For a model produced elsewhere (e.g. a pretrained base model), `cortexgrid.import_model(source, MyServeApp, family, suffix)` uploads it once under `run_name=cortexgrid.IMPORTED` and on later runs only re-bundles `MyServeApp` if its code changed; deploy it with `deploy_model(family, suffix, cortexgrid.IMPORTED)`.
 
 `save_model` is synchronous (registry lifecycle: `uploading` -> `ready`); `deploy_model` schedules the serving lifecycle (`deploying` -> `running`). With `wait=True` a failed deploy raises `cortexgrid.ModelDeployFailed`; `cortexgrid.wait_for_model_serving(family, suffix, run_name, timeout=...)` waits on a deploy started elsewhere, and re-deploying a failed model retries it from scratch. See [model-serving.md](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md) for both lifecycles end to end - upload/deploy/undeploy/delete, status queries (`model_registry_status`, `model_serving_status`), and error handling.

{cortexgrid-0.2.96 → cortexgrid-0.2.97}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.2.96"
+version = "0.2.97"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"