cortexgrid 0.3.15__tar.gz → 0.3.16__tar.gz

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.15
+Version: 0.3.16
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
@@ -213,8 +213,15 @@ from fastapi import FastAPI
 
 app = FastAPI()
 
+class MyClient(cortexgrid.DeploymentClient):
+    def complete(self, prompt: str) -> str: ...
+
 @serve.ingress(app)
 class MyServeApp:
+    @classmethod
+    def client(cls, deployment: cortexgrid.Deployment[MyClient]) -> MyClient:
+        return MyClient(key=deployment.key, url=deployment.url)
+
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
@@ -228,8 +235,8 @@ saved = cortexgrid.save_model(
     # What one replica needs; the model is deployed only on a host that has it.
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
-deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
-print(deployed.url)
+deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
+model = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/cortexgrid/__init__.py

@@ -102,6 +102,7 @@ from cortexgrid.model_storage import register_model as _register_model_storage
 from cortexgrid.model_storage import save_model as _save_model_storage
 from cortexgrid.model_serving import (
     Deployment,
+    DeploymentClient,
     DeploymentConfig,
     DeploymentKey,
     ModelDeployFailed,
@@ -240,7 +241,7 @@ def deploy_model(
     wait: bool = False,
     timeout: float | None = 300.0,
     config: dict[str, str] | None = None,
-) -> Deployment:
+) -> Deployment[Any]:
     experiment = active_experiment()
     return _deploy_model_serving(
         family,
@@ -349,6 +350,7 @@ __all__ = [
     "delete_model",
     # Model serving
     "Deployment",
+    "DeploymentClient",
     "DeploymentConfig",
     "DeploymentKey",
     "ModelDeployFailed",

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/cortexgrid/model_serving/__init__.py

@@ -35,9 +35,12 @@ from cortexgrid.model_serving.deployment_key import (
     deployment_key,
 )
 from cortexgrid.model_serving.lifecycle import (
+    Deployment,
+    DeploymentClient,
     ModelDeployFailed,
     ModelNotDeployed,
     deploy_model,
+    list_deployed_models,
     redeploy_model,
     required_models,
     undeploy_model,
@@ -62,12 +65,10 @@ from cortexgrid.model_serving.serve_bundle import (
     upload_bundle,
 )
 from cortexgrid.model_serving.status import (
-    Deployment,
     ReplicaPlacement,
     ServingMessage,
     ServingStatus,
     deployment_config,
-    list_deployed_models,
     model_replica_placements,
     model_serving_messages,
     model_serving_status,

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/cortexgrid/model_serving/lifecycle.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import asyncio
+import importlib
 import time
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Generic, TypeVar
 
 from ray.serve.schema import ApplicationStatus
 
@@ -22,6 +25,8 @@ from cortexgrid.model_serving.deployment_records import (
     DeploymentRecord,
     delete_deployment_record,
     get_deployment_record,
+    key_of_record,
+    list_deployment_records,
     patch_deployment_record,
     put_deployment_record,
 )
@@ -29,9 +34,8 @@ from cortexgrid.model_serving.placement import ModelRequirements, vram_tiers
 from cortexgrid.model_serving.registry_tags import load_deploy_metadata
 from cortexgrid.model_serving.serve_bundle import BundleMetadata
 from cortexgrid.model_serving.status import (
+    PHASE_NOT_DEPLOYED,
     PHASE_PAUSED,
-    Deployment,
-    deployment_of_record,
     observed,
     replaced_bundle_fingerprint_until_rolled_out,
 )
@@ -101,16 +105,12 @@ def _wait_for_application_running(
     """`wait_for_model_serving` against a deadline already running. `timeout`
     only labels the TimeoutError."""
     while True:
-        app = get_serve_details().get("applications", {}).get(name)
-        if app is None:
-            raise ModelDeployFailed(f"Serve app {name!r} does not exist")
+        app = _application_still_able_to_serve(name)
         status = str(app.get("status", "(missing)"))
-        message = str(app.get("message", ""))
         if status == ApplicationStatus.RUNNING.value:
             return
-        if status == ApplicationStatus.DEPLOY_FAILED.value:
-            raise ModelDeployFailed(f"Serve app {name!r} DEPLOY_FAILED: {message}")
         if _past(deadline):
+            message = str(app.get("message", ""))
             raise TimeoutError(
                 f"Serve app {name!r} did not reach RUNNING within {timeout}s "
                 f"(last status={status!r}, message={message!r})"
@@ -118,6 +118,90 @@ def _wait_for_application_running(
         time.sleep(_SERVING_POLL_INTERVAL_S)
 
 
+def _application_still_able_to_serve(name: str) -> dict[str, Any]:
+    serve_details = get_serve_details()
+    applications = serve_details.get("applications", {})
+    app: dict[str, Any] | None = applications.get(name)
+    if app is None:
+        raise ModelDeployFailed(f"Serve app {name!r} does not exist")
+    status = app.get("status")
+    if status == ApplicationStatus.DEPLOY_FAILED.value:
+        message = app.get("message", "")
+        raise ModelDeployFailed(f"Serve app {name!r} DEPLOY_FAILED: {message}")
+    return app
+
+
+def _model_is_serving(key: DeploymentKey) -> bool:
+    name = app_name(key)
+    app = _application_still_able_to_serve(name)
+    status = app.get("status")
+    return status == ApplicationStatus.RUNNING.value
+
+
+@dataclass
+class DeploymentClient:
+    key: DeploymentKey
+    url: str
+
+    async def is_ready(self) -> bool:
+        serving = await asyncio.to_thread(_model_is_serving, self.key)
+        return serving
+
+
+DeploymentClientT = TypeVar("DeploymentClientT", bound=DeploymentClient)
+
+
+@dataclass
+class Deployment(Generic[DeploymentClientT]):
+    """A scheduled Ray Serve app fronting a model. `phase` is the normalized
+    serving lifecycle phase (see `ServingStatus`); an app that appears in a
+    listing always exists, so its phase is never "not_deployed"."""
+
+    key: DeploymentKey
+    config: dict[str, str]
+    url: str
+    phase: str
+    bundle_fingerprint: str
+    replaced_bundle_fingerprint: str
+    experiment_name: str
+    class_import_path: str
+
+    def client(self) -> DeploymentClientT:
+        wait_for_model_serving(self.key)
+        deployment_client = self.client_async()
+        return deployment_client
+
+    def client_async(self) -> DeploymentClientT:
+        module_name, class_name = self.class_import_path.split(":")
+        serve_app_module = importlib.import_module(module_name)
+        serve_app = getattr(serve_app_module, class_name)
+        deployment_client: DeploymentClientT = serve_app.client(self)
+        return deployment_client
+
+
+def list_deployed_models() -> list[Deployment[Any]]:
+    """Return a Deployment for every model `deploy_model` put on Ray Serve
+    whose app the control plane last saw existing."""
+    return [
+        deployment_of_record(record)
+        for record in list_deployment_records()
+        if record["phase"] != PHASE_NOT_DEPLOYED
+    ]
+
+
+def deployment_of_record(record: DeploymentRecord) -> Deployment[Any]:
+    return Deployment(
+        key=key_of_record(record),
+        config=record["config"],
+        url=record["url"],
+        phase=record["phase"],
+        bundle_fingerprint=bundle_fingerprint_in_spec(record["spec"]),
+        replaced_bundle_fingerprint=record["replaced_bundle_fingerprint"],
+        experiment_name=record["experiment_name"],
+        class_import_path=record["spec"]["args"]["class_import_path"],
+    )
+
+
 def _clear_failed_application(
     key: DeploymentKey, timeout: float | None, deadline: float | None
 ) -> None:
@@ -225,11 +309,10 @@ def deploy_model(
     timeout: float | None = 300.0,
     config: dict[str, str] | None = None,
     experiment_name: str = "",
-) -> Deployment:
+) -> Deployment[Any]:
     """Schedule a Ray Serve app for a previously-saved model and return a
-    handle carrying its base URL. The caller (e.g. model-gateway) builds
-    whatever client the app's routes need - streaming, long timeouts, custom
-    request schemas - against that URL; cortexgrid imposes no traffic contract.
+    handle on it. The handle's `client()` is the client the serve-app's own
+    `client` factory makes for its routes; cortexgrid imposes no traffic contract.
 
     The serve-app class is pulled from the registry entry's tags `save_model`
     wrote at save time; the caller does not need to hold the class object.
@@ -332,10 +415,11 @@ def deploy_model(
         bundle_fingerprint=meta.fingerprint,
         replaced_bundle_fingerprint=observation["replaced_bundle_fingerprint"],
         experiment_name=experiment_name,
+        class_import_path=meta.class_import_path,
     )
 
 
-def redeploy_model(key: DeploymentKey) -> Deployment:
+def redeploy_model(key: DeploymentKey) -> Deployment[Any]:
     record = get_deployment_record(key)
     if record is None:
         raise ModelNotDeployed(f"{key} is not deployed")
@@ -349,14 +433,14 @@ def redeploy_model(key: DeploymentKey) -> Deployment:
     )
 
 
-def required_models(deployment: DeploymentKey) -> list[Deployment]:
+def required_models(deployment: DeploymentKey) -> list[Deployment[Any]]:
     _, requirements = load_deploy_metadata(
         deployment.family, deployment.suffix, deployment.run_name
     )
     return [_deployment_of(model) for model in requirements.models]
 
 
-def _deployment_of(model: DeploymentConfig) -> Deployment:
+def _deployment_of(model: DeploymentConfig) -> Deployment[Any]:
     key = deployment_key(model.family, model.suffix, model.run_name, model.config)
     record = get_deployment_record(key)
     if record is None:

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/cortexgrid/model_serving/serve_bundle.py

@@ -55,12 +55,19 @@ def build_bundle(cls: type) -> ServeBundle:
     is a subclass Ray defines in its own module, and on older Ray (e.g. 2.9) it
     reports that module as its own, so the class's source and import path would
     resolve to Ray instead of the serve-app. `cortexgrid.serve.ingress` leaves
-    the class unwrapped."""
+    the class unwrapped. Raises ValueError, too, for a class with no `client`
+    factory, which every `Deployment` of it makes its client with."""
     if any(klass.__module__.startswith("ray.serve") for klass in cls.__mro__):
         raise ValueError(
             f"{cls.__name__} is wrapped by ray.serve.ingress; decorate it with "
             "cortexgrid.serve.ingress instead (from cortexgrid import serve)"
         )
+    if not callable(getattr(cls, "client", None)):
+        raise ValueError(
+            f"{cls.__name__} has no client factory; give it a classmethod "
+            "client(cls, deployment: cortexgrid.Deployment) returning its "
+            "cortexgrid.DeploymentClient"
+        )
     entry_file = Path(inspect.getfile(cls)).resolve()
     serve_entry = Path(__file__).parent.with_name("_serve_entry.py")
     desc = bundle(entry_file).merge(bundle(serve_entry))

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/cortexgrid/model_serving/status.py

@@ -6,13 +6,9 @@ from typing import Any
 
 from ray.serve.schema import ApplicationStatus, ReplicaState
 
-from cortexgrid.model_serving.application_spec import (
-    app_name,
-    bundle_fingerprint_in_spec,
-)
+from cortexgrid.model_serving.application_spec import app_name
 from cortexgrid.model_serving.deployment_key import DeploymentKey
 from cortexgrid.model_serving.deployment_records import (
-    DeploymentRecord,
     get_deployment_record,
     key_of_record,
     list_deployment_records,
@@ -21,7 +17,7 @@ from cortexgrid.model_serving.deployment_records import (
 from cortexgrid.ray_util import get_serve_details
 
 
-_PHASE_NOT_DEPLOYED = "not_deployed"
+PHASE_NOT_DEPLOYED = "not_deployed"
 PHASE_PAUSED = "paused"
 _ROLLED_OUT_PHASES = ("running", PHASE_PAUSED)
 
@@ -44,49 +40,12 @@ def _serve_phase(raw_status: str) -> str:
     return _PHASE_BY_SERVE_STATUS.get(raw_status, "deploying")
 
 
-@dataclass
-class Deployment:
-    """A scheduled Ray Serve app fronting a model. `phase` is the normalized
-    serving lifecycle phase (see `ServingStatus`); an app that appears in a
-    listing always exists, so its phase is never "not_deployed"."""
-
-    key: DeploymentKey
-    config: dict[str, str]
-    url: str
-    phase: str
-    bundle_fingerprint: str
-    replaced_bundle_fingerprint: str
-    experiment_name: str
-
-
 def replaced_bundle_fingerprint_until_rolled_out(
     replaced_bundle_fingerprint: str, phase: str
 ) -> str:
     return "" if phase in _ROLLED_OUT_PHASES else replaced_bundle_fingerprint
 
 
-def list_deployed_models() -> list[Deployment]:
-    """Return a Deployment for every model `deploy_model` put on Ray Serve
-    whose app the control plane last saw existing."""
-    return [
-        deployment_of_record(record)
-        for record in list_deployment_records()
-        if record["phase"] != _PHASE_NOT_DEPLOYED
-    ]
-
-
-def deployment_of_record(record: DeploymentRecord) -> Deployment:
-    return Deployment(
-        key=key_of_record(record),
-        config=record["config"],
-        url=record["url"],
-        phase=record["phase"],
-        bundle_fingerprint=bundle_fingerprint_in_spec(record["spec"]),
-        replaced_bundle_fingerprint=record["replaced_bundle_fingerprint"],
-        experiment_name=record["experiment_name"],
-    )
-
-
 def deployment_config(key: DeploymentKey) -> dict[str, str]:
     record = get_deployment_record(key)
     if record is None:
@@ -134,8 +93,8 @@ def model_serving_status(key: DeploymentKey) -> ServingStatus:
     `cortexgrid.model_storage.model_registry_status`.
     """
     record = get_deployment_record(key)
-    if record is None or record["phase"] == _PHASE_NOT_DEPLOYED:
-        return ServingStatus(key, _PHASE_NOT_DEPLOYED, "", None)
+    if record is None or record["phase"] == PHASE_NOT_DEPLOYED:
+        return ServingStatus(key, PHASE_NOT_DEPLOYED, "", None)
     return ServingStatus(
         key=key,
         phase=record["phase"],
@@ -193,7 +152,7 @@ def observed(app: dict[str, Any] | None) -> dict[str, Any]:
     deployment record keeps; an app that does not exist reads as
     "not_deployed"."""
     if app is None:
-        return {"phase": _PHASE_NOT_DEPLOYED, "message": "", "replicas": []}
+        return {"phase": PHASE_NOT_DEPLOYED, "message": "", "replicas": []}
     raw = str(app.get("status", ""))
     return {
         "phase": _phase(app),

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/docs/cortexgrid/README.md

@@ -185,8 +185,15 @@ from fastapi import FastAPI
 
 app = FastAPI()
 
+class MyClient(cortexgrid.DeploymentClient):
+    def complete(self, prompt: str) -> str: ...
+
 @serve.ingress(app)
 class MyServeApp:
+    @classmethod
+    def client(cls, deployment: cortexgrid.Deployment[MyClient]) -> MyClient:
+        return MyClient(key=deployment.key, url=deployment.url)
+
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
@@ -200,8 +207,8 @@ saved = cortexgrid.save_model(
     # What one replica needs; the model is deployed only on a host that has it.
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
-deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
-print(deployed.url)
+deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
+model = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.15 → cortexgrid-0.3.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.3.15"
+version = "0.3.16"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"