cortexgrid 0.3.15__tar.gz → 0.3.17__tar.gz

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.15
+Version: 0.3.17
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
@@ -11,7 +11,9 @@ Requires-Python: <3.12,>=3.11
 Requires-Dist: boto3>=1.34
 Requires-Dist: cloudpickle>=3.0
 Requires-Dist: fabric>=3.2.3
+Requires-Dist: fastapi<1,>=0.115
 Requires-Dist: haikunator>=2.1.0
+Requires-Dist: httpx>=0.27
 Requires-Dist: mlflow<4,>=3.11
 Requires-Dist: packaging>=24
 Requires-Dist: pip>=23.0
@@ -205,31 +207,28 @@ s3_client = cortexgrid.get_s3_client()           # boto3 S3 client
 
 #### Model registry and serving
 
-Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class fronted by a FastAPI app, marked with cortexgrid's `serve.ingress` (not Ray's):
+Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class marked with cortexgrid's `serve.ingress` (not Ray's), whose `serve.endpoint` methods it serves and whose client it generates:
 
 ```python
 from cortexgrid import serve
-from fastapi import FastAPI
 
-app = FastAPI()
-
-@serve.ingress(app)
+@serve.ingress
 class MyServeApp:
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
         )
 
-    @app.post("/complete")
-    async def complete(self, body: dict): ...
+    @serve.endpoint
+    async def complete(self, prompt: str) -> str: ...
 
 saved = cortexgrid.save_model(
     weights_dir, MyServeApp, family="qwen", suffix="instruct",
     # What one replica needs; the model is deployed only on a host that has it.
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
-deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
-print(deployed.url)
+deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
+model: MyServeApp = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/cortexgrid/__init__.py

@@ -102,6 +102,7 @@ from cortexgrid.model_storage import register_model as _register_model_storage
 from cortexgrid.model_storage import save_model as _save_model_storage
 from cortexgrid.model_serving import (
     Deployment,
+    DeploymentClient,
     DeploymentConfig,
     DeploymentKey,
     ModelDeployFailed,
@@ -240,7 +241,7 @@ def deploy_model(
     wait: bool = False,
     timeout: float | None = 300.0,
     config: dict[str, str] | None = None,
-) -> Deployment:
+) -> Deployment[Any]:
     experiment = active_experiment()
     return _deploy_model_serving(
         family,
@@ -349,6 +350,7 @@ __all__ = [
     "delete_model",
     # Model serving
     "Deployment",
+    "DeploymentClient",
     "DeploymentConfig",
     "DeploymentKey",
     "ModelDeployFailed",

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/cortexgrid/model_serving/__init__.py

@@ -35,9 +35,12 @@ from cortexgrid.model_serving.deployment_key import (
     deployment_key,
 )
 from cortexgrid.model_serving.lifecycle import (
+    Deployment,
+    DeploymentClient,
     ModelDeployFailed,
     ModelNotDeployed,
     deploy_model,
+    list_deployed_models,
     redeploy_model,
     required_models,
     undeploy_model,
@@ -62,12 +65,10 @@ from cortexgrid.model_serving.serve_bundle import (
     upload_bundle,
 )
 from cortexgrid.model_serving.status import (
-    Deployment,
     ReplicaPlacement,
     ServingMessage,
     ServingStatus,
     deployment_config,
-    list_deployed_models,
     model_replica_placements,
     model_serving_messages,
     model_serving_status,

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/cortexgrid/model_serving/lifecycle.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import asyncio
+import importlib
 import time
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Generic, TypeVar
 
 from ray.serve.schema import ApplicationStatus
 
@@ -22,6 +25,8 @@ from cortexgrid.model_serving.deployment_records import (
     DeploymentRecord,
     delete_deployment_record,
     get_deployment_record,
+    key_of_record,
+    list_deployment_records,
     patch_deployment_record,
     put_deployment_record,
 )
@@ -29,9 +34,8 @@ from cortexgrid.model_serving.placement import ModelRequirements, vram_tiers
 from cortexgrid.model_serving.registry_tags import load_deploy_metadata
 from cortexgrid.model_serving.serve_bundle import BundleMetadata
 from cortexgrid.model_serving.status import (
+    PHASE_NOT_DEPLOYED,
     PHASE_PAUSED,
-    Deployment,
-    deployment_of_record,
     observed,
     replaced_bundle_fingerprint_until_rolled_out,
 )
@@ -101,16 +105,12 @@ def _wait_for_application_running(
     """`wait_for_model_serving` against a deadline already running. `timeout`
     only labels the TimeoutError."""
     while True:
-        app = get_serve_details().get("applications", {}).get(name)
-        if app is None:
-            raise ModelDeployFailed(f"Serve app {name!r} does not exist")
+        app = _application_still_able_to_serve(name)
         status = str(app.get("status", "(missing)"))
-        message = str(app.get("message", ""))
         if status == ApplicationStatus.RUNNING.value:
             return
-        if status == ApplicationStatus.DEPLOY_FAILED.value:
-            raise ModelDeployFailed(f"Serve app {name!r} DEPLOY_FAILED: {message}")
         if _past(deadline):
+            message = str(app.get("message", ""))
             raise TimeoutError(
                 f"Serve app {name!r} did not reach RUNNING within {timeout}s "
                 f"(last status={status!r}, message={message!r})"
@@ -118,6 +118,90 @@ def _wait_for_application_running(
         time.sleep(_SERVING_POLL_INTERVAL_S)
 
 
+def _application_still_able_to_serve(name: str) -> dict[str, Any]:
+    serve_details = get_serve_details()
+    applications = serve_details.get("applications", {})
+    app: dict[str, Any] | None = applications.get(name)
+    if app is None:
+        raise ModelDeployFailed(f"Serve app {name!r} does not exist")
+    status = app.get("status")
+    if status == ApplicationStatus.DEPLOY_FAILED.value:
+        message = app.get("message", "")
+        raise ModelDeployFailed(f"Serve app {name!r} DEPLOY_FAILED: {message}")
+    return app
+
+
+def _model_is_serving(key: DeploymentKey) -> bool:
+    name = app_name(key)
+    app = _application_still_able_to_serve(name)
+    status = app.get("status")
+    return status == ApplicationStatus.RUNNING.value
+
+
+@dataclass
+class DeploymentClient:
+    key: DeploymentKey
+    url: str
+
+    async def is_ready(self) -> bool:
+        serving = await asyncio.to_thread(_model_is_serving, self.key)
+        return serving
+
+
+ServeAppT = TypeVar("ServeAppT")
+
+
+@dataclass
+class Deployment(Generic[ServeAppT]):
+    """A scheduled Ray Serve app fronting a model. `phase` is the normalized
+    serving lifecycle phase (see `ServingStatus`); an app that appears in a
+    listing always exists, so its phase is never "not_deployed"."""
+
+    key: DeploymentKey
+    config: dict[str, str]
+    url: str
+    phase: str
+    bundle_fingerprint: str
+    replaced_bundle_fingerprint: str
+    experiment_name: str
+    class_import_path: str
+
+    def client(self) -> ServeAppT:
+        wait_for_model_serving(self.key)
+        deployment_client = self.client_async()
+        return deployment_client
+
+    def client_async(self) -> ServeAppT:
+        module_name, class_name = self.class_import_path.split(":")
+        serve_app_module = importlib.import_module(module_name)
+        serve_app = getattr(serve_app_module, class_name)
+        deployment_client: ServeAppT = serve_app.client(self)
+        return deployment_client
+
+
+def list_deployed_models() -> list[Deployment[Any]]:
+    """Return a Deployment for every model `deploy_model` put on Ray Serve
+    whose app the control plane last saw existing."""
+    return [
+        deployment_of_record(record)
+        for record in list_deployment_records()
+        if record["phase"] != PHASE_NOT_DEPLOYED
+    ]
+
+
+def deployment_of_record(record: DeploymentRecord) -> Deployment[Any]:
+    return Deployment(
+        key=key_of_record(record),
+        config=record["config"],
+        url=record["url"],
+        phase=record["phase"],
+        bundle_fingerprint=bundle_fingerprint_in_spec(record["spec"]),
+        replaced_bundle_fingerprint=record["replaced_bundle_fingerprint"],
+        experiment_name=record["experiment_name"],
+        class_import_path=record["spec"]["args"]["class_import_path"],
+    )
+
+
 def _clear_failed_application(
     key: DeploymentKey, timeout: float | None, deadline: float | None
 ) -> None:
@@ -225,11 +309,10 @@ def deploy_model(
     timeout: float | None = 300.0,
     config: dict[str, str] | None = None,
     experiment_name: str = "",
-) -> Deployment:
+) -> Deployment[Any]:
     """Schedule a Ray Serve app for a previously-saved model and return a
-    handle carrying its base URL. The caller (e.g. model-gateway) builds
-    whatever client the app's routes need - streaming, long timeouts, custom
-    request schemas - against that URL; cortexgrid imposes no traffic contract.
+    handle on it. The handle's `client()` is the client the serve-app's own
+    `client` factory makes for its routes; cortexgrid imposes no traffic contract.
 
     The serve-app class is pulled from the registry entry's tags `save_model`
     wrote at save time; the caller does not need to hold the class object.
@@ -332,10 +415,11 @@ def deploy_model(
         bundle_fingerprint=meta.fingerprint,
         replaced_bundle_fingerprint=observation["replaced_bundle_fingerprint"],
         experiment_name=experiment_name,
+        class_import_path=meta.class_import_path,
     )
 
 
-def redeploy_model(key: DeploymentKey) -> Deployment:
+def redeploy_model(key: DeploymentKey) -> Deployment[Any]:
     record = get_deployment_record(key)
     if record is None:
         raise ModelNotDeployed(f"{key} is not deployed")
@@ -349,14 +433,14 @@ def redeploy_model(key: DeploymentKey) -> Deployment:
     )
 
 
-def required_models(deployment: DeploymentKey) -> list[Deployment]:
+def required_models(deployment: DeploymentKey) -> list[Deployment[Any]]:
     _, requirements = load_deploy_metadata(
         deployment.family, deployment.suffix, deployment.run_name
     )
     return [_deployment_of(model) for model in requirements.models]
 
 
-def _deployment_of(model: DeploymentConfig) -> Deployment:
+def _deployment_of(model: DeploymentConfig) -> Deployment[Any]:
     key = deployment_key(model.family, model.suffix, model.run_name, model.config)
     record = get_deployment_record(key)
     if record is None:

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/cortexgrid/model_serving/serve_bundle.py

@@ -55,12 +55,19 @@ def build_bundle(cls: type) -> ServeBundle:
     is a subclass Ray defines in its own module, and on older Ray (e.g. 2.9) it
     reports that module as its own, so the class's source and import path would
     resolve to Ray instead of the serve-app. `cortexgrid.serve.ingress` leaves
-    the class unwrapped."""
+    the class unwrapped. Raises ValueError, too, for a class with no `client`
+    factory, which every `Deployment` of it makes its client with."""
     if any(klass.__module__.startswith("ray.serve") for klass in cls.__mro__):
         raise ValueError(
             f"{cls.__name__} is wrapped by ray.serve.ingress; decorate it with "
             "cortexgrid.serve.ingress instead (from cortexgrid import serve)"
         )
+    if not callable(getattr(cls, "client", None)):
+        raise ValueError(
+            f"{cls.__name__} has no client factory; give it a classmethod "
+            "client(cls, deployment: cortexgrid.Deployment) returning its "
+            "cortexgrid.DeploymentClient"
+        )
     entry_file = Path(inspect.getfile(cls)).resolve()
     serve_entry = Path(__file__).parent.with_name("_serve_entry.py")
     desc = bundle(entry_file).merge(bundle(serve_entry))

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/cortexgrid/model_serving/status.py

@@ -6,13 +6,9 @@ from typing import Any
 
 from ray.serve.schema import ApplicationStatus, ReplicaState
 
-from cortexgrid.model_serving.application_spec import (
-    app_name,
-    bundle_fingerprint_in_spec,
-)
+from cortexgrid.model_serving.application_spec import app_name
 from cortexgrid.model_serving.deployment_key import DeploymentKey
 from cortexgrid.model_serving.deployment_records import (
-    DeploymentRecord,
     get_deployment_record,
     key_of_record,
     list_deployment_records,
@@ -21,7 +17,7 @@ from cortexgrid.model_serving.deployment_records import (
 from cortexgrid.ray_util import get_serve_details
 
 
-_PHASE_NOT_DEPLOYED = "not_deployed"
+PHASE_NOT_DEPLOYED = "not_deployed"
 PHASE_PAUSED = "paused"
 _ROLLED_OUT_PHASES = ("running", PHASE_PAUSED)
 
@@ -44,49 +40,12 @@ def _serve_phase(raw_status: str) -> str:
     return _PHASE_BY_SERVE_STATUS.get(raw_status, "deploying")
 
 
-@dataclass
-class Deployment:
-    """A scheduled Ray Serve app fronting a model. `phase` is the normalized
-    serving lifecycle phase (see `ServingStatus`); an app that appears in a
-    listing always exists, so its phase is never "not_deployed"."""
-
-    key: DeploymentKey
-    config: dict[str, str]
-    url: str
-    phase: str
-    bundle_fingerprint: str
-    replaced_bundle_fingerprint: str
-    experiment_name: str
-
-
 def replaced_bundle_fingerprint_until_rolled_out(
     replaced_bundle_fingerprint: str, phase: str
 ) -> str:
     return "" if phase in _ROLLED_OUT_PHASES else replaced_bundle_fingerprint
 
 
-def list_deployed_models() -> list[Deployment]:
-    """Return a Deployment for every model `deploy_model` put on Ray Serve
-    whose app the control plane last saw existing."""
-    return [
-        deployment_of_record(record)
-        for record in list_deployment_records()
-        if record["phase"] != _PHASE_NOT_DEPLOYED
-    ]
-
-
-def deployment_of_record(record: DeploymentRecord) -> Deployment:
-    return Deployment(
-        key=key_of_record(record),
-        config=record["config"],
-        url=record["url"],
-        phase=record["phase"],
-        bundle_fingerprint=bundle_fingerprint_in_spec(record["spec"]),
-        replaced_bundle_fingerprint=record["replaced_bundle_fingerprint"],
-        experiment_name=record["experiment_name"],
-    )
-
-
 def deployment_config(key: DeploymentKey) -> dict[str, str]:
     record = get_deployment_record(key)
     if record is None:
@@ -134,8 +93,8 @@ def model_serving_status(key: DeploymentKey) -> ServingStatus:
     `cortexgrid.model_storage.model_registry_status`.
     """
     record = get_deployment_record(key)
-    if record is None or record["phase"] == _PHASE_NOT_DEPLOYED:
-        return ServingStatus(key, _PHASE_NOT_DEPLOYED, "", None)
+    if record is None or record["phase"] == PHASE_NOT_DEPLOYED:
+        return ServingStatus(key, PHASE_NOT_DEPLOYED, "", None)
     return ServingStatus(
         key=key,
         phase=record["phase"],
@@ -193,7 +152,7 @@ def observed(app: dict[str, Any] | None) -> dict[str, Any]:
     deployment record keeps; an app that does not exist reads as
     "not_deployed"."""
     if app is None:
-        return {"phase": _PHASE_NOT_DEPLOYED, "message": "", "replicas": []}
+        return {"phase": PHASE_NOT_DEPLOYED, "message": "", "replicas": []}
     raw = str(app.get("status", ""))
     return {
         "phase": _phase(app),

cortexgrid-0.3.17/cortexgrid/serve.py

@@ -0,0 +1,167 @@
+"""Declare a serve-app's HTTP ingress without importing Ray.
+
+    from cortexgrid import serve
+
+    @serve.ingress
+    class MyServeApp:
+        @serve.endpoint
+        async def predict(self, xs: list[float]) -> list[float]: ...
+
+Unlike `ray.serve.ingress`, it builds the FastAPI app from the class's
+`serve.endpoint` methods, and the class is left unwrapped: the FastAPI app
+and the client generated for it are only recorded on it, and
+`cortexgrid._serve_entry.build` applies Ray's ingress when it builds the Serve
+application on the cluster.
+
+Ray's decorator replaces the class with a wrapper subclass defined in
+ray/serve/api.py; older Ray (e.g. 2.9) leaves the wrapper's __module__ naming
+that module. Everything that locates a serve-app by its module - bundling its
+source, recording its import path - would then find Ray instead of the user's
+code. Deferring the wrap to the one place Serve needs it keeps the class
+locatable everywhere else (the laptop, Ray jobs, tests).
+"""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass
+from typing import Any, Callable, TypeVar, get_type_hints
+
+import httpx
+from fastapi import FastAPI
+from pydantic import TypeAdapter
+
+from cortexgrid.model_serving.lifecycle import Deployment, DeploymentClient
+
+
+_T = TypeVar("_T", bound=type)
+_Method = TypeVar("_Method", bound=Callable[..., Any])
+
+_INGRESS_APP_ATTR = "__cortexgrid_ingress_app__"
+_ENDPOINT_ATTR = "__cortexgrid_endpoint__"
+
+
+def endpoint(method: _Method) -> _Method:
+    setattr(method, _ENDPOINT_ATTR, True)
+    return method
+
+
+def ingress(cls: _T) -> _T:
+    """Mark a serve-app class as fronted by a FastAPI app serving its
+    endpoints, and give it the client that calls them. Returns the class
+    itself, unwrapped."""
+    app = FastAPI()
+    calls: dict[str, Any] = {"__module__": cls.__module__}
+    methods = inspect.getmembers(cls, inspect.isfunction)
+    for name, method in methods:
+        if getattr(method, _ENDPOINT_ATTR, False):
+            marshalling = _EndpointMarshalling.of(method)
+            route = _route_of(method, marshalling)
+            app.add_api_route(f"/{name}", route, methods=["POST"])
+            calls[name] = _call_of(name, method, marshalling)
+    client = type(f"{cls.__name__}Client", (_EndpointsClient,), calls)
+    setattr(cls, _INGRESS_APP_ATTR, app)
+    setattr(cls, "client", client)
+    return cls
+
+
+def ingress_app(cls: type) -> Any | None:
+    """The app `cls` was marked with by `ingress`, or None if it was not."""
+    return getattr(cls, _INGRESS_APP_ATTR, None)
+
+
+class _EndpointsClient(DeploymentClient):
+    def __init__(self, deployment: Deployment[Any]) -> None:
+        super().__init__(key=deployment.key, url=deployment.url)
+
+
+@dataclass(frozen=True)
+class _EndpointMarshalling:
+    signature_without_self: inspect.Signature
+    parameters: dict[str, TypeAdapter[Any]]
+    answer: TypeAdapter[Any]
+
+    @classmethod
+    def of(cls, method: Callable[..., Any]) -> _EndpointMarshalling:
+        signature = inspect.signature(method)
+        parameter_values = signature.parameters.values()
+        parameters_in_order = list(parameter_values)
+        signature_without_self = signature.replace(parameters=parameters_in_order[1:])
+        hints = get_type_hints(method)
+        answer_hint = hints.pop("return")
+        answer = TypeAdapter(answer_hint)
+        parameters = {name: TypeAdapter(hint) for name, hint in hints.items()}
+        return cls(signature_without_self, parameters, answer)
+
+    def arguments_to_json(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        bound = self.signature_without_self.bind(*args, **kwargs)
+        return {
+            name: self.parameters[name].dump_python(value, mode="json")
+            for name, value in bound.arguments.items()
+        }
+
+    def arguments_from_json(self, body: dict[str, Any]) -> dict[str, Any]:
+        return {
+            name: parameter.validate_python(body[name])
+            for name, parameter in self.parameters.items()
+            if name in body
+        }
+
+    def answer_to_json(self, answer: Any) -> Any:
+        return self.answer.dump_python(answer, mode="json")
+
+    def answer_from_json(self, answered: Any) -> Any:
+        return self.answer.validate_python(answered)
+
+
+def _route_of(
+    method: Callable[..., Any], marshalling: _EndpointMarshalling
+) -> Callable[..., Any]:
+    if inspect.iscoroutinefunction(method):
+
+        async def route(self: Any, body: dict[str, Any]) -> Any:
+            arguments = marshalling.arguments_from_json(body)
+            answer = await method(self, **arguments)
+            answered = marshalling.answer_to_json(answer)
+            return answered
+
+    else:
+
+        def route(self: Any, body: dict[str, Any]) -> Any:
+            arguments = marshalling.arguments_from_json(body)
+            answer = method(self, **arguments)
+            answered = marshalling.answer_to_json(answer)
+            return answered
+
+    route.__name__ = method.__name__
+    route.__qualname__ = method.__qualname__
+    return route
+
+
+def _call_of(
+    name: str, method: Callable[..., Any], marshalling: _EndpointMarshalling
+) -> Callable[..., Any]:
+    if inspect.iscoroutinefunction(method):
+
+        async def call(self: _EndpointsClient, *args: Any, **kwargs: Any) -> Any:
+            body = marshalling.arguments_to_json(*args, **kwargs)
+            async with httpx.AsyncClient(timeout=None) as client:
+                response = await client.post(f"{self.url}/{name}", json=body)
+            response.raise_for_status()
+            answered = response.json()
+            answer = marshalling.answer_from_json(answered)
+            return answer
+
+    else:
+
+        def call(self: _EndpointsClient, *args: Any, **kwargs: Any) -> Any:
+            body = marshalling.arguments_to_json(*args, **kwargs)
+            with httpx.Client(timeout=None) as client:
+                response = client.post(f"{self.url}/{name}", json=body)
+            response.raise_for_status()
+            answered = response.json()
+            answer = marshalling.answer_from_json(answered)
+            return answer
+
+    call.__name__ = name
+    return call

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/docs/cortexgrid/README.md

@@ -177,31 +177,28 @@ s3_client = cortexgrid.get_s3_client()           # boto3 S3 client
 
 #### Model registry and serving
 
-Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class fronted by a FastAPI app, marked with cortexgrid's `serve.ingress` (not Ray's):
+Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class marked with cortexgrid's `serve.ingress` (not Ray's), whose `serve.endpoint` methods it serves and whose client it generates:
 
 ```python
 from cortexgrid import serve
-from fastapi import FastAPI
 
-app = FastAPI()
-
-@serve.ingress(app)
+@serve.ingress
 class MyServeApp:
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
         )
 
-    @app.post("/complete")
-    async def complete(self, body: dict): ...
+    @serve.endpoint
+    async def complete(self, prompt: str) -> str: ...
 
 saved = cortexgrid.save_model(
     weights_dir, MyServeApp, family="qwen", suffix="instruct",
     # What one replica needs; the model is deployed only on a host that has it.
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
-deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
-print(deployed.url)
+deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
+model: MyServeApp = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.15 → cortexgrid-0.3.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.3.15"
+version = "0.3.17"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"
@@ -26,6 +26,8 @@ dependencies = [
     "pydotenv>=0.0.7",
     "pydantic>=2.13.3",
     "requests>=2.31",
+    "fastapi>=0.115,<1",
+    "httpx>=0.27",
 ]
 
 [project.urls]

cortexgrid-0.3.15/cortexgrid/serve.py

@@ -1,43 +0,0 @@
-"""Declare a serve-app's HTTP ingress without importing Ray.
-
-    from cortexgrid import serve
-
-    @serve.ingress(app)
-    class MyServeApp: ...
-
-Same shape as `ray.serve.ingress`, but the class is left exactly as written: the
-FastAPI app is only recorded on it, and `cortexgrid._serve_entry.build` applies
-Ray's ingress when it builds the Serve application on the cluster.
-
-Ray's decorator replaces the class with a wrapper subclass defined in
-ray/serve/api.py; older Ray (e.g. 2.9) leaves the wrapper's __module__ naming
-that module. Everything that locates a serve-app by its module - bundling its
-source, recording its import path - would then find Ray instead of the user's
-code. Deferring the wrap to the one place Serve needs it keeps the class
-locatable everywhere else (the laptop, Ray jobs, tests).
-"""
-
-from __future__ import annotations
-
-from typing import Any, Callable, TypeVar
-
-
-_T = TypeVar("_T", bound=type)
-
-_INGRESS_APP_ATTR = "__cortexgrid_ingress_app__"
-
-
-def ingress(app: Any) -> Callable[[_T], _T]:
-    """Mark a serve-app class as fronted by the ASGI `app` (e.g. a FastAPI
-    instance). Returns the class itself, unwrapped."""
-
-    def decorator(cls: _T) -> _T:
-        setattr(cls, _INGRESS_APP_ATTR, app)
-        return cls
-
-    return decorator
-
-
-def ingress_app(cls: type) -> Any | None:
-    """The app `cls` was marked with by `ingress`, or None if it was not."""
-    return getattr(cls, _INGRESS_APP_ATTR, None)