cortexgrid 0.3.16__tar.gz → 0.3.17__tar.gz

{cortexgrid-0.3.16 → cortexgrid-0.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.16
+Version: 0.3.17
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
@@ -11,7 +11,9 @@ Requires-Python: <3.12,>=3.11
 Requires-Dist: boto3>=1.34
 Requires-Dist: cloudpickle>=3.0
 Requires-Dist: fabric>=3.2.3
+Requires-Dist: fastapi<1,>=0.115
 Requires-Dist: haikunator>=2.1.0
+Requires-Dist: httpx>=0.27
 Requires-Dist: mlflow<4,>=3.11
 Requires-Dist: packaging>=24
 Requires-Dist: pip>=23.0
@@ -205,30 +207,20 @@ s3_client = cortexgrid.get_s3_client()           # boto3 S3 client
 
 #### Model registry and serving
 
-Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class fronted by a FastAPI app, marked with cortexgrid's `serve.ingress` (not Ray's):
+Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class marked with cortexgrid's `serve.ingress` (not Ray's), whose `serve.endpoint` methods it serves and whose client it generates:
 
 ```python
 from cortexgrid import serve
-from fastapi import FastAPI
 
-app = FastAPI()
-
-class MyClient(cortexgrid.DeploymentClient):
-    def complete(self, prompt: str) -> str: ...
-
-@serve.ingress(app)
+@serve.ingress
 class MyServeApp:
-    @classmethod
-    def client(cls, deployment: cortexgrid.Deployment[MyClient]) -> MyClient:
-        return MyClient(key=deployment.key, url=deployment.url)
-
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
         )
 
-    @app.post("/complete")
-    async def complete(self, body: dict): ...
+    @serve.endpoint
+    async def complete(self, prompt: str) -> str: ...
 
 saved = cortexgrid.save_model(
     weights_dir, MyServeApp, family="qwen", suffix="instruct",
@@ -236,7 +228,7 @@ saved = cortexgrid.save_model(
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
 deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
-model = deployed.client()   # MyServeApp's client, once the app serves
+model: MyServeApp = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.16 → cortexgrid-0.3.17}/cortexgrid/model_serving/lifecycle.py

@@ -148,11 +148,11 @@ class DeploymentClient:
         return serving
 
 
-DeploymentClientT = TypeVar("DeploymentClientT", bound=DeploymentClient)
+ServeAppT = TypeVar("ServeAppT")
 
 
 @dataclass
-class Deployment(Generic[DeploymentClientT]):
+class Deployment(Generic[ServeAppT]):
     """A scheduled Ray Serve app fronting a model. `phase` is the normalized
     serving lifecycle phase (see `ServingStatus`); an app that appears in a
     listing always exists, so its phase is never "not_deployed"."""
@@ -166,16 +166,16 @@ class Deployment(Generic[DeploymentClientT]):
     experiment_name: str
     class_import_path: str
 
-    def client(self) -> DeploymentClientT:
+    def client(self) -> ServeAppT:
         wait_for_model_serving(self.key)
         deployment_client = self.client_async()
         return deployment_client
 
-    def client_async(self) -> DeploymentClientT:
+    def client_async(self) -> ServeAppT:
         module_name, class_name = self.class_import_path.split(":")
         serve_app_module = importlib.import_module(module_name)
         serve_app = getattr(serve_app_module, class_name)
-        deployment_client: DeploymentClientT = serve_app.client(self)
+        deployment_client: ServeAppT = serve_app.client(self)
         return deployment_client
 
 

cortexgrid-0.3.17/cortexgrid/serve.py

@@ -0,0 +1,167 @@
+"""Declare a serve-app's HTTP ingress without importing Ray.
+
+    from cortexgrid import serve
+
+    @serve.ingress
+    class MyServeApp:
+        @serve.endpoint
+        async def predict(self, xs: list[float]) -> list[float]: ...
+
+Unlike `ray.serve.ingress`, it builds the FastAPI app from the class's
+`serve.endpoint` methods, and the class is left unwrapped: the FastAPI app
+and the client generated for it are only recorded on it, and
+`cortexgrid._serve_entry.build` applies Ray's ingress when it builds the Serve
+application on the cluster.
+
+Ray's decorator replaces the class with a wrapper subclass defined in
+ray/serve/api.py; older Ray (e.g. 2.9) leaves the wrapper's __module__ naming
+that module. Everything that locates a serve-app by its module - bundling its
+source, recording its import path - would then find Ray instead of the user's
+code. Deferring the wrap to the one place Serve needs it keeps the class
+locatable everywhere else (the laptop, Ray jobs, tests).
+"""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass
+from typing import Any, Callable, TypeVar, get_type_hints
+
+import httpx
+from fastapi import FastAPI
+from pydantic import TypeAdapter
+
+from cortexgrid.model_serving.lifecycle import Deployment, DeploymentClient
+
+
+_T = TypeVar("_T", bound=type)
+_Method = TypeVar("_Method", bound=Callable[..., Any])
+
+_INGRESS_APP_ATTR = "__cortexgrid_ingress_app__"
+_ENDPOINT_ATTR = "__cortexgrid_endpoint__"
+
+
+def endpoint(method: _Method) -> _Method:
+    setattr(method, _ENDPOINT_ATTR, True)
+    return method
+
+
+def ingress(cls: _T) -> _T:
+    """Mark a serve-app class as fronted by a FastAPI app serving its
+    endpoints, and give it the client that calls them. Returns the class
+    itself, unwrapped."""
+    app = FastAPI()
+    calls: dict[str, Any] = {"__module__": cls.__module__}
+    methods = inspect.getmembers(cls, inspect.isfunction)
+    for name, method in methods:
+        if getattr(method, _ENDPOINT_ATTR, False):
+            marshalling = _EndpointMarshalling.of(method)
+            route = _route_of(method, marshalling)
+            app.add_api_route(f"/{name}", route, methods=["POST"])
+            calls[name] = _call_of(name, method, marshalling)
+    client = type(f"{cls.__name__}Client", (_EndpointsClient,), calls)
+    setattr(cls, _INGRESS_APP_ATTR, app)
+    setattr(cls, "client", client)
+    return cls
+
+
+def ingress_app(cls: type) -> Any | None:
+    """The app `cls` was marked with by `ingress`, or None if it was not."""
+    return getattr(cls, _INGRESS_APP_ATTR, None)
+
+
+class _EndpointsClient(DeploymentClient):
+    def __init__(self, deployment: Deployment[Any]) -> None:
+        super().__init__(key=deployment.key, url=deployment.url)
+
+
+@dataclass(frozen=True)
+class _EndpointMarshalling:
+    signature_without_self: inspect.Signature
+    parameters: dict[str, TypeAdapter[Any]]
+    answer: TypeAdapter[Any]
+
+    @classmethod
+    def of(cls, method: Callable[..., Any]) -> _EndpointMarshalling:
+        signature = inspect.signature(method)
+        parameter_values = signature.parameters.values()
+        parameters_in_order = list(parameter_values)
+        signature_without_self = signature.replace(parameters=parameters_in_order[1:])
+        hints = get_type_hints(method)
+        answer_hint = hints.pop("return")
+        answer = TypeAdapter(answer_hint)
+        parameters = {name: TypeAdapter(hint) for name, hint in hints.items()}
+        return cls(signature_without_self, parameters, answer)
+
+    def arguments_to_json(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        bound = self.signature_without_self.bind(*args, **kwargs)
+        return {
+            name: self.parameters[name].dump_python(value, mode="json")
+            for name, value in bound.arguments.items()
+        }
+
+    def arguments_from_json(self, body: dict[str, Any]) -> dict[str, Any]:
+        return {
+            name: parameter.validate_python(body[name])
+            for name, parameter in self.parameters.items()
+            if name in body
+        }
+
+    def answer_to_json(self, answer: Any) -> Any:
+        return self.answer.dump_python(answer, mode="json")
+
+    def answer_from_json(self, answered: Any) -> Any:
+        return self.answer.validate_python(answered)
+
+
+def _route_of(
+    method: Callable[..., Any], marshalling: _EndpointMarshalling
+) -> Callable[..., Any]:
+    if inspect.iscoroutinefunction(method):
+
+        async def route(self: Any, body: dict[str, Any]) -> Any:
+            arguments = marshalling.arguments_from_json(body)
+            answer = await method(self, **arguments)
+            answered = marshalling.answer_to_json(answer)
+            return answered
+
+    else:
+
+        def route(self: Any, body: dict[str, Any]) -> Any:
+            arguments = marshalling.arguments_from_json(body)
+            answer = method(self, **arguments)
+            answered = marshalling.answer_to_json(answer)
+            return answered
+
+    route.__name__ = method.__name__
+    route.__qualname__ = method.__qualname__
+    return route
+
+
+def _call_of(
+    name: str, method: Callable[..., Any], marshalling: _EndpointMarshalling
+) -> Callable[..., Any]:
+    if inspect.iscoroutinefunction(method):
+
+        async def call(self: _EndpointsClient, *args: Any, **kwargs: Any) -> Any:
+            body = marshalling.arguments_to_json(*args, **kwargs)
+            async with httpx.AsyncClient(timeout=None) as client:
+                response = await client.post(f"{self.url}/{name}", json=body)
+            response.raise_for_status()
+            answered = response.json()
+            answer = marshalling.answer_from_json(answered)
+            return answer
+
+    else:
+
+        def call(self: _EndpointsClient, *args: Any, **kwargs: Any) -> Any:
+            body = marshalling.arguments_to_json(*args, **kwargs)
+            with httpx.Client(timeout=None) as client:
+                response = client.post(f"{self.url}/{name}", json=body)
+            response.raise_for_status()
+            answered = response.json()
+            answer = marshalling.answer_from_json(answered)
+            return answer
+
+    call.__name__ = name
+    return call

{cortexgrid-0.3.16 → cortexgrid-0.3.17}/docs/cortexgrid/README.md

@@ -177,30 +177,20 @@ s3_client = cortexgrid.get_s3_client()           # boto3 S3 client
 
 #### Model registry and serving
 
-Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class fronted by a FastAPI app, marked with cortexgrid's `serve.ingress` (not Ray's):
+Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application. A serve-app is a class marked with cortexgrid's `serve.ingress` (not Ray's), whose `serve.endpoint` methods it serves and whose client it generates:
 
 ```python
 from cortexgrid import serve
-from fastapi import FastAPI
 
-app = FastAPI()
-
-class MyClient(cortexgrid.DeploymentClient):
-    def complete(self, prompt: str) -> str: ...
-
-@serve.ingress(app)
+@serve.ingress
 class MyServeApp:
-    @classmethod
-    def client(cls, deployment: cortexgrid.Deployment[MyClient]) -> MyClient:
-        return MyClient(key=deployment.key, url=deployment.url)
-
     def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
         self._weights_dir = cortexgrid.load_model(
             deployment.family, deployment.suffix, deployment.run_name
         )
 
-    @app.post("/complete")
-    async def complete(self, body: dict): ...
+    @serve.endpoint
+    async def complete(self, prompt: str) -> str: ...
 
 saved = cortexgrid.save_model(
     weights_dir, MyServeApp, family="qwen", suffix="instruct",
@@ -208,7 +198,7 @@ saved = cortexgrid.save_model(
     requirements=cortexgrid.ModelRequirements(num_gpus=1, ram_gb=8, vram_gb=16),
 )
 deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name)
-model = deployed.client()   # MyServeApp's client, once the app serves
+model: MyServeApp = deployed.client()   # MyServeApp's client, once the app serves
 ```
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.

{cortexgrid-0.3.16 → cortexgrid-0.3.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cortexgrid"
-version = "0.3.16"
+version = "0.3.17"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"
@@ -26,6 +26,8 @@ dependencies = [
     "pydotenv>=0.0.7",
     "pydantic>=2.13.3",
     "requests>=2.31",
+    "fastapi>=0.115,<1",
+    "httpx>=0.27",
 ]
 
 [project.urls]

cortexgrid-0.3.16/cortexgrid/serve.py

@@ -1,43 +0,0 @@
-"""Declare a serve-app's HTTP ingress without importing Ray.
-
-    from cortexgrid import serve
-
-    @serve.ingress(app)
-    class MyServeApp: ...
-
-Same shape as `ray.serve.ingress`, but the class is left exactly as written: the
-FastAPI app is only recorded on it, and `cortexgrid._serve_entry.build` applies
-Ray's ingress when it builds the Serve application on the cluster.
-
-Ray's decorator replaces the class with a wrapper subclass defined in
-ray/serve/api.py; older Ray (e.g. 2.9) leaves the wrapper's __module__ naming
-that module. Everything that locates a serve-app by its module - bundling its
-source, recording its import path - would then find Ray instead of the user's
-code. Deferring the wrap to the one place Serve needs it keeps the class
-locatable everywhere else (the laptop, Ray jobs, tests).
-"""
-
-from __future__ import annotations
-
-from typing import Any, Callable, TypeVar
-
-
-_T = TypeVar("_T", bound=type)
-
-_INGRESS_APP_ATTR = "__cortexgrid_ingress_app__"
-
-
-def ingress(app: Any) -> Callable[[_T], _T]:
-    """Mark a serve-app class as fronted by the ASGI `app` (e.g. a FastAPI
-    instance). Returns the class itself, unwrapped."""
-
-    def decorator(cls: _T) -> _T:
-        setattr(cls, _INGRESS_APP_ATTR, app)
-        return cls
-
-    return decorator
-
-
-def ingress_app(cls: type) -> Any | None:
-    """The app `cls` was marked with by `ingress`, or None if it was not."""
-    return getattr(cls, _INGRESS_APP_ATTR, None)