cortexgrid 0.2.92__tar.gz → 0.2.93__tar.gz

{cortexgrid-0.2.92 → cortexgrid-0.2.93}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.2.92
+Version: 0.2.93
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
 Project-URL: Documentation, https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/README.md
 License-Expression: Apache-2.0
 License-File: LICENSE
-Requires-Python: >=3.11
+Requires-Python: <3.12,>=3.11
 Requires-Dist: boto3>=1.34
 Requires-Dist: cloudpickle>=3.0
 Requires-Dist: fabric>=3.2.3

{cortexgrid-0.2.92 → cortexgrid-0.2.93}/cortexgrid/_serve_entry.py

@@ -3,7 +3,8 @@
 Ray Serve's REST `import_path` resolves to `cortexgrid._serve_entry:build`.
 On the cluster replica, `build` imports the serve-app class bundled at
 `save_model` time (its import path was stored as an MLflow tag), applies Ray's
-ingress with the app it was marked with by `cortexgrid.serve.ingress`, reads its
+ingress with the app it was marked with by `cortexgrid.serve.ingress` (again on
+each replica, see `_IngressOnReplica`), reads its
 `num_gpus`/`num_replicas` class attributes for actor placement, wraps it as a
 Ray Serve deployment, and binds it with the (family, suffix, run_name)
 identifiers.
@@ -31,6 +32,33 @@ from ray.serve.deployment import Application
 from cortexgrid.serve import ingress_app
 
 
+# Requests one replica handles at once before Ray queues the rest.
+_MAX_ONGOING_REQUESTS = 100
+
+class _IngressOnReplica:
+    """Mixin that re-applies Ray's ingress to `_serve_app` in the replica's own
+    process, as Ray creates the replica instance.
+
+    Ray's ingress rewrites the signature of each route method, in place, so
+    FastAPI injects the replica instance as `self`. `build` applies it in the
+    build process only. The replica imports the serve-app's module afresh, and
+    its route methods carry no rewrite. FastAPI < 0.137 analysed routes once,
+    in the build process, and the replica received the result. FastAPI >= 0.137
+    analyses them in the replica on the first request, and without the rewrite
+    reads `self` as a required query parameter (HTTP 422).
+
+    Hooked on `__new__`, not `__init__`: Ray calls `__new__` alone, before the
+    serve-app's `__init__`, whether that is sync or async."""
+
+    _serve_app: type
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> Any:
+        # Applied for its side effect on the route methods; the wrapper it
+        # returns is not needed.
+        serve.ingress(ingress_app(cls._serve_app))(cls._serve_app)
+        return super().__new__(cls)
+
+
 def build(args: dict[str, Any]) -> Application:
     module_name, class_name = args["class_import_path"].split(":")
     serve_app = getattr(importlib.import_module(module_name), class_name)
@@ -38,10 +66,18 @@ def build(args: dict[str, Any]) -> Application:
     # mark and are deployed as they are.
     app = ingress_app(serve_app)
     if app is not None:
-        serve_app = serve.ingress(app)(serve_app)
+        on_replica = type(
+            serve_app.__name__,
+            (_IngressOnReplica, serve_app),
+            {"_serve_app": serve_app},
+        )
+        serve_app = serve.ingress(app)(on_replica)
     num_gpus = getattr(serve_app, "num_gpus", 0)
     num_replicas = getattr(serve_app, "num_replicas", 1)
     return serve.deployment(serve_app).options(
         num_replicas=num_replicas,
+        # Ray 2.32 lowered the default from 100 to 5; keep what serve-apps
+        # had on Ray 2.9.
+        max_ongoing_requests=_MAX_ONGOING_REQUESTS,
         ray_actor_options={"num_gpus": num_gpus},
     ).bind(args["family"], args["suffix"], args["run_name"])

{cortexgrid-0.2.92 → cortexgrid-0.2.93}/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "cortexgrid"
-version = "0.2.92"
+version = "0.2.93"
 description = "Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3"
 readme = "docs/cortexgrid/README.md"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.12"
 dependencies = [
     "ray[default]>=2.9,<3",
     "mlflow>=3.11,<4",