cortexgrid 0.3.11__tar.gz → 0.3.13__tar.gz

{cortexgrid-0.3.11 → cortexgrid-0.3.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cortexgrid
-Version: 0.3.11
+Version: 0.3.13
 Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
 Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
 Project-URL: Repository, https://github.com/robodatalab/cortexgrid
@@ -215,8 +215,10 @@ app = FastAPI()
 
 @serve.ingress(app)
 class MyServeApp:
-    def __init__(self, family: str, suffix: str, run_name: str) -> None:
-        self._weights_dir = cortexgrid.load_model(family, suffix, run_name)
+    def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
+        self._weights_dir = cortexgrid.load_model(
+            deployment.family, deployment.suffix, deployment.run_name
+        )
 
     @app.post("/complete")
     async def complete(self, body: dict): ...
@@ -232,11 +234,11 @@ print(deployed.url)
 
 The requirements are part of the model, not of the serve-app class: GPUs, RAM and VRAM (GiB, 0 meaning no requirement) are stored with it and matched against what the cluster's hosts have free. Among the hosts that fit, the model goes to the **smallest GPU** that does, so a 4 GiB model does not occupy a 128 GiB card a bigger one needs; it moves up only once the smaller cards are full. `num_gpus` may be a fraction (`0.25`) to share one card between models, in which case `vram_gb` is what keeps them from overcommitting it. Correct them later with `cortexgrid.set_model_requirements(family, suffix, run_name, requirements)` or on the model card in the dashboard; `cortexgrid.deploy_model(..., num_replicas=2)` chooses how many copies to run.
 
-Anything else the serve-app has to know about the model - which model a provider should be asked for, an endpoint, the name of a secret to read - goes in a free-form string mapping on the same entry: `save_model(..., config={"model": "claude-opus-5"})`. The serve-app reads it in `__init__` with `cortexgrid.model_config(family, suffix, run_name)`; `cortexgrid.set_model_config(family, suffix, run_name, config)` or the model card replaces it. cortexgrid stores the mapping without interpreting it, and a tag is readable by anyone with registry access, so a credential belongs in `set_secret` with only its name in the config.
+Anything else the serve-app has to know about the model - which model a provider should be asked for, an endpoint, the name of a secret to read - goes in a free-form string mapping on the same entry: `save_model(..., config={"model": "claude-opus-5"})`. The serve-app reads it in `__init__` with `cortexgrid.model_config(deployment)`; `cortexgrid.set_model_config(family, suffix, run_name, config)` or the model card replaces it. How one deployment serves the model - thinking on or off, say - is that deployment's own: `deploy_model(..., config={"thinking": "false"})` deploys the model once per distinct config, each deployment named by the `DeploymentKey` on the returned `Deployment`, and `model_config` lays the deployment's config over the model's. cortexgrid stores the mapping without interpreting it, and a tag is readable by anyone with registry access, so a credential belongs in `set_secret` with only its name in the config.
 
 `save_model` saves a new copy under every run - meant for weights the run produced (e.g. a fine-tune). For a model produced elsewhere (e.g. a pretrained base model), `cortexgrid.import_model(source, MyServeApp, family, suffix)` uploads it once under `run_name=cortexgrid.IMPORTED` and on later runs only re-bundles `MyServeApp` if its code changed; deploy it with `deploy_model(family, suffix, cortexgrid.IMPORTED)`. A model with no weights to stage - one behind a provider's API, e.g. Gemini or OpenAI - is registered the same way by `cortexgrid.register_model(MyServeApp, family, suffix, config=...)`: same key, same reuse, only the bundle stored. [Serving a hosted-API model](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md#serving-a-hosted-api-model) walks through one end to end - serve-app, API key, registration, deploy.
 
-`save_model` is synchronous (registry lifecycle: `uploading` -> `ready`); `deploy_model` schedules the serving lifecycle (`deploying` -> `running`). With `wait=True` a failed deploy raises `cortexgrid.ModelDeployFailed`; `cortexgrid.wait_for_model_serving(family, suffix, run_name, timeout=...)` waits on a deploy started elsewhere, and re-deploying a failed model retries it from scratch. See [model-serving.md](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md) for both lifecycles end to end - upload/deploy/undeploy/delete, status queries (`model_registry_status`, `model_serving_status`), and error handling.
+`save_model` is synchronous (registry lifecycle: `uploading` -> `ready`); `deploy_model` schedules the serving lifecycle (`deploying` -> `running`). With `wait=True` a failed deploy raises `cortexgrid.ModelDeployFailed`; `cortexgrid.wait_for_model_serving(deployment.key, timeout=...)` waits on a deploy started elsewhere, and re-deploying a failed model retries it from scratch. See [model-serving.md](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md) for both lifecycles end to end - upload/deploy/undeploy/delete, status queries (`model_registry_status`, `model_serving_status`), and error handling.
 
 ### API reference
 

{cortexgrid-0.3.11 → cortexgrid-0.3.13}/cortexgrid/__init__.py

@@ -101,12 +101,16 @@ from cortexgrid.model_storage import register_model as _register_model_storage
 from cortexgrid.model_storage import save_model as _save_model_storage
 from cortexgrid.model_serving import (
     Deployment,
+    DeploymentConfig,
+    DeploymentKey,
     ModelDeployFailed,
+    ModelNotDeployed,
     ModelRequirements,
     ServingStatus,
     deploy_model,
     list_deployed_models,
     model_serving_status,
+    redeploy_model,
     undeploy_model,
     wait_for_model_serving,
 )
@@ -308,9 +312,13 @@ __all__ = [
     "delete_model",
     # Model serving
     "Deployment",
+    "DeploymentConfig",
+    "DeploymentKey",
     "ModelDeployFailed",
+    "ModelNotDeployed",
     "ServingStatus",
     "deploy_model",
+    "redeploy_model",
     "wait_for_model_serving",
     "model_serving_status",
     "undeploy_model",

{cortexgrid-0.3.11 → cortexgrid-0.3.13}/cortexgrid/_serve_entry.py

@@ -6,13 +6,14 @@ On the cluster replica, `build` imports the serve-app class bundled at
 ingress with the app it was marked with by `cortexgrid.serve.ingress` (again on
 each replica, see `_IngressOnReplica`), wraps it as a Ray Serve deployment
 with the replica count and Ray resource requests `deploy_model` derived from
-the model's requirements, and binds it with the (family, suffix, run_name)
-identifiers.
+the model's requirements, and binds it with its deployment's
+`cortexgrid.DeploymentKey`.
 
 The serve-app owns everything about traffic: its own routes, request schemas,
 streaming, and timeouts. cortexgrid does not interpose a request/response
-contract - it only schedules the app and hands it the identifiers it needs to
-fetch its own weights via `cortexgrid.load_model`.
+contract - it only schedules the app and hands it the key it needs to fetch
+its own weights via `cortexgrid.load_model` and its settings via
+`cortexgrid.model_config`.
 
 The serve-app declares no resources: the hardware a replica needs belongs to
 the model and is stored in the registry (`cortexgrid.ModelRequirements`), and
@@ -28,6 +29,7 @@ from ray import serve
 from ray.serve.deployment import Application
 
 from cortexgrid._model_scheduler import model_autoscaling_config
+from cortexgrid.model_serving.deployment_key import DeploymentKey
 from cortexgrid.serve import ingress_app
 
 
@@ -80,4 +82,11 @@ def build(args: dict[str, Any]) -> Application:
         # had on Ray 2.9.
         max_ongoing_requests=_MAX_ONGOING_REQUESTS,
         ray_actor_options=args.get("ray_actor_options", {}),
-    ).bind(args["family"], args["suffix"], args["run_name"])
+    ).bind(
+        DeploymentKey(
+            args["family"],
+            args["suffix"],
+            args["run_name"],
+            args.get("config_fingerprint", ""),
+        )
+    )

{cortexgrid-0.3.11 → cortexgrid-0.3.13}/cortexgrid/jobs.py

@@ -29,7 +29,7 @@ from pydantic import BaseModel, ConfigDict
 log = logging.getLogger(__name__)
 
 _JOB_POLL_INTERVAL_S = 5.0
-_TERMINAL_JOB_STATES = (JobStatus.FINISHED, JobStatus.FAILED, JobStatus.STOPPED)
+TERMINAL_JOB_STATES = (JobStatus.FINISHED, JobStatus.FAILED, JobStatus.STOPPED)
 
 
 class JobFailed(RuntimeError):
@@ -379,7 +379,7 @@ def wait_for_job_result(run_id: str, job_id: str, timeout: float | None = None)
             )
         ray_job_id = lifecycle.get_ray_job_id()
         status = get_ray_job_status(ray_job_id)
-        if status in _TERMINAL_JOB_STATES:
+        if status in TERMINAL_JOB_STATES:
             try:
                 result = JobResult.load_from_mlflow(run_id, job_id)
             except FileNotFoundError:
@@ -413,7 +413,7 @@ class JobFuture:
 
     def done(self) -> bool:
         """True once Ray reports the job finished, failed or stopped."""
-        return self.status() in _TERMINAL_JOB_STATES
+        return self.status() in TERMINAL_JOB_STATES
 
     def result(self, timeout: float | None = None) -> Any:
         """Block until the job finishes and return its function's value.

cortexgrid-0.3.13/cortexgrid/model_serving/__init__.py

@@ -0,0 +1,73 @@
+"""Cortexgrid wrappers around Ray Serve.
+
+Caller stays HTTP-only: deploy/undeploy/list talk to the Ray dashboard's
+declarative `/api/serve/applications/` endpoint via [cortexgrid.ray_util],
+never `ray.init`. The deployment class is bundled at `save_model` time, zipped,
+uploaded to MinIO under
+`serve-bundles/<run_name>/<family>__<suffix>/<fingerprint>.zip`, and
+referenced via `runtime_env.working_dir` so Ray workers fetch it from there.
+The bundle URL, class import path, pip list, and fingerprint are persisted as
+tags on the model's registry entry so `deploy_model` can find them later
+without the caller holding the class object. So are the model's
+`ModelRequirements`, which `deploy_model` turns into the replica's Ray resource
+requests.
+
+Every deployment `deploy_model` puts on Ray Serve gets a record with the jobs
+control plane, keyed by its `DeploymentKey` - the model's (family, suffix,
+run_name) plus a fingerprint of the config it was deployed with: that config,
+the spec it PUT, plus the phase, message and replica
+placements the control plane last observed (`observe_deployments`, run on
+every poll cycle). Listings and status reads come from those records; waits
+ask the Serve controller directly.
+
+Naming: the Ray Serve application is named "<family>__<suffix>__<run_name>",
+followed by "__<config_fingerprint>" for a deployment given a config.
+This relies on family/suffix/run_name not containing the literal "__".
+
+See [docs/cortexgrid/model-serving.md](../../docs/cortexgrid/model-serving.md)
+for the end-to-end design.
+"""
+
+from cortexgrid.model_serving.application_spec import app_name
+from cortexgrid.model_serving.deployment_key import (
+    DeploymentConfig,
+    DeploymentKey,
+    deployment_key,
+)
+from cortexgrid.model_serving.lifecycle import (
+    ModelDeployFailed,
+    ModelNotDeployed,
+    deploy_model,
+    redeploy_model,
+    undeploy_model,
+    wait_for_model_serving,
+)
+from cortexgrid.model_serving.placement import ModelRequirements, vram_tiers
+from cortexgrid.model_serving.registry_tags import (
+    bundle_fingerprint_from_tags,
+    has_requirement_tags,
+    metadata_from_tags,
+    metadata_to_tags,
+    requirements_from_tags,
+    requirements_to_tags,
+)
+from cortexgrid.model_serving.serve_bundle import (
+    BundleMetadata,
+    ServeBundle,
+    build_bundle,
+    bundle_class,
+    bundle_fingerprint_from_url,
+    upload_bundle,
+)
+from cortexgrid.model_serving.status import (
+    Deployment,
+    ReplicaPlacement,
+    ServingMessage,
+    ServingStatus,
+    deployment_config,
+    list_deployed_models,
+    model_replica_placements,
+    model_serving_messages,
+    model_serving_status,
+    observe_deployments,
+)

cortexgrid-0.3.13/cortexgrid/model_serving/application_spec.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Any
+
+from cortexgrid.model_serving.deployment_key import DeploymentKey
+from cortexgrid.model_serving.placement import ModelRequirements, ray_actor_options
+from cortexgrid.model_serving.serve_bundle import (
+    BundleMetadata,
+    bundle_fingerprint_from_url,
+)
+
+
+def app_name(key: DeploymentKey) -> str:
+    return "__".join(_name_segments(key))
+
+
+def route_prefix(key: DeploymentKey) -> str:
+    return "/r/" + "/".join(_name_segments(key))
+
+
+def _name_segments(key: DeploymentKey) -> list[str]:
+    model_segments = [key.family, key.suffix, key.run_name]
+    if not key.config_fingerprint:
+        return model_segments
+    return [*model_segments, key.config_fingerprint]
+
+
+def build_application_spec(
+    key: DeploymentKey,
+    meta: BundleMetadata,
+    requirements: ModelRequirements,
+    num_replicas: int,
+    tiers: list[int],
+) -> dict[str, Any]:
+    """Assemble a Ray Serve application schema from pre-bundled metadata, the
+    model's requirements, and the cluster's GPU size classes."""
+    # working_dir carries the serve-app's own source; Ray pip-installs the
+    # third-party distributions the image lacks into a per-node cached
+    # virtualenv layered on the image. No pip key when there are none, so Ray
+    # builds no virtualenv.
+    runtime_env: dict[str, Any] = {"working_dir": meta.bundle_url}
+    if meta.pip_requirements:
+        runtime_env["pip"] = meta.pip_requirements
+    return {
+        "name": app_name(key),
+        "route_prefix": route_prefix(key),
+        # Ray Serve REST requires import_path to point at an Application builder
+        # (callable returning a bound node) or an already-bound node. A bare
+        # Deployment class is rejected, so cortexgrid.deploy_model goes through
+        # a generic builder that re-imports the user's class and binds it.
+        "import_path": "cortexgrid._serve_entry:build",
+        "args": {
+            "class_import_path": meta.class_import_path,
+            "family": key.family,
+            "suffix": key.suffix,
+            "run_name": key.run_name,
+            "config_fingerprint": key.config_fingerprint,
+            "num_replicas": num_replicas,
+            "ray_actor_options": ray_actor_options(requirements, tiers),
+        },
+        "runtime_env": runtime_env,
+    }
+
+
+def bundle_fingerprint_in_spec(spec: dict[str, Any]) -> str:
+    return bundle_fingerprint_from_url(spec["runtime_env"]["working_dir"])
+
+
+def replica_count_in_spec(spec: dict[str, Any]) -> int:
+    return spec["args"]["num_replicas"]

cortexgrid-0.3.13/cortexgrid/model_serving/deployment_key.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+
+
+DeploymentConfig = dict[str, str]
+
+_CONFIG_FINGERPRINT_LENGTH = 12
+
+
+@dataclass(frozen=True)
+class DeploymentKey:
+    family: str
+    suffix: str
+    run_name: str
+    config_fingerprint: str = ""
+
+
+def deployment_key(
+    family: str, suffix: str, run_name: str, config: DeploymentConfig
+) -> DeploymentKey:
+    return DeploymentKey(family, suffix, run_name, _config_fingerprint(config))
+
+
+def _config_fingerprint(config: DeploymentConfig) -> str:
+    for name, value in config.items():
+        if not isinstance(name, str) or not isinstance(value, str):
+            raise ValueError(
+                f"Deployment config must map strings to strings: {name!r}: {value!r}"
+            )
+    if not config:
+        return ""
+    canonical_config = json.dumps(config, sort_keys=True)
+    return hashlib.sha256(canonical_config.encode()).hexdigest()[
+        :_CONFIG_FINGERPRINT_LENGTH
+    ]

cortexgrid-0.3.13/cortexgrid/model_serving/deployment_records.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import Any
+
+from cortexgrid import state
+from cortexgrid.model_serving.deployment_key import DeploymentKey
+
+
+DeploymentRecord = dict[str, Any]
+
+
+def get_deployment_record(key: DeploymentKey) -> DeploymentRecord | None:
+    return state.get(*_path(key), params=_params(key))
+
+
+def put_deployment_record(key: DeploymentKey, body: DeploymentRecord) -> None:
+    state.put(*_path(key), body=body, params=_params(key))
+
+
+def patch_deployment_record(key: DeploymentKey, body: DeploymentRecord) -> bool:
+    return state.patch(*_path(key), body=body, params=_params(key))
+
+
+def delete_deployment_record(key: DeploymentKey) -> None:
+    state.delete(*_path(key), params=_params(key))
+
+
+def list_deployment_records() -> list[DeploymentRecord]:
+    return state.get("deployments")
+
+
+def key_of_record(record: DeploymentRecord) -> DeploymentKey:
+    return DeploymentKey(
+        record["family"],
+        record["suffix"],
+        record["run_name"],
+        record["config_fingerprint"],
+    )
+
+
+def _path(key: DeploymentKey) -> tuple[str, ...]:
+    return ("deployments", key.family, key.suffix, key.run_name)
+
+
+def _params(key: DeploymentKey) -> dict[str, str]:
+    return {"config_fingerprint": key.config_fingerprint}