cortexgrid 0.2.85__py3-none-any.whl

cortexgrid/ray_util.py

@@ -0,0 +1,171 @@
+"""Talk to the Ray cluster's dashboard: job submission + Serve applications.
+
+Job submission goes through ray's `JobSubmissionClient`. Serve applications use
+the dashboard's declarative `/api/serve/applications/` REST endpoint directly -
+there's no public Python client for it that doesn't drag in `ray.init`, and the
+REST surface is tiny."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+import requests  # type: ignore
+
+from cortexgrid.infra import get_ray_job_server_uri, get_ray_serve_applications_uri
+from ray.job_submission import JobSubmissionClient
+
+
+class JobStatus(str, Enum):
+    PENDING = "pending"  # Pending scheduling
+    RUNNING = "running"
+    FINISHED = "finished"
+    FAILED = "failed"
+    STOPPED = "stopped"
+
+
+_ray_job_submission_client: JobSubmissionClient | None = None
+
+
+def get_ray_job_submission_client() -> JobSubmissionClient:
+    """Return a cached JobSubmissionClient. Each constructor call does a version-check RTT, so we share one."""
+    global _ray_job_submission_client
+    if _ray_job_submission_client is None:
+        _ray_job_submission_client = JobSubmissionClient(get_ray_job_server_uri())
+    return _ray_job_submission_client
+
+
+def get_ray_job_id_for_cortexgrid_job(
+    run_id: str, job_id: str, all_ray_submission_ids: list[str] | None = None
+) -> str | None:
+    if all_ray_submission_ids is None:
+        all_ray_submission_ids = list_ray_jobs_with_submission_id()
+    prefix = ray_submission_id(run_id, job_id, None) + "-"
+    attempts = [sid for sid in all_ray_submission_ids if sid.startswith(prefix)]
+    return max(attempts, key=get_ray_job_attempt) if attempts else None
+
+
+def get_ray_status(ray_job_id: str | None) -> str | None:
+    """Return the current status of a previously submitted ray job."""
+    if ray_job_id is None:
+        return None
+
+    client = get_ray_job_submission_client()
+    return client.get_job_status(ray_job_id).value
+
+
+def get_ray_job_status(ray_job_id: str | None) -> JobStatus:
+    """Derive a job's observable status from a live Ray query.
+
+    Returns ``PENDING`` both when ``ray_job_id is None`` (never submitted)
+    and when Ray itself reports ``PENDING`` (queued). Callers that need
+    to distinguish those two must check ``ray_job_id is None`` first.
+    """
+    ray_status = get_ray_status(ray_job_id)
+    if ray_job_id is None:
+        return JobStatus.PENDING
+    if ray_status == "SUCCEEDED":
+        return JobStatus.FINISHED
+    if ray_status == "FAILED":
+        return JobStatus.FAILED
+    if ray_status == "STOPPED":
+        return JobStatus.STOPPED
+    return JobStatus.RUNNING
+
+
+def get_ray_logs(ray_job_id: str | None) -> str | None:
+    """Return the stdout/stderr of a previously submitted ray job."""
+    if ray_job_id is None:
+        return None
+
+    client = get_ray_job_submission_client()
+    return client.get_job_logs(ray_job_id)
+
+
+def get_ray_job_url(ray_job_id: str | None) -> str | None:
+    """Build the browser-facing URL to view a job in the Ray dashboard."""
+    if ray_job_id is None:
+        return None
+
+    base = get_ray_job_server_uri()
+    return f"{base}/#/jobs/{ray_job_id}"
+
+
+def stop_ray_job(ray_job_id: str) -> None:
+    """Stop a running ray job."""
+    client = get_ray_job_submission_client()
+    client.stop_job(ray_job_id)
+
+
+def list_ray_jobs_with_submission_id() -> list[str]:
+    """List all ray jobs, the ones that received submission id."""
+    client = get_ray_job_submission_client()
+    return [
+        job.submission_id for job in client.list_jobs() if job.submission_id is not None
+    ]
+
+
+def ray_submission_id(run_id: str, job_id: str, attempt: int | None) -> str:
+    """Deterministic Ray submission id derived from a job's identity."""
+    return (
+        f"{run_id}-{job_id}-{attempt}" if attempt is not None else f"{run_id}-{job_id}"
+    )
+
+
+def get_ray_job_attempt(ray_job_id: str | None) -> int:
+    if ray_job_id is None:
+        return 0
+    _, sep, suffix = ray_job_id.rpartition("-")
+    if not sep or not suffix.isdigit():
+        raise ValueError(f"Ray submission_id has no attempt suffix: {ray_job_id!r}")
+    return int(suffix)
+
+
+def submit_ray_job(
+    submission_id: str,
+    entrypoint: str,
+    runtime_env: dict,
+    num_gpus: int = 0,
+    num_cpus: int = 1,
+) -> None:
+    """Submit a job to Ray with a caller-supplied deterministic submission_id.
+
+    Raises whatever the Ray SDK raises on a duplicate submission_id; the
+    control plane relies on that exception to short-circuit re-submission
+    on retry paths.
+    """
+    client = get_ray_job_submission_client()
+    client.submit_job(
+        submission_id=submission_id,
+        entrypoint=entrypoint,
+        runtime_env=runtime_env,
+        entrypoint_num_gpus=num_gpus,
+        entrypoint_num_cpus=num_cpus,
+    )
+
+
+def get_serve_details() -> dict[str, Any]:
+    """GET the Serve controller's view of currently-running applications.
+
+    Returns the full ServeInstanceDetails JSON; callers project the parts they
+    care about. Raises for any non-2xx response (HTTPError carries the body).
+    """
+    response = requests.get(get_ray_serve_applications_uri(), timeout=30)
+    response.raise_for_status()
+    return response.json()
+
+
+def put_serve_applications(applications: list[dict[str, Any]]) -> None:
+    """PUT the full desired set of Serve applications.
+
+    The endpoint is declarative: any application not in `applications` is
+    deleted, any new application is created, any updated application is
+    rolled. Callers that want to mutate one app should GET first, splice,
+    and PUT the result.
+    """
+    response = requests.put(
+        get_ray_serve_applications_uri(),
+        json={"applications": applications},
+        timeout=60,
+    )
+    response.raise_for_status()

cortexgrid/s3_util.py

@@ -0,0 +1,135 @@
+"""S3/MinIO wrappers.
+
+Builds an S3 boto3 client from the S3_* entries in the head secrets store:
+S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT_URL. They are
+fetched at runtime via cortexgrid.secrets.get_secret, so no consumer has to
+export them as environment variables.
+
+On the AWS profile S3_ENDPOINT_URL is the regional s3.amazonaws.com URL and
+S3_* are the real AWS keys; on the on-prem profile S3_ENDPOINT_URL is the
+tailnet-reachable MinIO URL and S3_* are the MinIO admin creds.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import boto3  # type: ignore
+from tqdm import tqdm  # type: ignore
+
+from cortexgrid.infra import get_s3_bucket, get_s3_endpoint_url
+from cortexgrid.secrets import get_secret
+
+
+def get_s3_client() -> Any:
+    """Return a boto3 S3 client configured for the cluster's object store."""
+    return boto3.client(
+        "s3",
+        aws_access_key_id=get_secret("S3_ACCESS_KEY_ID"),
+        aws_secret_access_key=get_secret("S3_SECRET_ACCESS_KEY"),
+        endpoint_url=get_s3_endpoint_url(),
+        # Required: without it boto3 picks the local default region for SigV4,
+        # which mismatches the AWS endpoint and produces 301 Moved Permanently
+        # against real AWS. MinIO ignores the value.
+        region_name=get_secret("S3_REGION"),
+    )
+
+
+def _ensure_bucket(client: Any, bucket: str) -> None:
+    """Create the bucket if and only if head_bucket returns 404. Other errors
+    (region mismatch, perms) propagate so they aren't silently masked by an
+    unrelated create_bucket failure."""
+    try:
+        client.head_bucket(Bucket=bucket)
+    except client.exceptions.ClientError as e:
+        if e.response["Error"]["Code"] != "404":
+            raise
+        # LocationConstraint is required for any AWS region other than us-east-1.
+        # MinIO accepts it too.
+        client.create_bucket(
+            Bucket=bucket,
+            CreateBucketConfiguration={"LocationConstraint": get_secret("S3_REGION")},
+        )
+
+
+def upload(local_path: str, dest_path: str | None = None) -> str:
+    """Upload a local file to the canonical S3/MinIO bucket.
+
+    Args:
+        local_path: Path to the local file.
+        dest_path: Full object key in the bucket (folder/filename). Defaults
+            to the local file's basename at the bucket root.
+
+    Returns:
+        The s3://bucket/<dest_path> URI of the uploaded object.
+    """
+    bucket = get_s3_bucket()
+    dest_path = dest_path or os.path.basename(local_path)
+
+    client = get_s3_client()
+    _ensure_bucket(client, bucket)
+    file_size = os.path.getsize(local_path)
+    with tqdm(
+        total=file_size,
+        unit="B",
+        unit_scale=True,
+        desc=f"Uploading {os.path.basename(local_path)}",
+    ) as pbar:
+        client.upload_file(local_path, bucket, dest_path, Callback=pbar.update)
+    return f"s3://{bucket}/{dest_path}"
+
+
+def upload_dir(local_dir: str, dest_path: str = "") -> list[str]:
+    """Upload all files in a directory tree to the canonical S3/MinIO bucket.
+
+    Args:
+        local_dir: Path to the local directory.
+        dest_path: Folder inside the bucket where the tree lands. Each file's
+            key is dest_path/<path relative to local_dir>. Defaults to bucket root.
+
+    Returns:
+        List of s3://bucket/<key> URIs for uploaded objects.
+    """
+    bucket = get_s3_bucket()
+
+    client = get_s3_client()
+    _ensure_bucket(client, bucket)
+
+    uploaded: list[str] = []
+    for root, _dirs, files in os.walk(local_dir):
+        for filename in files:
+            local_path = os.path.join(root, filename)
+            rel_path = os.path.relpath(local_path, local_dir).replace(os.sep, "/")
+            key = f"{dest_path}/{rel_path}" if dest_path else rel_path
+            client.upload_file(local_path, bucket, key)
+            uploaded.append(f"s3://{bucket}/{key}")
+
+    return uploaded
+
+
+def delete_prefix(prefix: str) -> None:
+    """Delete every object under `prefix` in the canonical bucket."""
+    bucket = get_s3_bucket()
+    client = get_s3_client()
+    paginator = client.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        keys = [{"Key": obj["Key"]} for obj in page.get("Contents", [])]
+        if keys:
+            client.delete_objects(Bucket=bucket, Delete={"Objects": keys})
+
+
+def download(src_path: str, local_path: str | None = None) -> str:
+    """Download a file from the canonical S3/MinIO bucket.
+
+    Args:
+        src_path: Full object key in the bucket (folder/filename).
+        local_path: Where to save locally. Defaults to the src_path basename.
+
+    Returns:
+        The local file path.
+    """
+    local_path = local_path or os.path.basename(src_path)
+    client = get_s3_client()
+    client.download_file(get_s3_bucket(), src_path, local_path)
+    return local_path

cortexgrid/secrets.py

@@ -0,0 +1,56 @@
+"""Secrets API - client for the head's secrets server.
+
+The head runs a small HTTP server (k8s/seed/scripts/cortexgrid_head.py) that
+keeps every secret in a .env file on the head host. All calls go to
+$CORTEXGRID_HEAD_URL: http://robolab-head:7700 from a laptop on the tailnet,
+http://cortexgrid-head.default.svc.cluster.local:7700 from pods.
+
+There is no authentication: the head is only reachable over the tailnet.
+"""
+
+from __future__ import annotations
+
+import os
+from urllib.parse import quote
+
+import requests  # type: ignore
+
+
+_TIMEOUT_S = 10.0
+
+
+def _head_url() -> str:
+    url = os.environ.get("CORTEXGRID_HEAD_URL")
+    if not url:
+        raise RuntimeError(
+            "CORTEXGRID_HEAD_URL is not set; point it at the head's secrets "
+            "server, e.g. http://robolab-head:7700"
+        )
+    return url.rstrip("/")
+
+
+def _secret_url(id: str) -> str:
+    return f"{_head_url()}/secrets/{quote(id, safe='')}"
+
+
+def get_secret(id: str) -> str:
+    response = requests.get(_secret_url(id), timeout=_TIMEOUT_S)
+    response.raise_for_status()
+    return response.json()["value"]
+
+
+def list_secrets() -> list[str]:
+    response = requests.get(f"{_head_url()}/secrets", timeout=_TIMEOUT_S)
+    response.raise_for_status()
+    return response.json()
+
+
+def set_secret(id: str, value: str) -> None:
+    response = requests.put(_secret_url(id), json={"value": value}, timeout=_TIMEOUT_S)
+    response.raise_for_status()
+
+
+def delete_secret(id: str) -> None:
+    """Idempotent: already-gone is treated as success, matching HTTP DELETE semantics."""
+    response = requests.delete(_secret_url(id), timeout=_TIMEOUT_S)
+    response.raise_for_status()

cortexgrid-0.2.85.dist-info/METADATA

@@ -0,0 +1,199 @@
+Metadata-Version: 2.5
+Name: cortexgrid
+Version: 0.2.85
+Summary: Connect your ML code to the RoboLab compute cluster — Ray, MLflow, and S3
+Project-URL: Homepage, https://github.com/robodatalab/cortexgrid
+Project-URL: Repository, https://github.com/robodatalab/cortexgrid
+Project-URL: Documentation, https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/README.md
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.34
+Requires-Dist: cloudpickle>=3.0
+Requires-Dist: fabric>=3.2.3
+Requires-Dist: haikunator>=2.1.0
+Requires-Dist: mlflow<4,>=3.11
+Requires-Dist: pip>=23.0
+Requires-Dist: pydantic-settings>=2.13.1
+Requires-Dist: pydantic>=2.13.3
+Requires-Dist: pydotenv>=0.0.7
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: ray[default]<3,>=2.9
+Requires-Dist: requests>=2.31
+Requires-Dist: setuptools>=82.0.1
+Requires-Dist: tqdm>=4.60
+Description-Content-Type: text/markdown
+
+# cortexgrid
+
+`cortexgrid` is a Python library that connects your ML code to the deployed infrastructure. It wraps Ray, MLflow, and S3/MinIO so your training scripts don't need to know about URLs, credentials, or service endpoints.
+
+### Installation
+
+Add `cortexgrid` as a dependency in your project's `pyproject.toml`:
+
+```toml
+[project]
+dependencies = [
+    "cortexgrid",
+]
+```
+
+Then `uv sync` to install it, and point it at the head's secrets server (reachable over the tailnet):
+
+```bash
+export CORTEXGRID_HEAD_URL=http://robolab-head:7700
+```
+
+### Usage
+
+```python
+import cortexgrid
+
+cortexgrid.init(experiment="weather-forecast")
+```
+
+That single call reads the service URLs from the head's secrets server at `$CORTEXGRID_HEAD_URL` and connects to all services through them. It also creates (or finds) the named MLflow experiment and starts a new run inside it. Omit `experiment=` to auto-generate a unique name like `funky-koval-12`.
+
+**One experiment per binary run.** `cortexgrid.init()` may only be called once per process. Every subsequent `cortexgrid.log_metric`, `cortexgrid.log_artifact`, checkpoint, and `cortexgrid.remote()` submission is scoped to that experiment+run. Remote jobs dispatched by the control plane inherit the experiment+run via the pickled payload, so their logging flows into the same MLflow run as the parent binary.
+
+#### Experiment tracking (MLflow)
+
+```python
+cortexgrid.init(experiment="weather-forecast")
+
+cortexgrid.log_params({"lr": 1e-3, "epochs": 20, "batch_size": 64})
+
+for epoch in range(20):
+    loss = train_one_epoch(model, dataloader)
+    cortexgrid.log_metric("loss", loss, step=epoch)
+
+    if epoch % 5 == 0:
+        with cortexgrid.checkpoint() as ckpt:
+            ckpt.epoch = epoch
+            ckpt.save_training_state(model, optimizer)
+```
+
+No run-scoping context manager — `init()` starts the run, and every subsequent logging call flows into it. Metrics and artifacts are logged to the MLflow server on the DGX. View them at `http://<DGX_IP>:5000`.
+
+#### Checkpointing and resuming
+
+Inside a cortexgrid job, `cortexgrid.checkpoint()` returns an attribute-based checkpoint object that persists to MLflow artifacts when its `with` block exits. On job restart (either manual retry or `retry=True`), `cortexgrid.resume()` returns the last checkpoint for the same job ID, or `None` if there isn't one.
+
+```python
+ckpt = cortexgrid.resume()
+if ckpt:
+    ckpt.restore_training_state(model, optimizer)
+    start_epoch = ckpt.epoch + 1
+else:
+    start_epoch = 0
+
+for epoch in range(start_epoch, 20):
+    train_one_epoch(model, dataloader)
+    with cortexgrid.checkpoint() as ckpt:
+        ckpt.epoch = epoch
+        ckpt.save_training_state(model, optimizer)
+```
+
+You can assign any cloudpickle-compatible or torch-serializable value as an attribute on the checkpoint (`ckpt.metric = 0.93`, `ckpt.weights = model.state_dict()`); the `save_training_state`/`restore_training_state` helpers are a shortcut for the common model+optimizer pair.
+
+#### Distributed compute (jobs control plane)
+
+```python
+def train_step(batch):
+    # runs on the DGX GPU
+    # MLflow and S3 env vars are injected automatically
+    return loss
+
+job_id = cortexgrid.remote(train_step, batch, num_gpus=1, retry=True)
+print(f"Submitted: {job_id}")
+```
+
+`cortexgrid.remote` submits a job *request* (a pickled payload plus a `JobLifecycle` record) to MLflow and returns a job ID string immediately. It does not wait for the job to run or finish — use the UI at `http://<DGX_IP>:8000`, or poll `cortexgrid.list_experiment_run_jobs(run_id)`, to observe status.
+
+A separate service — the **jobs control plane** — polls MLflow for pending job requests, matches them against the set of Ray submissions the cluster already has, and submits anything missing. It is also responsible for retrying failed jobs and honouring user-requested stops.
+
+Each submission captures the code and dependencies the entry function needs automatically ([_bundle.py](https://github.com/robodatalab/cortexgrid/blob/main/cortexgrid/_bundle.py)):
+- `bundle(entry)` traces the import graph from the function's source file, resolving each import the way the interpreter does (via `sys.path`), and returns every file needed to run it -- your own modules and third-party packages alike, wherever they live. The standard library is excluded (it ships with the interpreter)
+- Everything ships **as source**: the bundle is staged at each file's import path and tarred into the Ray `working_dir`. Nothing is `pip`-installed on the worker
+- Dependencies the worker image already has are subtracted rather than shipped: `bundle(entry) - worker_provides()`, where `worker_provides()` is the bundle of the packages baked into the ray image (torch and its CUDA stack, ray, mlflow, ...). See [k8s/docker/ray/Dockerfile](https://github.com/robodatalab/cortexgrid/blob/main/k8s/docker/ray/Dockerfile)
+- Injects MLflow/S3 credentials so task code running on the DGX can reach all services
+
+##### Retries
+
+Pass `retry=True` and the control plane will resubmit the job whenever Ray reports the most recent attempt as `FAILED`. Retries are **unbounded by design**: the intended way to end a retry loop is to stop the job manually from the UI (which flips the `stop_requested` latch on the lifecycle, and the control plane stops the current Ray attempt on its next poll). This keeps the retry policy simple — you don't have to predict a good `max_retries` up front — and puts the human in the loop for anything that's failing persistently.
+
+##### Stopping a job
+
+```python
+cortexgrid.stop_experiment_run_jobs(run_id)   # stops every job in the run
+```
+
+`stop_experiment_run_jobs` never touches Ray directly. It only flips `stop_requested` on each job's lifecycle record in MLflow. The control plane observes the flag on its next poll and calls `ray.stop_job` for any attempt that has reached Ray. For jobs that have not yet been submitted, the same flag short-circuits the submission path inside the worker.
+
+#### Object storage (S3/MinIO)
+
+```python
+cortexgrid.upload("data/output.parquet", bucket="ray-checkpoints", key="run-42/output.parquet")
+cortexgrid.download("ray-checkpoints", "run-42/output.parquet", local_path="./output.parquet")
+
+# or get the raw boto3 client
+s3 = cortexgrid.get_s3_client()
+```
+
+Works with MinIO on the DGX today, real S3 on AWS tomorrow — same code.
+
+#### Getting raw clients
+
+```python
+mlflow_client = cortexgrid.get_mlflow_client()   # mlflow.tracking.MlflowClient
+s3_client = cortexgrid.get_s3_client()           # boto3 S3 client
+```
+
+#### Model registry and serving
+
+Save a trained model's weights together with the serve-app that fronts it, then deploy it as a Ray Serve application:
+
+```python
+saved = cortexgrid.save_model(weights_dir, MyServeApp, family="qwen", suffix="instruct")
+deployed = cortexgrid.deploy_model("qwen", "instruct", saved.run_name, wait=True)
+print(deployed.url)
+```
+
+`save_model` is synchronous (registry lifecycle: `uploading` -> `ready`); `deploy_model` schedules the serving lifecycle (`deploying` -> `running`). See [model-serving.md](https://github.com/robodatalab/cortexgrid/blob/main/docs/cortexgrid/model-serving.md) for both lifecycles end to end - upload/deploy/undeploy/delete, status queries (`model_registry_status`, `model_serving_status`), and error handling.
+
+### API reference
+
+| Function | Description |
+|----------|-------------|
+| `cortexgrid.init(experiment=None)` | Configure connections + start a new MLflow run inside the named experiment. One call per binary. |
+| `cortexgrid.log_metric(key, value, step)` | Log a metric |
+| `cortexgrid.log_metrics(metrics, step)` | Log multiple metrics |
+| `cortexgrid.log_params(params)` | Log parameters |
+| `cortexgrid.log_artifact(path, artifact_path)` | Log a file as an artifact |
+| `cortexgrid.checkpoint()` | Context manager returning an attribute-based checkpoint saved to MLflow on exit |
+| `cortexgrid.resume()` | Load the latest checkpoint for the current job, or `None` |
+| `cortexgrid.remote(fn, *args, num_gpus=0, num_cpus=1, retry=False, **kwargs)` | Submit a function to the jobs control plane; returns a job ID |
+| `cortexgrid.list_experiment_run_jobs(run_id)` | List `JobLifecycle` records for every cortexgrid job in a run |
+| `cortexgrid.stop_experiment_run_jobs(run_id)` | Request every job in a run to stop (flips the `stop_requested` latch) |
+| `cortexgrid.get_ray_job_status(ray_job_id)` | Live Ray status for a submission id |
+| `cortexgrid.get_ray_logs(ray_job_id)` | Tail the stdout/stderr of a Ray submission |
+| `cortexgrid.upload(path, bucket, key)` | Upload a file to S3/MinIO |
+| `cortexgrid.download(bucket, key, path)` | Download a file from S3/MinIO |
+| `cortexgrid.get_mlflow_client()` | Raw configured MLflow client |
+| `cortexgrid.get_s3_client()` | Raw configured boto3 S3 client |
+
+## ML compute stack
+
+The DGX Spark runs the following services as k8s workloads managed by Argo CD (see [../k8s/argo_deployments/](https://github.com/robodatalab/cortexgrid/tree/main/k8s/argo_deployments/)):
+
+| Service | Port | Purpose |
+|---------|------|---------|
+| Ray | 8265 | Dashboard + job submission (NodePort 30265) |
+| MLflow | 5000 | Experiment tracking, model registry |
+| MinIO | 9000/9001 | S3-compatible artifact storage |
+| PostgreSQL | 5432 | MLflow metadata backend |
+| Prometheus | 9090 | Metrics collection |
+| Grafana | 3000 | Dashboards (GPU, jobs, system) |
+

cortexgrid-0.2.85.dist-info/RECORD

@@ -0,0 +1,19 @@
+cortexgrid/__init__.py,sha256=GBz4Y1QXNQL2sbcqBXK_A-xxyOgMx8_zHIjuRpwpa9s,4619
+cortexgrid/_bundle.py,sha256=udXCvPr9l_Uf4iLGJRVTG-SKfcMPS_jBiHUG2tnx3pM,7923
+cortexgrid/_ray_job_driver.py,sha256=kl1mDUb8PL5zX7f9YO1s4ufK8J8Gu6RrZjoFKB_1qxU,1459
+cortexgrid/_serve_entry.py,sha256=FdU0D9VJnjAOktuQbtqT6lOhLsxKh26IElVq5XHlQBM,1699
+cortexgrid/checkpoint.py,sha256=WPhngYX_Xu-eHwfxcS7r0fQw65FgagosbqmeaXqlzN8,7087
+cortexgrid/experiment.py,sha256=0kcIwylQJAv2j6pM2g7WlVvZb953nih89ri-bSmk_D0,8260
+cortexgrid/infra.py,sha256=EJwyQvhL_dtjE2Xh5U0S-d0aGYylbL2WSXhB70a0MxM,1217
+cortexgrid/jobs.py,sha256=E1BJhCLJe8BfuIuITTQfcQ_SvMM-TMvoMyrGvoLyC30,10732
+cortexgrid/mlflow_util.py,sha256=jA3JtURiB8SwXbGZ32yZbgnPIYRZPLko9ewFW-GvYI4,3650
+cortexgrid/model_serving.py,sha256=0EuCHnLHP99EpOomjPrItLIFywmvMiFLji5eSBi7S_M,13908
+cortexgrid/model_storage.py,sha256=KmXfmvEZgYl9iQwE3WLoLfHdBuMrcveKn64GGv632ng,10326
+cortexgrid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cortexgrid/ray_util.py,sha256=e3pvDL7LOHhKUGVHBvWhWH5p0kq0_qg9uN7ecUptnBY,5757
+cortexgrid/s3_util.py,sha256=ZftE8eGAO8M-GmKhX0BLVu9zYFEeulYrH1nmIdX0aUE,4850
+cortexgrid/secrets.py,sha256=DL0A884HKivrTNd7R0RX2ncMUddkdIbHWlu-9osXtHM,1659
+cortexgrid-0.2.85.dist-info/METADATA,sha256=GYqrUaN-lVpxxlLMqcZJJmjIslQ7JMd2JxEYUcxreks,10380
+cortexgrid-0.2.85.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+cortexgrid-0.2.85.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+cortexgrid-0.2.85.dist-info/RECORD,,

cortexgrid-0.2.85.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any