marin-iris 0.99__tar.gz

marin_iris-0.99/.gitignore

@@ -0,0 +1,243 @@
+# redundant, but Ray looks for this otherwise.
+.git
+
+logs/
+
+# CPU profiles
+prof/
+
+# Downloaded build tools (zig, etc.)
+.tools/
+
+tests/snapshots/outputs
+tests/snapshots/diffs
+
+# don't log data/MD outputs to git
+data/*
+output/*
+outputs/*
+
+# Snapshot diffs and outputs
+tests/snapshots/*/outputs/*
+tests/snapshots/*/diffs/*
+
+# This is mainly for Ray and using submodule
+*/**/.git
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# pypa/gh-action-pypi-publish caches its Docker action manifest here.
+.github/.tmp/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+local_store/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Ruff
+.ruff_cache/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+*.iml
+
+# IDE Config
+.vscode/
+
+# Mac OS
+.DS_Store
+
+# Secrets
+credentials.json
+marin/crawl/bigquery-gcs-key.json
+
+# Archive
+archive/
+
+# Caches and Outputs
+!/scripts/web/output/
+!/output/
+
+# csv
+*.csv
+
+# wandb logs
+wandb
+
+# Ignore generated credentials from google-github-actions/auth
+gha-creds-*.json
+
+.aider*
+.git/*
+
+*.jsonl
+**/*.jsonl
+scr/*
+.weaver/
+
+# Local host Marin config
+.marin.yaml
+
+/scratch
+
+.forge
+.claude
+!.claude/skills
+.agents/tmp/
+.codex
+.entire
+
+.worktrees
+.obsidian
+.cw_env

marin_iris-0.99/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: marin-iris
+Version: 0.99
+Requires-Python: <3.13,>=3.11
+Requires-Dist: click>=8.3.1
+Requires-Dist: cloudpickle>=3.1.2
+Requires-Dist: connect-python>=0.9.0
+Requires-Dist: fsspec>=2024.0.0
+Requires-Dist: gcsfs>=2024.0.0
+Requires-Dist: google-auth>=2.0
+Requires-Dist: google-cloud-tpu>=1.18.0
+Requires-Dist: grpcio>=1.76.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: humanfriendly>=10.0
+Requires-Dist: marin-finelog==0.99
+Requires-Dist: marin-rigging==0.99
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyjwt>=2.12.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: s3fs>=2024.0.0
+Requires-Dist: starlette>=0.50.0
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: typing-extensions>=4.0
+Requires-Dist: uvicorn[standard]>=0.23.0
+Requires-Dist: zstandard>=0.22.0
+Provides-Extra: controller
+Requires-Dist: duckdb>=1.0.0; extra == 'controller'
+Requires-Dist: kubernetes>=31.0.0; extra == 'controller'
+Requires-Dist: pyarrow>=19.0.0; extra == 'controller'
+Provides-Extra: worker

marin_iris-0.99/examples/coreweave-ci.yaml

@@ -0,0 +1,92 @@
+# Persistent CoreWeave CI cluster. The CPU pool and the first H100 node stay
+# warm between runs; the H100 pool may autoscale to a second node for manual
+# multi-host canary runs.
+
+platform:
+  label_prefix: iris-ci
+  coreweave:
+    region: US-WEST-04A
+    namespace: iris-ci
+    kubeconfig_path: ~/.kube/coreweave-iris
+    object_storage_endpoint: https://74981a43be0de7712369306c7b19133d.r2.cloudflarestorage.com
+
+storage:
+  remote_state_dir: s3://marin-na/iris/state/ci
+
+kubernetes_provider:
+  namespace: iris-ci
+  default_image: ghcr.io/marin-community/iris-task:latest
+  host_network: true
+  cache_dir: /mnt/local/iris-cache
+  controller_address: http://iris-ci-controller-svc.iris-ci.svc.cluster.local:10000
+
+controller:
+  image: ghcr.io/marin-community/iris-controller:latest
+  coreweave:
+    port: 10000
+    service_name: iris-ci-controller-svc
+    scale_group: cpu-erapids
+
+defaults:
+  autoscaler:
+    evaluation_interval:
+      milliseconds: 10000
+    scale_up_delay:
+      milliseconds: 60000
+    scale_down_delay:
+      milliseconds: 300000
+    startup_grace_period:
+      milliseconds: 1200000    # 20 min — nodes are pinned warm so this rarely fires
+  task_env:
+    MARIN_PREFIX: s3://marin-na/marin
+    # H100 hostNetwork pods also see IB and SR-IOV link-local interfaces.
+    NCCL_SOCKET_IFNAME: =enp157s0np0
+  worker:
+    docker_image: ghcr.io/marin-community/iris-worker:latest
+    port: 10001
+    cache_dir: /mnt/local/iris-cache
+    runtime: kubernetes
+    default_task_image: ghcr.io/marin-community/iris-task:latest
+
+scale_groups:
+  cpu-erapids:
+    num_vms: 1
+    resources:
+      cpu: 64
+      ram: 256GB
+      disk: 1TB
+      device_type: cpu
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: cpu-erapids
+    buffer_slices: 1
+    max_slices: 1
+    priority: 50
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: US-WEST-04A
+        instance_type: cd-gp-i64-erapids
+
+  h100-8x:
+    num_vms: 1
+    resources:
+      cpu: 128
+      ram: 2048GB
+      disk: 1TB
+      device_type: gpu
+      device_variant: H100
+      device_count: 8
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: h100-8x
+    buffer_slices: 1
+    max_slices: 2
+    priority: 100
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: US-WEST-04A
+        instance_type: gd-8xh100ib-i128

marin_iris-0.99/examples/coreweave-rno2a.yaml

@@ -0,0 +1,99 @@
+# Iris on CoreWeave RNO2A (GH200, 1 GPU per node).
+#
+# Single GPU NodePool: gd-1xgh200. The controller is pinned onto this same pool
+# because the cluster has no CPU nodes.
+#
+# Setup:
+#   1. Split the multi-context kubeconfig (~/.kube/coreweave-rno2a holds both
+#      rno2a and usw09b) into a single-context file:
+#        KUBECONFIG=~/.kube/coreweave-rno2a \
+#          kubectl config view --minify --flatten --context=rno2a \
+#          > ~/.kube/cw-rno2a.yaml
+#        chmod 600 ~/.kube/cw-rno2a.yaml
+#   2. CoreWeave object storage credentials in env (consumed by `iris cluster start`):
+#        export R2_ACCESS_KEY_ID=<...>
+#        export R2_SECRET_ACCESS_KEY=<...>
+#   3. Start the cluster:
+#        cd lib/iris && uv run --group dev iris \
+#          --config=examples/coreweave-rno2a.yaml cluster start
+#
+# The controller Deployment includes the standard NVIDIA GPU toleration, so it
+# can schedule onto this GPU-only cluster.
+
+platform:
+  label_prefix: iris-rno2a
+  coreweave:
+    region: RNO2A
+    namespace: iris
+    kubeconfig_path: ~/.kube/cw-rno2a.yaml
+    # CoreWeave native object storage: virtual-hosted-style addressing
+    # (bucket goes in the subdomain). iris auto-detects cwobject.com domains
+    # and sets s3 addressing_style=virtual — see _needs_virtual_host_addressing.
+    # Bucket-less base endpoint. Virtual-host addressing prepends the bucket
+    # name as a subdomain at request time (s3://marin-poc/foo →
+    # https://marin-poc.cwobject.com/foo). Including the bucket here would
+    # double it (https://marin-poc.marin-poc.cwobject.com/...).
+    object_storage_endpoint: https://cwobject.com
+
+storage:
+  remote_state_dir: s3://marin-poc/iris/state/rno2a
+
+kubernetes_provider:
+  namespace: iris
+  default_image: ghcr.io/marin-community/iris-task:latest
+  # RNO2A GH200 nodes do not advertise rdma/ib devices. Keep host networking off
+  # so GPU task pods request only nvidia.com/gpu and can schedule.
+  host_network: false
+  cache_dir: /mnt/local/iris-cache
+  controller_address: http://iris-controller-svc.iris.svc.cluster.local:10000
+
+controller:
+  image: ghcr.io/marin-community/iris-controller:latest
+  coreweave:
+    port: 10000
+    service_name: iris-controller-svc
+    scale_group: gh200-1x
+
+defaults:
+  autoscaler:
+    evaluation_interval:
+      milliseconds: 10000
+    scale_up_delay:
+      milliseconds: 60000
+    scale_down_delay:
+      milliseconds: 300000
+    startup_grace_period:
+      milliseconds: 2400000    # 40 min — covers bare-metal node provisioning + Pod startup
+  task_env:
+    MARIN_PREFIX: s3://marin-na/marin
+    NCCL_SOCKET_IFNAME: =eth0
+  worker:
+    docker_image: ghcr.io/marin-community/iris-worker:latest
+    port: 10001
+    cache_dir: /mnt/local/iris-cache
+    runtime: kubernetes
+    default_task_image: ghcr.io/marin-community/iris-task:latest
+
+scale_groups:
+  gh200-1x:
+    num_vms: 1
+    resources:
+      cpu: 64
+      ram: 256GB
+      disk: 1TB
+      device_type: gpu
+      device_variant: GH200
+      device_count: 1
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: gh200-1x
+    # buffer_slices keeps one node warm so the controller always has a home.
+    buffer_slices: 1
+    max_slices: 2
+    priority: 100
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: RNO2A
+        instance_type: gd-1xgh200

marin_iris-0.99/examples/coreweave-usw09b.yaml

@@ -0,0 +1,98 @@
+# Iris on CoreWeave US-WEST-09B (B200, 8 GPUs per node).
+#
+# Single GPU NodePool: b200-8x. The controller is pinned onto this same pool
+# because the cluster has no CPU nodes.
+#
+# Setup:
+#   1. Split the multi-context kubeconfig (~/.kube/coreweave-rno2a holds both
+#      rno2a and usw09b) into a single-context file:
+#        KUBECONFIG=~/.kube/coreweave-rno2a \
+#          kubectl config view --minify --flatten --context=usw09b \
+#          > ~/.kube/cw-usw09b.yaml
+#        chmod 600 ~/.kube/cw-usw09b.yaml
+#   2. CoreWeave object storage credentials in env (consumed by `iris cluster start`):
+#        export R2_ACCESS_KEY_ID=<...>
+#        export R2_SECRET_ACCESS_KEY=<...>
+#   3. Start the cluster:
+#        cd lib/iris && uv run --group dev iris \
+#          --config=examples/coreweave-usw09b.yaml cluster start
+#
+# The controller Deployment includes the standard NVIDIA GPU toleration, so it
+# can schedule onto this GPU-only cluster.
+
+platform:
+  label_prefix: iris-usw09b
+  coreweave:
+    region: US-WEST-09B
+    namespace: iris
+    kubeconfig_path: ~/.kube/cw-usw09b.yaml
+    # CoreWeave native object storage. The marin-poc bucket lives in RNO2A;
+    # accessing it from US-WEST-09B is cross-region but works (just slower).
+    # iris auto-detects cwobject.com and switches addressing_style=virtual.
+    # Bucket-less base endpoint. Virtual-host addressing prepends the bucket
+    # name as a subdomain at request time (s3://marin-poc/foo →
+    # https://marin-poc.cwobject.com/foo). Including the bucket here would
+    # double it (https://marin-poc.marin-poc.cwobject.com/...).
+    object_storage_endpoint: https://cwobject.com
+
+storage:
+  remote_state_dir: s3://marin-poc/iris/state/usw09b
+
+kubernetes_provider:
+  namespace: iris
+  default_image: ghcr.io/marin-community/iris-task:latest
+  host_network: true
+  cache_dir: /mnt/local/iris-cache
+  controller_address: http://iris-controller-svc.iris.svc.cluster.local:10000
+
+controller:
+  image: ghcr.io/marin-community/iris-controller:latest
+  coreweave:
+    port: 10000
+    service_name: iris-controller-svc
+    scale_group: b200-8x
+
+defaults:
+  autoscaler:
+    evaluation_interval:
+      milliseconds: 10000
+    scale_up_delay:
+      milliseconds: 60000
+    scale_down_delay:
+      milliseconds: 300000
+    startup_grace_period:
+      milliseconds: 2400000    # 40 min — covers bare-metal node provisioning + Pod startup
+  task_env:
+    MARIN_PREFIX: s3://marin-na/marin
+    # B200 hostNetwork pods also see IB and SR-IOV link-local interfaces.
+    NCCL_SOCKET_IFNAME: =enp44s0np0
+  worker:
+    docker_image: ghcr.io/marin-community/iris-worker:latest
+    port: 10001
+    cache_dir: /mnt/local/iris-cache
+    runtime: kubernetes
+    default_task_image: ghcr.io/marin-community/iris-task:latest
+
+scale_groups:
+  b200-8x:
+    num_vms: 1
+    resources:
+      cpu: 128
+      ram: 2048GB
+      disk: 1TB
+      device_type: gpu
+      device_variant: B200
+      device_count: 8
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: b200-8x
+    # buffer_slices keeps one node warm so the controller always has a home.
+    buffer_slices: 1
+    max_slices: 2
+    priority: 100
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: US-WEST-09B
+        instance_type: b200-8x

marin_iris-0.99/examples/coreweave.yaml

@@ -0,0 +1,116 @@
+# Iris development cluster configuration for CoreWeave CKS.
+#
+# Architecture: KubernetesProvider dispatches task pods directly to the k8s API.
+# No worker daemons are needed. CoreWeave NodePool autoscaling provisions nodes
+# on demand; when pods are deleted, NodePools scale to zero automatically.
+# CPU nodes are kept always-on (buffer_slices=2) for monitoring software.
+#
+# Workflow:
+#
+#   1. Set S3 object storage credentials (required for s3:// storage URIs):
+#        export R2_ACCESS_KEY_ID=<your-r2-access-key-id>
+#        export R2_SECRET_ACCESS_KEY=<your-r2-secret-access-key>
+#      These are created in the Cloudflare dashboard under R2 > Manage R2 API Tokens.
+#      `iris cluster start` creates a K8s Secret from these env vars automatically.
+#
+#   2. Start the cluster (creates RBAC, shared NodePools, ConfigMap, Deployment, Service):
+#        iris --config=lib/iris/examples/coreweave.yaml cluster start
+#
+#   3. Use the Iris CLI:
+#        iris --config=lib/iris/examples/coreweave.yaml cluster status
+#        iris --config=lib/iris/examples/coreweave.yaml cluster dashboard
+#
+# This config file is used by:
+#   - The CLI on the operator's laptop (for `cluster start`, `cluster status`, job submission)
+#   - The controller and workers inside the cluster (mounted as ConfigMap at /etc/iris/config.yaml)
+#
+# To use a local kubeconfig (e.g. from CoreWeave Console > Tokens > Download):
+#   Set platform.coreweave.kubeconfig_path below, or:
+#   export KUBECONFIG=~/.kube/coreweave-iris
+
+platform:
+  label_prefix: coreweave
+  coreweave:
+    region: US-WEST-04A
+    namespace: iris
+    kubeconfig_path: ~/.kube/coreweave-iris
+    object_storage_endpoint: https://74981a43be0de7712369306c7b19133d.r2.cloudflarestorage.com
+
+storage:
+  remote_state_dir: s3://marin-na/iris/coreweave/state
+
+kubernetes_provider:
+  namespace: iris
+  default_image: ghcr.io/marin-community/iris-task:latest
+  host_network: true              # Required for NCCL/IB multi-host traffic
+  cache_dir: /mnt/local/iris-cache  # NVMe; default /cache hits the 15GB ramdisk on bare-metal GPU nodes
+  controller_address: http://iris-controller-svc.iris.svc.cluster.local:10000
+
+controller:
+  image: ghcr.io/marin-community/iris-controller:latest
+  coreweave:
+    port: 10000
+    service_name: iris-controller-svc
+    scale_group: cpu-erapids
+
+defaults:
+  autoscaler:
+    evaluation_interval:
+      milliseconds: 10000
+    scale_up_delay:
+      milliseconds: 60000
+    scale_down_delay:
+      milliseconds: 300000
+    startup_grace_period:
+      milliseconds: 2400000    # 40 min — covers autoscaler node provisioning + Pod startup
+  task_env:
+    MARIN_PREFIX: s3://marin-na/marin
+  worker:
+    docker_image: ghcr.io/marin-community/iris-worker:latest
+    port: 10001
+    cache_dir: /mnt/local/iris-cache
+    runtime: kubernetes
+    default_task_image: ghcr.io/marin-community/iris-task:latest
+
+scale_groups:
+  cpu-erapids:
+    num_vms: 1
+    resources:
+      cpu: 64
+      ram: 256GB
+      disk: 1TB
+      device_type: cpu
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: cpu-erapids
+    buffer_slices: 1
+    max_slices: 4
+    priority: 50
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: US-WEST-04A
+        instance_type: cd-gp-i64-erapids
+
+  h100-8x:
+    num_vms: 1
+    resources:
+      cpu: 128
+      ram: 2048GB
+      disk: 1TB
+      device_type: gpu
+      device_variant: H100
+      device_count: 8
+      capacity_type: on-demand
+    worker:
+      attributes:
+        pool: h100-8x
+    buffer_slices: 0
+    max_slices: 8
+    priority: 100
+    slice_template:
+      num_vms: 1
+      coreweave:
+        region: US-WEST-04A
+        instance_type: gd-8xh100ib-i128

marin_iris-0.99/examples/local-auth-gcp.yaml

@@ -0,0 +1,35 @@
+# Iris local configuration with GCP OAuth2 access token authentication
+# Usage: iris --cluster=local-auth-gcp cluster start --local
+# Requires: gcloud auth login && gcloud auth application-default login
+
+platform:
+  local:
+
+controller:
+  local:
+    port: 0  # auto-assign
+
+worker_provider: {}
+
+auth:
+  gcp:
+    project_id: hai-gcp-models  # GCP project ID — users must have access to log in
+  admin_users:
+    - russell.power@gmail.com  # Replace with actual admin email
+  optional: true
+
+scale_groups:
+  cpu:
+    num_vms: 1
+    resources:
+      cpu: 16
+      ram: 32GB
+      disk: 100GB
+      device_type: cpu
+      device_variant: cpu
+      capacity_type: on-demand
+    buffer_slices: 1
+    max_slices: 4
+    slice_template:
+      num_vms: 1
+      local: