cluster-uv 0.0.1.post1__tar.gz

cluster_uv-0.0.1.post1/.claude-plugin/plugin.json

@@ -0,0 +1,8 @@
+{
+  "name": "cluv",
+  "description": "Assist with cluv — a UV-based multi-cluster HPC project manager",
+  "version": "0.1.0",
+  "keywords": ["hpc", "slurm", "uv", "ssh", "mila", "drac"],
+  "author": { "name": "fabrice" },
+  "license": "MIT"
+}

cluster_uv-0.0.1.post1/.copier-answers.yml

@@ -0,0 +1,8 @@
+# Changes here will be overwritten by Copier
+_commit: v0.0.5
+_src_path: gh:mila-iqia/tool_template
+project_description: A new project/tool called cluv.
+python_version: '3.13'
+tool_name: cluv
+your_email: normandf@mila.quebec
+your_name: Fabrice Normandin

cluster_uv-0.0.1.post1/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+
+name: Bug report
+about: Create a report to help us improve
+title: '[BUG]: '
+labels: bug
+assignees: ''
+
+---
+
+### What command did you run?
+<!-- Please paste the exact command you ran that caused the bug. -->
+
+### Describe the bug
+<!-- A clear and concise description of what the bug is. If there is an error traceback, please paste it here. -->
+
+### Screenshots
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+### Setup
+<!-- Please provide details about your setup : OS, Cluv config, Clusters... -->
+
+### Additional context
+<!-- Add any other context about the problem here. -->

cluster_uv-0.0.1.post1/.github/ISSUE_TEMPLATE/feature_proposal.md

@@ -0,0 +1,18 @@
+---
+
+name: Feature proposal
+about: Propose a new feature or improvement
+title: '[FEAT]: '
+labels: enhancement
+assignees: ''
+
+---
+
+### Context
+<!-- Why is this feature needed? What problem does it solve or what use case does it enable? -->
+
+### Proposed solution
+<!-- Describe how you'd like this to work. Include example commands or expected output if applicable. -->
+
+### Additional context
+<!-- Any other context, references, or screenshots that might be helpful. -->

cluster_uv-0.0.1.post1/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+## Summary
+<!-- A clear and concise description of the feature you're proposing. -->
+
+## Issues
+<!-- List any related issues here. If this is a new feature, you can create an issue first and link it here. -->

cluster_uv-0.0.1.post1/.github/workflows/build.yaml

@@ -0,0 +1,136 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+
+# https://stackoverflow.com/a/72408109/6388696
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  linting:
+    name: Run linting/pre-commit checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.13"
+      - run: pip install 'pre-commit<4.0.0'
+      - run: pre-commit --version
+      - run: pre-commit install
+      - run: pre-commit run --all-files --show-diff-on-failure
+
+  check_docs:
+    needs: [ linting ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+          enable-cache: true
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-suffix: "3.13"
+      - name: Install dependencies
+        run: uv sync --frozen
+      - name: Build the documentation (strict mode)
+        run: uv run mkdocs build --strict
+
+  unit_tests:
+    needs: [ linting ]
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      max-parallel: 4
+      matrix:
+        platform: [ "ubuntu-latest", "macos-latest" ]
+        python-version: [ "3.13" ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+          enable-cache: true
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-suffix: ${{ matrix.python-version }}
+      - name: Pin python-version ${{ matrix.python-version }}
+        run: uv python pin ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --frozen
+      - name: Test with pytest
+        run: uv run pytest -v --cov=cluv --cov-report=xml --cov-append
+      - name: Store coverage report as an artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-reports-unit-tests-${{ matrix.platform }}-${{
+            matrix.python-version }}
+          path: ./coverage.xml
+  integration_tests:
+    needs: [ unit_tests ]
+    runs-on: self-hosted
+    # TODO: remove this once we have a decently solid setup, not too reliant on the dev machine
+    # having active SSH connections to the Slurm clusters.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # Trying to checkout the branch (not just a commit) so that the cluv command
+          # that do a git push work in the self-hosted CI.
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "latest"
+          enable-cache: true
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install dependencies
+        run: uv sync --frozen
+      - name: Run all tests
+        run: uv run pytest -v --slow --cov=cluv --cov-report=xml --cov-append
+      - name: Store coverage report as an artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-reports-integration-tests
+          path: ./coverage.xml
+
+  # https://about.codecov.io/blog/uploading-code-coverage-in-a-separate-job-on-github-actions/
+  upload-coverage-codecov:
+    needs: [ unit_tests, integration_tests ]
+    runs-on: ubuntu-latest
+    name: Upload coverage reports to Codecov
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-reports-*
+          merge-multiple: false
+          # download all the artifacts in this directory (each .coverage.xml will be in a subdirectory)
+          # Next step if this doesn't work would be to give the coverage files a unique name and use merge-multiple: true
+          path: coverage_reports
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: coverage_reports
+          fail_ci_if_error: true

cluster_uv-0.0.1.post1/.github/workflows/docs.yaml

@@ -0,0 +1,35 @@
+name: Publish docs via GitHub Pages
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Deploy docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+          enable-cache: true  # no need, uses the local uv cache.
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-suffix: "3.13"
+
+      - name: Pin python-version
+        run: uv python pin 3.13
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Deploy docs
+        run: uv run mkdocs gh-deploy --force
+        # note: Checking if we really need the one below:
+        # uses: mhausenblas/mkdocs-deploy-gh-pages@1.9
+        # # Or use mhausenblas/mkdocs-deploy-gh-pages@nomaterial to build without the mkdocs-material theme
+        # env:
+        #   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

cluster_uv-0.0.1.post1/.github/workflows/publish.yaml

@@ -0,0 +1,47 @@
+# This workflow will upload a Python Package using Poetry when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+jobs:
+  # https://docs.pypi.org/trusted-publishers/using-a-publisher/
+  publish:
+    strategy:
+      matrix:
+        python-version: [3.13]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    environment:
+      name: pypi
+      url: https://pypi.org/p/cluster-uv
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+          enable-cache: true
+          # https://github.com/astral-sh/setup-uv?tab=readme-ov-file#github-authentication-token
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-suffix: ${{ matrix.python-version }}
+      - name: Pin python-version ${{ matrix.python-version }}
+        run: uv python pin ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Build package
+        run: |
+          uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

cluster_uv-0.0.1.post1/.gitignore

@@ -0,0 +1,18 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# uv lockfile
+# uv.lock
+.claude
+.vscode
+logs/
+# created by `uv run mkdocs serve` and such.
+site/

cluster_uv-0.0.1.post1/.pre-commit-config.yaml

@@ -0,0 +1,51 @@
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      # list of supported hooks: https://pre-commit.com/hooks.html
+      - id: trailing-whitespace
+        require_serial: true
+      - id: end-of-file-fixer
+        require_serial: true
+      # - id: check-docstring-first
+      - id: check-added-large-files
+        require_serial: true
+        exclude: uv.lock
+      - id: check-ast
+        require_serial: true
+      - id: check-yaml
+        require_serial: true
+        exclude: "mkdocs.yml"
+      - id: debug-statements
+        require_serial: true
+      - id: detect-private-key
+        require_serial: true
+      - id: check-executables-have-shebangs
+        require_serial: true
+      - id: check-toml
+        require_serial: true
+      - id: check-case-conflict
+        require_serial: true
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.15.16
+    hooks:
+      # Run the linter.
+      - id: ruff-check
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+
+  # word spelling linter
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.2
+    hooks:
+      - id: codespell
+        args:
+          - --skip=logs/**,data/**
+          # - --ignore-words-list=abc,def
+        require_serial: true

cluster_uv-0.0.1.post1/.python-version

@@ -0,0 +1 @@
+3.13

cluster_uv-0.0.1.post1/CLAUDE.md

@@ -0,0 +1,52 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+```bash
+# Install dependencies and activate the venv
+uv sync
+
+# Run cluv directly (no install needed)
+uv run cluv <command>
+
+# Run with verbose logging (-v, -vv, -vvv)
+uv run cluv -v status
+```
+
+Run tests with `uv run pytest`.
+
+## Architecture
+
+`cluv` is a CLI tool for managing UV-based Python projects across multiple HPC clusters (Mila, DRAC/Narval, etc.).
+
+**Entry point**: `cluv/__main__.py:main()` — builds an `argparse`/`simple_parsing` parser with subcommands, then dispatches to the appropriate async or sync function. Each subcommand registers its args in an `add_<cmd>_args()` function in `__main__.py`, and its implementation lives in `cluv/cli/<cmd>.py`.
+
+**Config**: `cluv/config.py` — reads the `[tool.cluv]` section from the nearest `pyproject.toml`. Key fields: `clusters` (list of hostnames or table with per-cluster settings), `slurm` (global `SBATCH_*` env var defaults), `cluster_configs` (per-cluster `SBATCH_*` overrides). Config is cached with `@functools.cache`.
+
+**SSH connections**: All remote operations go through `milatools.utils.remote_v2.Remote`. Connections reuse existing SSH ControlMaster sockets (checked via `control_socket_is_running_async`) to avoid triggering 2FA prompts. `cluv login` establishes fresh connections sequentially (to avoid concurrent 2FA prompts). The `login.get_remote_without_2fa_prompt()` helper is used by `sync` to only operate on already-connected clusters.
+
+**Async pattern**: Multi-cluster operations use `asyncio.gather` for parallelism. The `sync` command uses `milatools.utils.parallel_progress.run_tasks_with_progress_bar` to display per-cluster progress. The top-level `main()` uses `asyncio.run()` when the subcommand function is a coroutine.
+
+**Rich console**: A single `rich.Console` instance is created in `cluv/utils.py` and patched into `milatools` internals (`milatools.cli.console`, etc.) so all output goes through one stream.
+
+**`cluv status`** (`cluv/cli/status.py`): Currently entirely mock data. The `ClusterStatus` / `JobStats` / `StorageStats` dataclasses define the data model. The `get_mock_cluster_status()` function is the stub to replace with real implementations. The display logic (`_build_cluster_table`, `_build_my_jobs_table`) is separate and should not need changes when real data is wired in.
+
+**`cluv sync`** (`cluv/cli/sync.py`): The most complete command. Runs: `git push` locally → `git clone`/`fetch`/`checkout`/`pull` on each remote → `uv sync` on each remote → optional `rsync` of results back. Each cluster's work is encapsulated in `sync_task_function`.
+
+**`cluv submit`** (`cluv/cli/submit.py`): `cluv submit <cluster> <job.sh> [--no-sync] [sbatch-flags...] [-- program-args...]`. Enforces a clean git tree, injects `GIT_COMMIT`, merges global + per-cluster `SBATCH_*` env vars from config, then runs `sbatch` on the remote. Arguments before `--` are forwarded as sbatch flags; arguments after `--` are passed to the job script.
+
+**`cluv init`** and **`cluv run`**: Not yet implemented (stubs/`NotImplementedError`).
+
+## Cluster notes
+
+- **Mila cluster** is typically the "home base". Detected via `Path("/home/mila").exists()` in `utils.current_cluster()`.
+- **DRAC clusters** (narval, tamia, rorqual, fir, etc.) are detected via `$CC_CLUSTER` env var. They expose a `partition-stats` command that prints a text table of queued/running/idle node counts by partition type (Regular vs GPU) and walltime bucket. See `partiton-stats_output.txt` for a sample.
+- DRAC clusters require a special `uv.toml` pointing to the DRAC wheelhouse for offline installs.
+- Cluster hostnames in `[tool.cluv]` must match the SSH hostnames configured in `~/.ssh/config`.
+
+## Work etiquette
+
+- Make small, clean commits regularly. Each commit should only contain changes related to a single "action" or "theme".
+- Write minimalist, clean, pythonic code. Avoid being overly general or abstract.

cluster_uv-0.0.1.post1/CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# Contributing to cluv
+
+Thank you for your interest in contributing to cluv!
+
+## Issues
+
+Use [GitHub Issues](https://github.com/mila-iqia/cluv/issues) to report bugs or suggest features.
+
+## Development setup
+
+Clone and install project:
+```bash
+git clone https://github.com/mila-iqia/cluv
+cd cluv
+uv sync
+```
+
+Use your local version as the cluv tool:
+```bash
+uv tool install --editable <path_to_cluv_repo>
+```
+
+### Testing
+
+```bash
+uv run pytest
+```
+
+Tests marked `integration` require live SSH connections to real clusters. Skip them locally:
+
+```bash
+uv run pytest -m "not integration"
+```
+
+### Linting
+
+```bash
+uv run pre-commit run --all-files
+```

cluster_uv-0.0.1.post1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mila
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cluster_uv-0.0.1.post1/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: cluster-uv
+Version: 0.0.1.post1
+Summary: cluv (from 'cluster' + 'uv') - A versatile tool to work with uv python projects across HPC clusters.
+Author-email: Fabrice Normandin <normandf@mila.quebec>
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: milatools>=0.1.9
+Requires-Dist: platformdirs>=4.9.6
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: rich-argparse>=1.7.2
+Requires-Dist: rich>=13.0
+Requires-Dist: simple-parsing>=0.1.8
+Description-Content-Type: text/markdown
+
+# cluv
+
+cluv — sync UV-based Python projects across HPC clusters.
+
+## Status
+
+In early development. Commands are functional, but expect bugs or missing features.
+
+## Requirements
+
+- Python >= 3.13
+- [UV](https://docs.astral.sh/uv/)
+- SSH access configured for each cluster in `~/.ssh/config` (run `cluv login` to open ControlMaster sessions)
+- A GitHub repository with your project
+
+## Installation
+
+Install as a UV tool:
+
+```bash
+uv tool install git+https://github.com/mila-iqia/cluv
+```
+
+Then you can run `cluv` directly as a command:
+
+```bash
+cluv init
+cluv login mila
+cluv sync mila
+cluv submit mila job.sh
+```
+
+## Quick Start
+
+1. Initialize your project with:
+   ```bash
+   cluv init
+   ```
+2. Establish SSH connections to all configured clusters:
+   ```bash
+   cluv login
+   ```
+3. Sync your project to all clusters and run `uv sync` on each:
+   ```bash
+   cluv sync
+   ```
+
+## Examples
+
+See the [examples](examples) folder for sample projects using cluv. Each example includes a README with instructions specific to that project.
+
+## Configuration
+
+Add a `[tool.cluv]` section to the `pyproject.toml` of your project. `cluv init` generates a default config, or you can write it by hand.
+See the config at the project root for an example, or refer to the schema below.
+
+### Top-level fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `clusters` | table | Per-cluster settings, keyed by SSH hostname from `~/.ssh/config`. |
+| `env` | table | Global environment variables applied to all clusters. |
+| `results_path` | string | Path relative to the project root for storing results. `cluv sync` rsyncs that directory back from each remote cluster. |
+
+### `[tool.cluv.clusters.<name>.env]`
+Environment variables for a specific cluster. Values here are merged on top of `[tool.cluv.env]` when submitting.
+
+### Variables priority
+Environment variables can be set at multiple levels when submitting jobs, with the following precedence (highest to lowest):
+1. Command-line arguments to `cluv submit`.
+2. Cluster-specific variables in `[tool.cluv.clusters.<name>.env]`
+3. Global variables in `[tool.cluv.env]`
+4. SBATCH directives inside the job script (e.g. `#SBATCH --export=VAR=value`)
+5. Default values from the cluster (e.g. `SBATCH_PARTITION`)
+
+### Example
+
+Here's an example `pyproject.toml` with cluv configuration for three clusters, and some global and cluster-specific environment variables:
+
+```toml
+[tool.cluv]
+results_path = "logs"
+
+[tool.cluv.env]
+SBATCH_TIMELIMIT = "3:00:00"
+WANDB_MODE = "offline"
+
+[tool.cluv.clusters.mila]
+env = { WANDB_MODE="online", SBATCH_PARTITION="long" }
+
+[tool.cluv.clusters.narval]
+
+[tool.cluv.clusters.tamia]
+```
+
+## Commands
+
+### `cluv init`
+
+Initialize the current directory as a cluv project. Must be run from inside your `$HOME` directory.
+
+```
+cluv init
+```
+
+Default project structure after `cluv init`:
+```
+my_project/
+├── README.md
+├── logs -> $SCRATCH/logs/my_project   # symlink to $SCRATCH
+├── pyproject.toml        # includes [tool.cluv] config
+├── scripts/
+│   ├── job.sh            # Slurm job script template
+│   └── safe_job.sh       # Slurm job script template (copies .venv and prior results)
+└── src/
+    └── my_project/
+        └── __init__.py
+```
+
+### `cluv login`
+
+Open SSH ControlMaster connections to all configured clusters. Run this before any command that requires a live connection.
+
+```
+cluv login [<cluster> ...]
+```
+
+### `cluv sync`
+
+Push local git changes, then on each cluster: clone or fetch the repo, check out the current branch, and run `uv sync`. Optionally rsyncs results back if `results_path` is set in the config.
+
+```
+cluv sync [<cluster> ...]
+```
+
+
+### `cluv status`
+
+Display an overview of :
+* Cluster: GPU availability, running/queued jobs and disk usage.
+* Jobs: cached jobs from `cluv submit` with their status.
+
+```
+cluv status [<table>]
+```
+
+### `cluv submit`
+
+Submit a SLURM job on a remote cluster.
+
+```
+cluv submit <cluster> <job.sh> [<sbatch-flags> ...] [-- <program-args> ...]
+```
+
+For example:
+
+```bash
+cluv submit rorqual script/job.sh --time=00:10:00 -- python main.py
+```
+
+### `cluv run`
+
+Sync the project to a cluster, then run a command there with `uv run`.
+
+```
+cluv run <cluster> <command> [<args> ...]
+```