dask-setup 2.1.0__tar.gz → 2.3.0__tar.gz

dask_setup-2.3.0/MANIFEST.in

@@ -0,0 +1,5 @@
+# setuptools only auto-includes tests/test*.py in the sdist, which leaves out
+# conftest.py (the shared fixtures) and __init__.py -- so the suite could not
+# run from the sdist, as downstream packagers like conda-forge do.
+recursive-include tests *.py
+global-exclude __pycache__ *.py[cod] .DS_Store

{dask_setup-2.1.0/src/dask_setup.egg-info → dask_setup-2.3.0}/PKG-INFO

@@ -1,9 +1,19 @@
 Metadata-Version: 2.4
 Name: dask_setup
-Version: 2.1.0
+Version: 2.3.0
 Summary: HPC-tuned Dask helpers for single-node runs on NCI Gadi.
 Author: Sam Green
+License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/21centuryweather/dask_setup
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -11,21 +21,35 @@ Requires-Dist: dask>=2024.1.0
 Requires-Dist: distributed>=2024.1.0
 Requires-Dist: psutil>=5.9
 Requires-Dist: pyyaml>=6.0
+Provides-Extra: multinode
+Requires-Dist: dask-jobqueue>=0.8; extra == "multinode"
 Provides-Extra: dev
-Requires-Dist: ruff~=0.4; extra == "dev"
+Requires-Dist: ruff==0.16.4; extra == "dev"
 Requires-Dist: pytest~=8.2; extra == "dev"
 Requires-Dist: pytest-cov~=5.0; extra == "dev"
 Requires-Dist: xarray; extra == "dev"
 Requires-Dist: numpy; extra == "dev"
+Requires-Dist: dask-jobqueue>=0.8; extra == "dev"
 Dynamic: license-file
 
 # dask_setup
 
-[![CI](https://github.com/21centuryweather/dask_setup/workflows/CI/badge.svg)](https://github.com/21centuryweather/dask_setup/actions)
+[![CI](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml)
+[![Release](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml/badge.svg)](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml)
+[![PyPI](https://img.shields.io/pypi/v/dask_setup.svg)](https://pypi.org/project/dask_setup/)
+[![conda-forge](https://img.shields.io/conda/vn/conda-forge/dask-setup.svg)](https://anaconda.org/conda-forge/dask-setup)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://pypi.org/project/dask_setup/)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
 HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask.distributed.LocalCluster` + `Client` with sensible defaults for single-node jobs, and extends to multi-node PBS/SLURM clusters via `dask-jobqueue` in v2.0.
 
-**Python 3.11+** | `pip install dask-setup`
+```bash
+pip install dask-setup              # single-node
+pip install dask-setup[multinode]   # + PBS/SLURM via dask-jobqueue
+
+conda install -c conda-forge dask-setup   # or via conda-forge
+```
 
 ---
 
@@ -35,9 +59,9 @@ HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask
 from dask_setup import setup_dask_client
 
 # Pick a workload type and go
-client, cluster, dask_tmp = setup_dask_client("cpu")   # heavy compute
-client, cluster, dask_tmp = setup_dask_client("io")    # heavy file I/O
-client, cluster, dask_tmp = setup_dask_client("mixed") # both
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="cpu")   # heavy compute
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # Zarr / object storage
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="mixed") # both
 ```
 
 `dask_tmp` is the path to the spill/temp directory (on `$PBS_JOBFS` if available). Pass it to Rechunker, Zarr, or anywhere else you want fast local I/O.
@@ -81,12 +105,35 @@ client, cluster, shared_tmp = setup_dask_client(
 
 | Type | Topology | Best for |
 |------|----------|----------|
-| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions |
-| `"io"` | 1 process, 8–16 threads | Opening many NetCDF/Zarr files concurrently |
+| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions, **and reading NetCDF/HDF5** |
+| `"io"` | 1 process, 8–16 threads | Zarr, object storage (S3), HTTP — libraries that release the GIL |
 | `"mixed"` | Processes with 2 threads each | Pipelines that both read and compute |
 | `"gpu"` | 1 process per GPU, up to 8 threads | CuPy/RAPIDS CUDA workloads |
 | `"auto"` | Inferred from dataset | Let `dask_setup` decide based on your data |
 
+### Reading NetCDF? Use `"cpu"`, not `"io"`
+
+The split is not "I/O vs compute" — it is **whether the library releases the GIL
+and is thread-safe**. `"io"` runs one process with many threads, which only wins
+when threads can genuinely overlap.
+
+HDF5 (underneath NetCDF4) is usually not built thread-safe, so xarray routes
+*every* NetCDF read through a single process-wide lock. Under `"io"` your threads
+do not become parallel readers — they queue on that one lock, one at a time, and
+zlib decompression is serialised inside it too. Separate processes each get their
+own lock, so `"cpu"` is what actually parallelises NetCDF reads.
+
+Zarr is the opposite: it decompresses via numcodecs, which releases the GIL, and
+has no global lock, so threads scale well. Opening, concatenating and writing
+NetCDF is a `"cpu"` job even though it feels like I/O.
+
+> **Rule of thumb: NetCDF in → `"cpu"`. Zarr or object storage in → `"io"`.**
+> Pass `ds=` and `dask_setup` will warn if you get this pair the wrong way round.
+>
+> Also avoid `open_mfdataset(..., parallel=True)` together with `"io"`: concurrent
+> metadata reads can kill the worker outright, after which Dask silently
+> recomputes the lost tasks and the job appears to hang rather than fail.
+
 ---
 
 ## Key Parameters
@@ -95,14 +142,27 @@ client, cluster, shared_tmp = setup_dask_client(
 |-----------|---------|-------------|
 | `workload_type` | `"io"` | Worker topology: `"cpu"`, `"io"`, `"mixed"`, `"gpu"`, `"auto"` |
 | `max_workers` | all cores | Hard cap on worker count |
-| `reserve_mem_gb` | auto (20% RAM) | Memory held back for OS/cache (GiB) |
+| `reserve_mem_gb` | auto | Memory held back for OS/cache (GiB): 20% of RAM, clamped to [4, 50] |
 | `max_mem_gb` | total RAM | Upper bound on Dask's total memory use |
 | `dashboard` | `True` | Start dashboard and print SSH tunnel hint |
 | `profile` | `None` | Named config profile |
-| `config` | `None` | Pre-built `DaskSetupConfig` object — mutually exclusive with `profile` |
-| `mode` | `"auto"` | `"local"`, `"pbs"`, `"slurm"`, or `"auto"` (v2.0) |
+| `config` | `None` | Pre-built `DaskSetupConfig` object — same layer as `profile`; if both are given, `profile` wins |
+| `mode` | `"interactive"` | `"interactive"` uses the current allocation and never submits jobs (a single node or no job at all is a plain `LocalCluster`); `"auto"` picks `"pbs"`/`"slurm"` in batch jobs, which submit worker jobs; also `"local"`, `"pbs"`, `"slurm"` |
 | `multi_node_config` | `None` | `MultiNodeConfig` for PBS/SLURM multi-node jobs (v2.0) |
 
+Settings are layered, lowest to highest:
+
+```
+library defaults  <  config= or profile=  <  explicit keyword arguments
+```
+
+A parameter you leave unset inherits from the layer below. Passing a value
+always overrides, even when that value happens to equal the default — so
+`reserve_mem_gb=50.0` overrides a profile that says 40.0.
+
+`config=` and `profile=` share a layer rather than stacking: pass one or the
+other, and use explicit keyword arguments for the differences.
+
 ---
 
 ## Common Patterns
@@ -177,6 +237,14 @@ Tunnel from your laptop:
 Then open: http://localhost:8787
 ```
 
+The login host is inferred from the compute node's DNS domain, so it is correct
+at other sites too. Override it with `$DASK_SETUP_LOGIN_HOST` if the guess is
+wrong:
+
+```bash
+export DASK_SETUP_LOGIN_HOST=login.mycluster.edu
+```
+
 ---
 
 ## CLI
@@ -214,19 +282,30 @@ Full documentation lives in the [GitHub wiki](https://github.com/21centuryweathe
 | [Benchmarking](https://github.com/21centuryweather/dask_setup/wiki/Benchmarking) | `benchmark_config`, `scaling_analysis`, `chunk_impact`, `dask-setup benchmark` |
 | [Internals](https://github.com/21centuryweather/dask_setup/wiki/Internals) | Resource detection, topology decisions, temp/spill routing, module layout |
 | [Troubleshooting](https://github.com/21centuryweather/dask_setup/wiki/Troubleshooting) | Common errors, OOM, multi-node issues, migration guide |
+| [User Feedback](https://github.com/21centuryweather/dask_setup/wiki/User-Feedback) | Questions from users, the answers, and what changed as a result |
 
 ---
 
 ## Installation
 
+From PyPI:
+
 ```bash
 pip install dask-setup
 ```
 
+From conda-forge:
+
+```bash
+conda install -c conda-forge dask-setup
+```
+
 For multi-node PBS/SLURM support:
 
 ```bash
 pip install dask-setup dask-jobqueue
+# or
+conda install -c conda-forge dask-setup dask-jobqueue
 ```
 
 For GPU workloads (CuPy auto-detection):

dask_setup-2.1.0/PKG-INFO → dask_setup-2.3.0/README.md

@@ -1,31 +1,21 @@
-Metadata-Version: 2.4
-Name: dask_setup
-Version: 2.1.0
-Summary: HPC-tuned Dask helpers for single-node runs on NCI Gadi.
-Author: Sam Green
-Project-URL: Homepage, https://github.com/21centuryweather/dask_setup
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: dask>=2024.1.0
-Requires-Dist: distributed>=2024.1.0
-Requires-Dist: psutil>=5.9
-Requires-Dist: pyyaml>=6.0
-Provides-Extra: dev
-Requires-Dist: ruff~=0.4; extra == "dev"
-Requires-Dist: pytest~=8.2; extra == "dev"
-Requires-Dist: pytest-cov~=5.0; extra == "dev"
-Requires-Dist: xarray; extra == "dev"
-Requires-Dist: numpy; extra == "dev"
-Dynamic: license-file
-
 # dask_setup
 
-[![CI](https://github.com/21centuryweather/dask_setup/workflows/CI/badge.svg)](https://github.com/21centuryweather/dask_setup/actions)
+[![CI](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml)
+[![Release](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml/badge.svg)](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml)
+[![PyPI](https://img.shields.io/pypi/v/dask_setup.svg)](https://pypi.org/project/dask_setup/)
+[![conda-forge](https://img.shields.io/conda/vn/conda-forge/dask-setup.svg)](https://anaconda.org/conda-forge/dask-setup)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://pypi.org/project/dask_setup/)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
 HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask.distributed.LocalCluster` + `Client` with sensible defaults for single-node jobs, and extends to multi-node PBS/SLURM clusters via `dask-jobqueue` in v2.0.
 
-**Python 3.11+** | `pip install dask-setup`
+```bash
+pip install dask-setup              # single-node
+pip install dask-setup[multinode]   # + PBS/SLURM via dask-jobqueue
+
+conda install -c conda-forge dask-setup   # or via conda-forge
+```
 
 ---
 
@@ -35,9 +25,9 @@ HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask
 from dask_setup import setup_dask_client
 
 # Pick a workload type and go
-client, cluster, dask_tmp = setup_dask_client("cpu")   # heavy compute
-client, cluster, dask_tmp = setup_dask_client("io")    # heavy file I/O
-client, cluster, dask_tmp = setup_dask_client("mixed") # both
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="cpu")   # heavy compute
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # Zarr / object storage
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="mixed") # both
 ```
 
 `dask_tmp` is the path to the spill/temp directory (on `$PBS_JOBFS` if available). Pass it to Rechunker, Zarr, or anywhere else you want fast local I/O.
@@ -81,12 +71,35 @@ client, cluster, shared_tmp = setup_dask_client(
 
 | Type | Topology | Best for |
 |------|----------|----------|
-| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions |
-| `"io"` | 1 process, 8–16 threads | Opening many NetCDF/Zarr files concurrently |
+| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions, **and reading NetCDF/HDF5** |
+| `"io"` | 1 process, 8–16 threads | Zarr, object storage (S3), HTTP — libraries that release the GIL |
 | `"mixed"` | Processes with 2 threads each | Pipelines that both read and compute |
 | `"gpu"` | 1 process per GPU, up to 8 threads | CuPy/RAPIDS CUDA workloads |
 | `"auto"` | Inferred from dataset | Let `dask_setup` decide based on your data |
 
+### Reading NetCDF? Use `"cpu"`, not `"io"`
+
+The split is not "I/O vs compute" — it is **whether the library releases the GIL
+and is thread-safe**. `"io"` runs one process with many threads, which only wins
+when threads can genuinely overlap.
+
+HDF5 (underneath NetCDF4) is usually not built thread-safe, so xarray routes
+*every* NetCDF read through a single process-wide lock. Under `"io"` your threads
+do not become parallel readers — they queue on that one lock, one at a time, and
+zlib decompression is serialised inside it too. Separate processes each get their
+own lock, so `"cpu"` is what actually parallelises NetCDF reads.
+
+Zarr is the opposite: it decompresses via numcodecs, which releases the GIL, and
+has no global lock, so threads scale well. Opening, concatenating and writing
+NetCDF is a `"cpu"` job even though it feels like I/O.
+
+> **Rule of thumb: NetCDF in → `"cpu"`. Zarr or object storage in → `"io"`.**
+> Pass `ds=` and `dask_setup` will warn if you get this pair the wrong way round.
+>
+> Also avoid `open_mfdataset(..., parallel=True)` together with `"io"`: concurrent
+> metadata reads can kill the worker outright, after which Dask silently
+> recomputes the lost tasks and the job appears to hang rather than fail.
+
 ---
 
 ## Key Parameters
@@ -95,14 +108,27 @@ client, cluster, shared_tmp = setup_dask_client(
 |-----------|---------|-------------|
 | `workload_type` | `"io"` | Worker topology: `"cpu"`, `"io"`, `"mixed"`, `"gpu"`, `"auto"` |
 | `max_workers` | all cores | Hard cap on worker count |
-| `reserve_mem_gb` | auto (20% RAM) | Memory held back for OS/cache (GiB) |
+| `reserve_mem_gb` | auto | Memory held back for OS/cache (GiB): 20% of RAM, clamped to [4, 50] |
 | `max_mem_gb` | total RAM | Upper bound on Dask's total memory use |
 | `dashboard` | `True` | Start dashboard and print SSH tunnel hint |
 | `profile` | `None` | Named config profile |
-| `config` | `None` | Pre-built `DaskSetupConfig` object — mutually exclusive with `profile` |
-| `mode` | `"auto"` | `"local"`, `"pbs"`, `"slurm"`, or `"auto"` (v2.0) |
+| `config` | `None` | Pre-built `DaskSetupConfig` object — same layer as `profile`; if both are given, `profile` wins |
+| `mode` | `"interactive"` | `"interactive"` uses the current allocation and never submits jobs (a single node or no job at all is a plain `LocalCluster`); `"auto"` picks `"pbs"`/`"slurm"` in batch jobs, which submit worker jobs; also `"local"`, `"pbs"`, `"slurm"` |
 | `multi_node_config` | `None` | `MultiNodeConfig` for PBS/SLURM multi-node jobs (v2.0) |
 
+Settings are layered, lowest to highest:
+
+```
+library defaults  <  config= or profile=  <  explicit keyword arguments
+```
+
+A parameter you leave unset inherits from the layer below. Passing a value
+always overrides, even when that value happens to equal the default — so
+`reserve_mem_gb=50.0` overrides a profile that says 40.0.
+
+`config=` and `profile=` share a layer rather than stacking: pass one or the
+other, and use explicit keyword arguments for the differences.
+
 ---
 
 ## Common Patterns
@@ -177,6 +203,14 @@ Tunnel from your laptop:
 Then open: http://localhost:8787
 ```
 
+The login host is inferred from the compute node's DNS domain, so it is correct
+at other sites too. Override it with `$DASK_SETUP_LOGIN_HOST` if the guess is
+wrong:
+
+```bash
+export DASK_SETUP_LOGIN_HOST=login.mycluster.edu
+```
+
 ---
 
 ## CLI
@@ -214,19 +248,30 @@ Full documentation lives in the [GitHub wiki](https://github.com/21centuryweathe
 | [Benchmarking](https://github.com/21centuryweather/dask_setup/wiki/Benchmarking) | `benchmark_config`, `scaling_analysis`, `chunk_impact`, `dask-setup benchmark` |
 | [Internals](https://github.com/21centuryweather/dask_setup/wiki/Internals) | Resource detection, topology decisions, temp/spill routing, module layout |
 | [Troubleshooting](https://github.com/21centuryweather/dask_setup/wiki/Troubleshooting) | Common errors, OOM, multi-node issues, migration guide |
+| [User Feedback](https://github.com/21centuryweather/dask_setup/wiki/User-Feedback) | Questions from users, the answers, and what changed as a result |
 
 ---
 
 ## Installation
 
+From PyPI:
+
 ```bash
 pip install dask-setup
 ```
 
+From conda-forge:
+
+```bash
+conda install -c conda-forge dask-setup
+```
+
 For multi-node PBS/SLURM support:
 
 ```bash
 pip install dask-setup dask-jobqueue
+# or
+conda install -c conda-forge dask-setup dask-jobqueue
 ```
 
 For GPU workloads (CuPy auto-detection):

{dask_setup-2.1.0 → dask_setup-2.3.0}/pyproject.toml

@@ -1,14 +1,27 @@
 [build-system]
-requires = ["setuptools>=64", "wheel"]
+requires = ["setuptools>=77", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "dask_setup"
-version = "2.1.0"
+version = "2.3.0"
 description = "HPC-tuned Dask helpers for single-node runs on NCI Gadi."
 authors = [{name="Sam Green"}]
 readme = "README.md"
 requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering",
+    "Topic :: System :: Distributed Computing",
+]
 dependencies = [
     "dask>=2024.1.0",
     "distributed>=2024.1.0",
@@ -23,13 +36,28 @@ Homepage = "https://github.com/21centuryweather/dask_setup"
 dask-setup = "dask_setup.cli:main"
 
 [project.optional-dependencies]
+# Multi-node PBS/SLURM cluster support. Optional: the single-node path never
+# imports it, and setup_pbs_cluster()/setup_slurm_cluster() raise a helpful
+# ImportError if it is missing.
+multinode = [
+    "dask-jobqueue>=0.8",
+]
+
 dev = [
-    "ruff ~= 0.4",
+    # Pinned exactly, not a range. `ruff ~= 0.4` means >=0.4,<1.0, so CI picked
+    # up whatever ruff had released that morning -- and ruff's formatter changes
+    # between minor versions. That turns `ruff format --check` into a job that
+    # can fail on a commit that touched nothing. Bump this deliberately.
+    "ruff == 0.16.4",
     "pytest ~= 8.2",
     "pytest-cov ~= 5.0",
     # Xarray integration testing
     "xarray",
     "numpy",
+    # Multi-node tests exercise a real PBSJob. Without this, CI silently skipped
+    # the multi-node path -- including the regression test for the 4x worker
+    # under-provisioning fixed in 2.2.0.
+    "dask-jobqueue>=0.8",
 ]
 
 [tool.setuptools.package-data]

{dask_setup-2.1.0 → dask_setup-2.3.0}/src/dask_setup/__init__.py

@@ -281,7 +281,7 @@ except ImportError:
         )
 
 
-__version__ = "2.1.0"
+__version__ = "2.3.0"
 
 __all__ = [
     # Core API — always available