dask-setup 2.2.0__tar.gz → 2.3.0__tar.gz

dask_setup-2.3.0/MANIFEST.in

@@ -0,0 +1,5 @@
+# setuptools only auto-includes tests/test*.py in the sdist, which leaves out
+# conftest.py (the shared fixtures) and __init__.py -- so the suite could not
+# run from the sdist, as downstream packagers like conda-forge do.
+recursive-include tests *.py
+global-exclude __pycache__ *.py[cod] .DS_Store

{dask_setup-2.2.0/src/dask_setup.egg-info → dask_setup-2.3.0}/PKG-INFO

@@ -1,9 +1,19 @@
 Metadata-Version: 2.4
 Name: dask_setup
-Version: 2.2.0
+Version: 2.3.0
 Summary: HPC-tuned Dask helpers for single-node runs on NCI Gadi.
 Author: Sam Green
+License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/21centuryweather/dask_setup
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -24,11 +34,22 @@ Dynamic: license-file
 
 # dask_setup
 
-[![CI](https://github.com/21centuryweather/dask_setup/workflows/CI/badge.svg)](https://github.com/21centuryweather/dask_setup/actions)
+[![CI](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml)
+[![Release](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml/badge.svg)](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml)
+[![PyPI](https://img.shields.io/pypi/v/dask_setup.svg)](https://pypi.org/project/dask_setup/)
+[![conda-forge](https://img.shields.io/conda/vn/conda-forge/dask-setup.svg)](https://anaconda.org/conda-forge/dask-setup)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://pypi.org/project/dask_setup/)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
 HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask.distributed.LocalCluster` + `Client` with sensible defaults for single-node jobs, and extends to multi-node PBS/SLURM clusters via `dask-jobqueue` in v2.0.
 
-**Python 3.11+** | `pip install dask-setup`
+```bash
+pip install dask-setup              # single-node
+pip install dask-setup[multinode]   # + PBS/SLURM via dask-jobqueue
+
+conda install -c conda-forge dask-setup   # or via conda-forge
+```
 
 ---
 
@@ -39,7 +60,7 @@ from dask_setup import setup_dask_client
 
 # Pick a workload type and go
 client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="cpu")   # heavy compute
-client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # heavy file I/O
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # Zarr / object storage
 client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="mixed") # both
 ```
 
@@ -84,12 +105,35 @@ client, cluster, shared_tmp = setup_dask_client(
 
 | Type | Topology | Best for |
 |------|----------|----------|
-| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions |
-| `"io"` | 1 process, 8–16 threads | Opening many NetCDF/Zarr files concurrently |
+| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions, **and reading NetCDF/HDF5** |
+| `"io"` | 1 process, 8–16 threads | Zarr, object storage (S3), HTTP — libraries that release the GIL |
 | `"mixed"` | Processes with 2 threads each | Pipelines that both read and compute |
 | `"gpu"` | 1 process per GPU, up to 8 threads | CuPy/RAPIDS CUDA workloads |
 | `"auto"` | Inferred from dataset | Let `dask_setup` decide based on your data |
 
+### Reading NetCDF? Use `"cpu"`, not `"io"`
+
+The split is not "I/O vs compute" — it is **whether the library releases the GIL
+and is thread-safe**. `"io"` runs one process with many threads, which only wins
+when threads can genuinely overlap.
+
+HDF5 (underneath NetCDF4) is usually not built thread-safe, so xarray routes
+*every* NetCDF read through a single process-wide lock. Under `"io"` your threads
+do not become parallel readers — they queue on that one lock, one at a time, and
+zlib decompression is serialised inside it too. Separate processes each get their
+own lock, so `"cpu"` is what actually parallelises NetCDF reads.
+
+Zarr is the opposite: it decompresses via numcodecs, which releases the GIL, and
+has no global lock, so threads scale well. Opening, concatenating and writing
+NetCDF is a `"cpu"` job even though it feels like I/O.
+
+> **Rule of thumb: NetCDF in → `"cpu"`. Zarr or object storage in → `"io"`.**
+> Pass `ds=` and `dask_setup` will warn if you get this pair the wrong way round.
+>
+> Also avoid `open_mfdataset(..., parallel=True)` together with `"io"`: concurrent
+> metadata reads can kill the worker outright, after which Dask silently
+> recomputes the lost tasks and the job appears to hang rather than fail.
+
 ---
 
 ## Key Parameters
@@ -103,7 +147,7 @@ client, cluster, shared_tmp = setup_dask_client(
 | `dashboard` | `True` | Start dashboard and print SSH tunnel hint |
 | `profile` | `None` | Named config profile |
 | `config` | `None` | Pre-built `DaskSetupConfig` object — same layer as `profile`; if both are given, `profile` wins |
-| `mode` | `"auto"` | `"local"`, `"pbs"`, `"slurm"`, or `"auto"` (v2.0) |
+| `mode` | `"interactive"` | `"interactive"` uses the current allocation and never submits jobs (a single node or no job at all is a plain `LocalCluster`); `"auto"` picks `"pbs"`/`"slurm"` in batch jobs, which submit worker jobs; also `"local"`, `"pbs"`, `"slurm"` |
 | `multi_node_config` | `None` | `MultiNodeConfig` for PBS/SLURM multi-node jobs (v2.0) |
 
 Settings are layered, lowest to highest:
@@ -238,19 +282,30 @@ Full documentation lives in the [GitHub wiki](https://github.com/21centuryweathe
 | [Benchmarking](https://github.com/21centuryweather/dask_setup/wiki/Benchmarking) | `benchmark_config`, `scaling_analysis`, `chunk_impact`, `dask-setup benchmark` |
 | [Internals](https://github.com/21centuryweather/dask_setup/wiki/Internals) | Resource detection, topology decisions, temp/spill routing, module layout |
 | [Troubleshooting](https://github.com/21centuryweather/dask_setup/wiki/Troubleshooting) | Common errors, OOM, multi-node issues, migration guide |
+| [User Feedback](https://github.com/21centuryweather/dask_setup/wiki/User-Feedback) | Questions from users, the answers, and what changed as a result |
 
 ---
 
 ## Installation
 
+From PyPI:
+
 ```bash
 pip install dask-setup
 ```
 
+From conda-forge:
+
+```bash
+conda install -c conda-forge dask-setup
+```
+
 For multi-node PBS/SLURM support:
 
 ```bash
 pip install dask-setup dask-jobqueue
+# or
+conda install -c conda-forge dask-setup dask-jobqueue
 ```
 
 For GPU workloads (CuPy auto-detection):

dask_setup-2.2.0/PKG-INFO → dask_setup-2.3.0/README.md

@@ -1,34 +1,21 @@
-Metadata-Version: 2.4
-Name: dask_setup
-Version: 2.2.0
-Summary: HPC-tuned Dask helpers for single-node runs on NCI Gadi.
-Author: Sam Green
-Project-URL: Homepage, https://github.com/21centuryweather/dask_setup
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: dask>=2024.1.0
-Requires-Dist: distributed>=2024.1.0
-Requires-Dist: psutil>=5.9
-Requires-Dist: pyyaml>=6.0
-Provides-Extra: multinode
-Requires-Dist: dask-jobqueue>=0.8; extra == "multinode"
-Provides-Extra: dev
-Requires-Dist: ruff==0.16.4; extra == "dev"
-Requires-Dist: pytest~=8.2; extra == "dev"
-Requires-Dist: pytest-cov~=5.0; extra == "dev"
-Requires-Dist: xarray; extra == "dev"
-Requires-Dist: numpy; extra == "dev"
-Requires-Dist: dask-jobqueue>=0.8; extra == "dev"
-Dynamic: license-file
-
 # dask_setup
 
-[![CI](https://github.com/21centuryweather/dask_setup/workflows/CI/badge.svg)](https://github.com/21centuryweather/dask_setup/actions)
+[![CI](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/21centuryweather/dask_setup/actions/workflows/ci.yml)
+[![Release](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml/badge.svg)](https://github.com/21centuryweather/dask_setup/actions/workflows/publish-to-pypi.yml)
+[![PyPI](https://img.shields.io/pypi/v/dask_setup.svg)](https://pypi.org/project/dask_setup/)
+[![conda-forge](https://img.shields.io/conda/vn/conda-forge/dask-setup.svg)](https://anaconda.org/conda-forge/dask-setup)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://pypi.org/project/dask_setup/)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
 HPC-tuned Dask helpers for **NCI Gadi** and other PBS/SLURM systems. Wraps `dask.distributed.LocalCluster` + `Client` with sensible defaults for single-node jobs, and extends to multi-node PBS/SLURM clusters via `dask-jobqueue` in v2.0.
 
-**Python 3.11+** | `pip install dask-setup`
+```bash
+pip install dask-setup              # single-node
+pip install dask-setup[multinode]   # + PBS/SLURM via dask-jobqueue
+
+conda install -c conda-forge dask-setup   # or via conda-forge
+```
 
 ---
 
@@ -39,7 +26,7 @@ from dask_setup import setup_dask_client
 
 # Pick a workload type and go
 client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="cpu")   # heavy compute
-client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # heavy file I/O
+client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="io")    # Zarr / object storage
 client, cluster, dask_tmp = setup_dask_client(mode="interactive", workload_type="mixed") # both
 ```
 
@@ -84,12 +71,35 @@ client, cluster, shared_tmp = setup_dask_client(
 
 | Type | Topology | Best for |
 |------|----------|----------|
-| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions |
-| `"io"` | 1 process, 8–16 threads | Opening many NetCDF/Zarr files concurrently |
+| `"cpu"` | Many processes, 1 thread each | NumPy/Numba math, xarray reductions, **and reading NetCDF/HDF5** |
+| `"io"` | 1 process, 8–16 threads | Zarr, object storage (S3), HTTP — libraries that release the GIL |
 | `"mixed"` | Processes with 2 threads each | Pipelines that both read and compute |
 | `"gpu"` | 1 process per GPU, up to 8 threads | CuPy/RAPIDS CUDA workloads |
 | `"auto"` | Inferred from dataset | Let `dask_setup` decide based on your data |
 
+### Reading NetCDF? Use `"cpu"`, not `"io"`
+
+The split is not "I/O vs compute" — it is **whether the library releases the GIL
+and is thread-safe**. `"io"` runs one process with many threads, which only wins
+when threads can genuinely overlap.
+
+HDF5 (underneath NetCDF4) is usually not built thread-safe, so xarray routes
+*every* NetCDF read through a single process-wide lock. Under `"io"` your threads
+do not become parallel readers — they queue on that one lock, one at a time, and
+zlib decompression is serialised inside it too. Separate processes each get their
+own lock, so `"cpu"` is what actually parallelises NetCDF reads.
+
+Zarr is the opposite: it decompresses via numcodecs, which releases the GIL, and
+has no global lock, so threads scale well. Opening, concatenating and writing
+NetCDF is a `"cpu"` job even though it feels like I/O.
+
+> **Rule of thumb: NetCDF in → `"cpu"`. Zarr or object storage in → `"io"`.**
+> Pass `ds=` and `dask_setup` will warn if you get this pair the wrong way round.
+>
+> Also avoid `open_mfdataset(..., parallel=True)` together with `"io"`: concurrent
+> metadata reads can kill the worker outright, after which Dask silently
+> recomputes the lost tasks and the job appears to hang rather than fail.
+
 ---
 
 ## Key Parameters
@@ -103,7 +113,7 @@ client, cluster, shared_tmp = setup_dask_client(
 | `dashboard` | `True` | Start dashboard and print SSH tunnel hint |
 | `profile` | `None` | Named config profile |
 | `config` | `None` | Pre-built `DaskSetupConfig` object — same layer as `profile`; if both are given, `profile` wins |
-| `mode` | `"auto"` | `"local"`, `"pbs"`, `"slurm"`, or `"auto"` (v2.0) |
+| `mode` | `"interactive"` | `"interactive"` uses the current allocation and never submits jobs (a single node or no job at all is a plain `LocalCluster`); `"auto"` picks `"pbs"`/`"slurm"` in batch jobs, which submit worker jobs; also `"local"`, `"pbs"`, `"slurm"` |
 | `multi_node_config` | `None` | `MultiNodeConfig` for PBS/SLURM multi-node jobs (v2.0) |
 
 Settings are layered, lowest to highest:
@@ -238,19 +248,30 @@ Full documentation lives in the [GitHub wiki](https://github.com/21centuryweathe
 | [Benchmarking](https://github.com/21centuryweather/dask_setup/wiki/Benchmarking) | `benchmark_config`, `scaling_analysis`, `chunk_impact`, `dask-setup benchmark` |
 | [Internals](https://github.com/21centuryweather/dask_setup/wiki/Internals) | Resource detection, topology decisions, temp/spill routing, module layout |
 | [Troubleshooting](https://github.com/21centuryweather/dask_setup/wiki/Troubleshooting) | Common errors, OOM, multi-node issues, migration guide |
+| [User Feedback](https://github.com/21centuryweather/dask_setup/wiki/User-Feedback) | Questions from users, the answers, and what changed as a result |
 
 ---
 
 ## Installation
 
+From PyPI:
+
 ```bash
 pip install dask-setup
 ```
 
+From conda-forge:
+
+```bash
+conda install -c conda-forge dask-setup
+```
+
 For multi-node PBS/SLURM support:
 
 ```bash
 pip install dask-setup dask-jobqueue
+# or
+conda install -c conda-forge dask-setup dask-jobqueue
 ```
 
 For GPU workloads (CuPy auto-detection):

{dask_setup-2.2.0 → dask_setup-2.3.0}/pyproject.toml

@@ -1,14 +1,27 @@
 [build-system]
-requires = ["setuptools>=64", "wheel"]
+requires = ["setuptools>=77", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "dask_setup"
-version = "2.2.0"
+version = "2.3.0"
 description = "HPC-tuned Dask helpers for single-node runs on NCI Gadi."
 authors = [{name="Sam Green"}]
 readme = "README.md"
 requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering",
+    "Topic :: System :: Distributed Computing",
+]
 dependencies = [
     "dask>=2024.1.0",
     "distributed>=2024.1.0",

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/__init__.py

@@ -281,7 +281,7 @@ except ImportError:
         )
 
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"
 
 __all__ = [
     # Core API — always available

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/benchmark.py

@@ -47,6 +47,7 @@ from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
 from .logging import get_logger
+from .reporting import scheduler_workers
 
 if TYPE_CHECKING:
     from dask.distributed import Client
@@ -561,7 +562,7 @@ def _measure_one(
 
     # Worker count
     with contextlib.suppress(Exception):
-        n_workers = len(client.scheduler_info().get("workers", {}))
+        n_workers = len(scheduler_workers(client))
 
     # Optional warmup (not sampled -- it is not part of the measured run)
     if warmup:
@@ -1286,7 +1287,7 @@ def run_synthetic_benchmark(
             dashboard=False,
         )
 
-        n_workers = len(client.scheduler_info().get("workers", {}))
+        n_workers = len(scheduler_workers(client))
 
         if verbose:
             print(f"Cluster ready: {n_workers} workers. Running {operation!r} × {repeats} …")

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/client.py

@@ -25,6 +25,7 @@ from .logging import get_logger
 from .multinode import (
     MultiNodeConfig,
     detect_cluster_mode,
+    discover_allocated_nodes,
     setup_interactive_cluster,
     setup_pbs_cluster,
     setup_slurm_cluster,
@@ -82,6 +83,43 @@ def _compute_smart_reserve_default() -> float:
         return 50.0  # Safe HPC fallback if psutil is unexpectedly unavailable
 
 
+def _warn_if_threaded_topology_reads_netcdf(ds: Any, workload_type: str) -> bool:
+    """Warn when ``workload_type="io"`` is paired with NetCDF/HDF5 input.
+
+    ``"io"`` runs one process with many threads, which only pays off when the
+    reading library releases the GIL.  HDF5 -- underneath NetCDF4 -- is usually
+    not built thread-safe, so xarray funnels every read through a single
+    process-wide lock: the threads queue on it instead of reading in parallel,
+    and decompression is serialised inside it too.  ``"cpu"`` gives each process
+    its own lock and is typically far faster on the same data.
+
+    Returns ``True`` if a warning was emitted, so callers and tests can tell.
+    """
+    if workload_type != "io":
+        return False
+
+    try:
+        from .xarray import detect_storage_format
+
+        storage = detect_storage_format(ds)
+    except Exception as e:  # pragma: no cover - never block setup on a hint
+        logger.debug("Could not determine dataset storage format", error=str(e))
+        return False
+
+    if storage != "netcdf":
+        return False
+
+    logger.warning(
+        "workload_type='io' is usually the wrong choice for NetCDF/HDF5 input",
+        reason="HDF5 is not thread-safe, so xarray serialises every read through "
+        "one process-wide lock; the threads queue rather than read in parallel",
+        suggestion="use workload_type='cpu' (one lock per process) for NetCDF",
+        note="also avoid open_mfdataset(..., parallel=True) with 'io' -- concurrent "
+        "metadata reads can kill the worker and Dask's silent retries look like a hang",
+    )
+    return True
+
+
 def _chunk_recommendations_for(
     ds: Any,
     client: Client,
@@ -458,7 +496,7 @@ def setup_dask_client(
     ds: Any = None,  # xr.Dataset | xr.DataArray | None
     fallback_on_detection_failure: bool = False,
     adaptive_memory: bool = False,
-    mode: str = "auto",
+    mode: str = "interactive",
     multi_node_config: MultiNodeConfig | None = None,
 ) -> tuple[Client, LocalCluster, str] | tuple[Client, LocalCluster, str, dict[str, int]]:
     """Create a single-node Dask LocalCluster tuned for HPC login/compute nodes.
@@ -542,21 +580,22 @@ def setup_dask_client(
         and tightens the worker ``memory.target`` / ``memory.spill`` thresholds
         slightly, giving workers more head-room from the start.  Default
         ``False``.
-    mode : {"auto", "local", "pbs", "slurm", "interactive"}
+    mode : {"interactive", "auto", "local", "pbs", "slurm"}
         Backend selection.
 
+        - ``"interactive"`` (default) — use resources already allocated in
+          the current PBS or SLURM job, interactive (``qsub -I``,
+          ``salloc``) or batch. Never submits new jobs. Multi-node
+          allocations create an ``SSHCluster`` across all nodes in
+          ``PBS_NODEFILE`` / ``SLURM_NODELIST``; a single node, or no
+          allocation at all, gets the same ``LocalCluster`` as ``"local"``.
         - ``"local"`` — always use a single-node ``LocalCluster`` (default
           behaviour prior to v2.0).
         - ``"pbs"`` — launch via ``dask-jobqueue.PBSCluster`` (submits new
           batch jobs).  Requires ``pip install dask-jobqueue``.
         - ``"slurm"`` — launch via ``dask-jobqueue.SLURMCluster`` (submits
           new batch jobs).
-        - ``"interactive"`` — use resources already allocated in the current
-          interactive PBS (``qsub -I``) or SLURM (``salloc``) session.
-          Single-node allocations create a ``LocalCluster``; multi-node
-          allocations create an ``SSHCluster`` across all nodes in
-          ``PBS_NODEFILE`` / ``SLURM_NODELIST``.
-        - ``"auto"`` (default) — inspect the environment and choose
+        - ``"auto"`` — inspect the environment and choose
           ``"interactive"`` when inside a PBS interactive job
           (``PBS_ENVIRONMENT=PBS_INTERACTIVE``) or a SLURM interactive
           allocation (``SLURM_BATCH_FLAG`` not set to ``"1"``);
@@ -610,7 +649,7 @@ def setup_dask_client(
         client, cluster, tmp, chunks = setup_dask_client(ds=ds, suggest_chunks=True)
         ds_opt = ds.chunk(chunks)
 
-        # Multi-node PBS — auto-detects from environment
+        # Multi-node PBS — submits worker jobs via dask-jobqueue
         client, cluster, tmp = setup_dask_client(
             mode="pbs",
             multi_node_config=MultiNodeConfig(
@@ -634,6 +673,14 @@ def setup_dask_client(
         resolved_mode = detect_cluster_mode()
         logger.debug("Mode auto-resolved", mode=resolved_mode)
 
+    # Interactive on one node (or none — a laptop, a login node) is just a
+    # LocalCluster, so take the local path directly. Going through
+    # setup_interactive_cluster() would drop ds-based workload inference,
+    # fallback_on_detection_failure and adaptive_memory.
+    if resolved_mode == "interactive" and len(discover_allocated_nodes()) <= 1:
+        resolved_mode = "local"
+        logger.debug("Interactive mode with at most one allocated node — using local path")
+
     # --- Profile auto-selection -----------------------------------------
     # Must happen before _resolve_configuration so the selected profile name
     # can be passed in. Requires a preliminary resource detection pass.
@@ -697,18 +744,13 @@ def setup_dask_client(
         # Chunk recommendations need a live client, which we now have — so
         # the ds= contract (4-tuple) holds on these paths too.
         if ds is not None:
+            _warn_if_threaded_topology_reads_netcdf(ds, mn_resolved.workload_type)
             chunks = _chunk_recommendations_for(
                 ds, client, mn_resolved.workload_type, mn_resolved.suggest_chunks
             )
             return client, cluster, tmp_path, chunks  # type: ignore[return-value]
         return client, cluster, tmp_path  # type: ignore[return-value]
 
-    logger.info(
-        "Starting Dask client setup",
-        workload_type=workload_type or _DEFAULT_WORKLOAD_TYPE,
-        environment=env_type,
-    )
-
     # Load and merge configuration.
     # Save the caller-supplied config object before the local variable is
     # rebound by _resolve_configuration so it can be used as the base layer.
@@ -733,11 +775,14 @@ def setup_dask_client(
     config.fallback_on_detection_failure = fallback_on_detection_failure
     config.adaptive_memory = adaptive_memory
 
-    logger.debug(
-        "Configuration resolved",
+    # Logged after resolution: before it, an unset workload_type reads as the
+    # library default, so profile= and config= users saw "io" whatever they chose.
+    logger.info(
+        "Starting Dask client setup",
         workload_type=config.workload_type,
-        reserve_mem_gb=config.reserve_mem_gb,
+        environment=env_type,
     )
+    logger.debug("Configuration resolved", reserve_mem_gb=config.reserve_mem_gb)
 
     # Detect system resources
     resources = detect_resources(fallback=config.fallback_on_detection_failure)
@@ -826,26 +871,29 @@ def setup_dask_client(
             max_mem_gb=config.max_mem_gb,
         )
     except ValueError as e:
-        # Extract memory values for better error reporting
+        # The only failure here is "nothing left after the reserve": the worker
+        # count was already fitted to memory above, so fewer workers can't help.
+        # Measure against the same budget compute_usable_mem_gb() used --
+        # max_mem_gb caps the total *before* the reserve comes off, and
+        # ignoring it reported memory the caller had explicitly excluded.
         total_gib = resources.total_mem_bytes / (1024**3)
-        available_gb = total_gib - config.reserve_mem_gb
-        required_gb = topology.n_workers * 1.0  # Rough estimate: 1 GB per worker minimum
+        budget_gb = min(config.max_mem_gb or total_gib, total_gib)
+        available_gb = max(0.0, budget_gb - config.reserve_mem_gb)
+        required_gb = MIN_MEM_PER_WORKER_GB  # enough for a single worker
 
-        # Generate suggested actions based on the configuration
         suggestions = []
-        if config.reserve_mem_gb > available_gb / 2:  # Reserve more than half of available
+        if config.max_mem_gb is not None and config.max_mem_gb < total_gib:
             suggestions.append(
-                f"Reduce reserve_mem_gb from {config.reserve_mem_gb:.1f} GB to {available_gb * 0.3:.1f} GB"
+                f"Raise max_mem_gb above {config.reserve_mem_gb + required_gb:.1f} GB: it caps "
+                f"total memory and reserve_mem_gb ({config.reserve_mem_gb:.1f} GB) comes out of it"
             )
-        if topology.n_workers > 1:
+        max_reserve_gb = budget_gb - required_gb
+        if max_reserve_gb > 0:
             suggestions.append(
-                f"Limit max_workers to 1 or 2 workers instead of {topology.n_workers}"
+                f"Reduce reserve_mem_gb from {config.reserve_mem_gb:.1f} GB to at most "
+                f"{max_reserve_gb:.1f} GB"
             )
-        if not suggestions:  # Fallback suggestions
-            suggestions = [
-                "Close other applications to free up memory",
-                "Request a larger memory allocation for your job",
-            ]
+        suggestions.append("Request a larger memory allocation for your job")
 
         raise InsufficientResourcesError(
             required_mem=required_gb, available_mem=available_gb, suggested_actions=suggestions
@@ -931,6 +979,7 @@ def setup_dask_client(
     chunk_recommendations: dict[str, int] | None = None
 
     if ds is not None:
+        _warn_if_threaded_topology_reads_netcdf(ds, config.workload_type)
         chunk_recommendations = _chunk_recommendations_for(
             ds, client, config.workload_type, config.suggest_chunks
         )

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/dashboard.py

@@ -25,13 +25,18 @@ def get_login_host() -> str:
        unconditionally, including to SLURM users who have never heard of it.
     3. ``"<login-node>"`` as an obvious placeholder when there is no domain
        to work from (a laptop, a container with no DNS suffix).
+
+    ``getfqdn()`` falls back to a reverse-DNS pointer name when the host has no
+    forward entry -- on macOS typically ``1.0.0.[...].ip6.arpa`` -- and splitting
+    that gave a 60-character "login host".  Pointer names never name a host you
+    can SSH to, so they get the placeholder too.
     """
     override = os.environ.get(LOGIN_HOST_ENV)
     if override:
         return override
 
     fqdn = socket.getfqdn()
-    if "." in fqdn:
+    if "." in fqdn and not fqdn.lower().rstrip(".").endswith(".arpa"):
         domain = fqdn.split(".", 1)[1]
         if domain and domain not in {"local", "localdomain"}:
             return domain

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/exceptions.py

@@ -23,7 +23,7 @@ class InsufficientResourcesError(DaskSetupError):
             f"❌ Insufficient memory for configuration:\n"
             f"   - Required: {required_mem:.1f} GB\n"
             f"   - Available: {available_mem:.1f} GB\n"
-            f"   - Shortfall: {required_mem - available_mem:.1f} GB"
+            f"   - Shortfall: {max(0.0, required_mem - available_mem):.1f} GB"
         )
 
         if self.suggested_actions:

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/multinode.py

@@ -480,7 +480,9 @@ def _wait_for_workers(
             f"worker job logs (usually in ~/dask-worker-space/ or the PBS log dir)\n"
         ) from exc
 
-    n_connected = len(client.scheduler_info().get("workers", {}))
+    from .reporting import scheduler_workers
+
+    n_connected = len(scheduler_workers(client))
     logger.debug("Workers connected", n_connected=n_connected, expected_total=expected_total)
 
 
@@ -756,6 +758,33 @@ def _parse_slurm_nodelist(nodelist: str, cpus_per_node: int = 1) -> dict[str, in
     return dict.fromkeys(hosts, cpus_per_node)
 
 
+def discover_allocated_nodes() -> dict[str, int]:
+    """Return ``{hostname: cores}`` for the nodes allocated to the current job.
+
+    Reads ``PBS_NODEFILE`` first, then ``SLURM_NODELIST`` /
+    ``SLURM_JOB_NODELIST``.  Returns an empty dict outside a PBS or SLURM
+    allocation (a laptop, a login node).
+    """
+    nodefile = os.getenv("PBS_NODEFILE", "")
+    slurm_nodelist = os.getenv("SLURM_NODELIST") or os.getenv("SLURM_JOB_NODELIST", "")
+
+    if nodefile and Path(nodefile).exists():
+        node_cores = _parse_pbs_nodefile(nodefile)
+        logger.debug("PBS_NODEFILE parsed", nodes=list(node_cores.keys()))
+        return node_cores
+    if slurm_nodelist:
+        # SLURM_CPUS_ON_NODE can be a comma-separated list (one per node)
+        cpus_str = os.getenv("SLURM_CPUS_ON_NODE", "1")
+        try:
+            cpus_per_node = int(cpus_str.split(",")[0])
+        except ValueError:
+            cpus_per_node = 1
+        node_cores = _parse_slurm_nodelist(slurm_nodelist, cpus_per_node)
+        logger.debug("SLURM nodelist parsed", nodes=list(node_cores.keys()))
+        return node_cores
+    return {}
+
+
 def setup_interactive_cluster(
     workload_type: str = "cpu",
     workers_per_node: int | None = None,
@@ -811,25 +840,7 @@ def setup_interactive_cluster(
     """
     from dask.distributed import Client
 
-    # --- Discover allocated nodes ------------------------------------------
-    node_cores: dict[str, int] = {}
-
-    nodefile = os.getenv("PBS_NODEFILE", "")
-    slurm_nodelist = os.getenv("SLURM_NODELIST") or os.getenv("SLURM_JOB_NODELIST", "")
-
-    if nodefile and Path(nodefile).exists():
-        node_cores = _parse_pbs_nodefile(nodefile)
-        logger.debug("PBS_NODEFILE parsed", nodes=list(node_cores.keys()))
-    elif slurm_nodelist:
-        # SLURM_CPUS_ON_NODE can be a comma-separated list (one per node)
-        cpus_str = os.getenv("SLURM_CPUS_ON_NODE", "1")
-        try:
-            cpus_per_node = int(cpus_str.split(",")[0])
-        except ValueError:
-            cpus_per_node = 1
-        node_cores = _parse_slurm_nodelist(slurm_nodelist, cpus_per_node)
-        logger.debug("SLURM nodelist parsed", nodes=list(node_cores.keys()))
-
+    node_cores = discover_allocated_nodes()
     unique_nodes = list(node_cores.keys())
 
     # --- Single-node path (LocalCluster) ------------------------------------

{dask_setup-2.2.0 → dask_setup-2.3.0}/src/dask_setup/parquet.py

@@ -177,8 +177,9 @@ def recommend_parquet_chunks(
     max_partition_mb = target_partition_mb[1]
     if client is not None:
         try:
-            info = client.scheduler_info()
-            workers = info.get("workers", {})
+            from .reporting import scheduler_workers
+
+            workers = scheduler_workers(client)
             if workers:
                 min_worker_mem_bytes = min(
                     w.get("memory_limit", float("inf")) for w in workers.values()