sdrbench 0.1.0__tar.gz

sdrbench-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+src/sdrbench/_version.py
+check.json

sdrbench-0.1.0/LICENSE

@@ -0,0 +1,32 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, SDRBench team
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This license covers the code in this repository only. The datasets are
+provided by their original owners under the terms stated on
+https://sdrbench.github.io/ and on each Hugging Face dataset card.

sdrbench-0.1.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.5
+Name: sdrbench
+Version: 0.1.0
+Summary: Download and load SDRBench scientific datasets from Hugging Face or the original Globus archives
+Project-URL: Homepage, https://sdrbench.github.io/
+Project-URL: Source, https://github.com/szcompressor/sdrbench
+Project-URL: Hugging Face, https://huggingface.co/sdrbench
+Author: SDRBench team
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Keywords: benchmark,hpc,hugging face,lossy compression,scientific data,sdrbench
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Requires-Dist: filelock>=3.8
+Requires-Dist: huggingface-hub>=0.23
+Requires-Dist: numpy>=1.21
+Provides-Extra: sz3
+Requires-Dist: pysz>=1.1; extra == 'sz3'
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == 'test'
+Description-Content-Type: text/markdown
+
+# sdrbench
+
+The [SDRBench](https://sdrbench.github.io/) scientific datasets as numpy arrays, served from the
+[Hugging Face mirror](https://huggingface.co/sdrbench) (byte-for-byte copies of the original
+archives, pinned and sha256-verified), with automatic fallback to the original Globus archives at
+Argonne.
+
+```bash
+pip install sdrbench            # or "sdrbench[sz3]" to also get pysz (SZ3)
+```
+
+```python
+import sdrbench
+
+sdrbench.list()                          # ['basilisk-turbulence', 'cesm-atm', 'exaalt', ...]
+sdrbench.list(variants=True)             # ['basilisk-turbulence/small', ..., 'nyx/original', 'nyx/log', ...]
+
+nyx = sdrbench.dataset("nyx")            # default variant; sdrbench.dataset("nyx", "log")
+nyx.fields                               # ['baryon_density', 'dark_matter_density', 'temperature', ...]
+t = nyx["temperature"]                   # numpy memmap, float32, shape (512, 512, 512), C order
+sdrbench.load("cesm-atm", "CLDHGH")      # one-liner
+
+# time series: fields are "var/step"
+hur = sdrbench.dataset("hurricane-isabel", "P")
+hur.steps                                # ['01', ..., '48']
+p = hur.series("P")                      # ordered sequence of fields
+x = hur["P", "07"]                       # one step
+block = p.stack(slice(0, 4))             # (4, 100, 500, 500) copy
+
+# stacked files are split into zero-copy views per variable
+T = sdrbench.dataset("s3d")["T/1.1000E-03"]      # (500, 500, 500), component 6 of the stored file
+
+# raw files to a directory (original SDRBench names, downloaded in parallel)
+nyx.download("data/")                    # -> data/512x512x512/*.f32
+sdrbench.dataset("nyx", root="data/")    # later: use local files first (also $SDRBENCH_DATA)
+
+print(nyx.citation())                    # BibTeX + provider request + exact data version
+```
+
+Compress with SZ3 through [pysz](https://pypi.org/project/pysz/):
+
+```python
+import numpy as np
+from pysz import sz, szConfig, szErrorBoundMode
+
+x = sdrbench.dataset("scale-letkf")["T"]
+conf = szConfig()
+conf.errorBoundMode = szErrorBoundMode.REL
+conf.relErrorBound = 1e-3
+compressed, ratio = sz.compress(np.ascontiguousarray(x), conf)
+y, _ = sz.decompress(compressed, x.dtype.type, x.shape)
+print(ratio, sz.verify(np.asarray(x), y))  # ratio, (max error, PSNR, NRMSE)
+```
+
+Command line:
+
+```bash
+sdrbench list                                   # every dataset/variant with field count and size
+sdrbench info hurricane-isabel/P                # description, fields, dtype, shape, files
+sdrbench download nyx temperature -o data/      # plain files under data/<repo path>, -j parallel
+sdrbench path cesm-atm CLDHGH                   # local path (downloads if needed), e.g. for `sz3 -i`
+sdrbench cite qmcpack
+```
+
+## How it works
+
+- **Datasets, variants, fields.** A dataset (`nyx`) has variants with descriptive names (`original`,
+  `log`; `cesm-atm`: `2d`, `2d-cleared`, `3d`; `hacc`: `medium`, `big`, `sbig`, `region1`...;
+  `hurricane-isabel`: `snapshot`, `P`, `QCLOUD`, ...). The first is the default. The SDRBench / Hugging
+  Face folder name also works as a variant name (`dataset("nyx", "512x512x512")`).
+- **Field names are physical variables** (`CLDHGH`, `T`, `temperature`), or `var/step` for time series
+  and slabs (`P/07`, `xx/00042`, `density/31`); `ds.variables`, `ds.steps` and `ds.series(var)` expose the
+  structure. The original SDRBench file name also works (`ds["CLDHGH_1_1800_3600.f32"]`), lookups are
+  case-insensitive, and errors suggest close matches. Non-array files are in `ds.extra_files`.
+- **dtype and shape come from the catalog, never from names. Shapes are C order** (slowest first; the
+  convention of the SZ3 test-suite table): `np.fromfile(path, dtype).reshape(field.stored_shape)` reads
+  any stored file. `field.shape_fastest_first` is the order command-line compressors expect.
+- **Views, no copies of the data.** QMCPACK's default variant `preconditioned` (288 x 115 x 69 x 69, the
+  layout of the SDRBench examples) is computed on load by transposing the stored native file (variant
+  `original`); S3D's stacked files are split into per-variable memmap views. `field.save(path)` writes
+  the array exactly as loaded.
+- **Downloads** come from Hugging Face, pinned to the commit this release's catalog was built from and
+  checked against the catalog sha256, and are cached (`SDRBENCH_CACHE` moves the cache). Local copies are
+  used first when `root=` / `$SDRBENCH_DATA` is given (an untarred SDRBench directory works). If Hugging
+  Face cannot deliver, the package falls back to the original archive on Globus (whole archive, md5 and
+  sha256 verified, size shown in a warning; resumes dropped connections). `source="globus"` forces it.
+- Arrays are read-only memmaps; `np.array(x)` (or `load(mmap=False)`) gives a writable copy, e.g. for
+  `torch.from_numpy`.
+
+## Datasets
+
+| Dataset | Hugging Face | Variants | Source |
+|---|---|---|---|
+| CESM-ATM | [sdrbench/cesm-atm](https://huggingface.co/datasets/sdrbench/cesm-atm) | 2d, 2d-cleared, 3d | climate (SNL) |
+| EXAALT | [sdrbench/exaalt](https://huggingface.co/datasets/sdrbench/exaalt) | small, copper-1/2, helium-1/2 | molecular dynamics |
+| Hurricane ISABEL | [sdrbench/hurricane-isabel](https://huggingface.co/datasets/sdrbench/hurricane-isabel) | snapshot, P, U, ..., CLOUD_log10, ... | weather (NCAR, IEEE Vis 2004) |
+| EXAFEL | [sdrbench/exafel](https://huggingface.co/datasets/sdrbench/exafel) | small, large, assembled | LCLS X-ray images |
+| HACC | [sdrbench/hacc](https://huggingface.co/datasets/sdrbench/hacc) | medium, big, sbig, region1-6 | cosmology particles |
+| NYX | [sdrbench/nyx](https://huggingface.co/datasets/sdrbench/nyx) | original, log | cosmology |
+| NWChem | [sdrbench/nwchem](https://huggingface.co/datasets/sdrbench/nwchem) | default, f32 | quantum chemistry |
+| SCALE-LETKF | [sdrbench/scale-letkf](https://huggingface.co/datasets/sdrbench/scale-letkf) | original, log | weather (RIKEN) |
+| QMCPACK | [sdrbench/qmcpack](https://huggingface.co/datasets/sdrbench/qmcpack) | preconditioned, original | quantum Monte Carlo |
+| Miranda | [sdrbench/miranda](https://huggingface.co/datasets/sdrbench/miranda) | small, big | turbulence (LLNL) |
+| S3D | [sdrbench/s3d](https://huggingface.co/datasets/sdrbench/s3d) | default | combustion (SNL) |
+| Basilisk-Turbulence | [sdrbench/basilisk-turbulence](https://huggingface.co/datasets/sdrbench/basilisk-turbulence) | small, large | 2D turbulence |
+| XGC | [sdrbench/xgc](https://huggingface.co/datasets/sdrbench/xgc) | raw, adios | fusion (PPPL) |
+
+`sdrbench list` and `sdrbench info` show fields and sizes. NSTX GPI is not mirrored (its owner asks to
+be contacted before results are published); HACC region 5 is not available on Globus (HTTP 404).
+
+Please cite SDRBench and the data provider named on each dataset card (`sdrbench cite <dataset>`), and
+report the `sdrbench` version you used: a release always reads the same data.
+
+## Keeping the mirror in sync
+
+Globus remains the source of truth. `.github/workflows/release.yml` runs weekly:
+
+1. `sync/sync.py check` HEADs every archive linked from the SDRBench page and compares size, ETag and
+   Last-Modified with `sync/state.json`; archives that appear on, or disappear from, the page open an issue.
+2. Changed archives are re-mirrored in parallel jobs (`sync.py mirror`). Archives are streamed: each file is
+   unpacked, hashed and committed to `sdrbench/<dataset>` in batches and then deleted, so a runner only needs
+   room for the largest single file (~17 GB); the job frees disk like the SZ3 CI. Files removed from an
+   archive are removed from the repo, but only if the whole archive arrived and at most 25% of a variant
+   goes (`--allow-delete` overrides). The same command works on any machine:
+   `HF_TOKEN=... python sync/sync.py mirror --only <dataset>`.
+3. `sync.py catalog` regenerates `src/sdrbench/catalog.json` (pinned to the new Hugging Face revisions),
+   `sync.py verify` checks dtypes, element counts and dimension order against the data, `sync.py cards`
+   refreshes the dataset cards and file tables, and a new patch version is tagged and published to PyPI.
+
+### Adding a dataset
+
+1. Add an entry to `sync/datasets.json` (see its `_comment`): title, provider, science description, and per
+   variant the Hugging Face `folder`, Globus `archive`, optional `select`/`exclude`, and `rules` mapping file
+   globs to dtype, C-order shape and `var`/`step` name templates.
+2. `python sync/sync.py mirror --only <dataset> --dry --workdir /tmp/w` downloads and hashes without uploading
+   (writes `/tmp/w/state.dry.json`); fix rules until `sync.py catalog` builds without errors (it refuses files
+   without a rule or with a size that does not match the shape).
+3. Mirror for real, then `sync.py catalog`, `sync.py verify --only <dataset>`, `sync.py cards --only <dataset>`,
+   and `python sync/selftest.py --only <dataset> --workdir /big/tmp` (every field through the package).
+
+## Development
+
+```bash
+pip install -e ".[test,sz3]"
+pytest                       # offline tests
+pytest -m online             # end-to-end against Hugging Face and Globus
+python sync/sync.py verify   # data-level check of every dtype/shape (network)
+python sync/selftest.py --workdir /big/tmp   # every field of every dataset through the package
+```

sdrbench-0.1.0/README.md

@@ -0,0 +1,149 @@
+# sdrbench
+
+The [SDRBench](https://sdrbench.github.io/) scientific datasets as numpy arrays, served from the
+[Hugging Face mirror](https://huggingface.co/sdrbench) (byte-for-byte copies of the original
+archives, pinned and sha256-verified), with automatic fallback to the original Globus archives at
+Argonne.
+
+```bash
+pip install sdrbench            # or "sdrbench[sz3]" to also get pysz (SZ3)
+```
+
+```python
+import sdrbench
+
+sdrbench.list()                          # ['basilisk-turbulence', 'cesm-atm', 'exaalt', ...]
+sdrbench.list(variants=True)             # ['basilisk-turbulence/small', ..., 'nyx/original', 'nyx/log', ...]
+
+nyx = sdrbench.dataset("nyx")            # default variant; sdrbench.dataset("nyx", "log")
+nyx.fields                               # ['baryon_density', 'dark_matter_density', 'temperature', ...]
+t = nyx["temperature"]                   # numpy memmap, float32, shape (512, 512, 512), C order
+sdrbench.load("cesm-atm", "CLDHGH")      # one-liner
+
+# time series: fields are "var/step"
+hur = sdrbench.dataset("hurricane-isabel", "P")
+hur.steps                                # ['01', ..., '48']
+p = hur.series("P")                      # ordered sequence of fields
+x = hur["P", "07"]                       # one step
+block = p.stack(slice(0, 4))             # (4, 100, 500, 500) copy
+
+# stacked files are split into zero-copy views per variable
+T = sdrbench.dataset("s3d")["T/1.1000E-03"]      # (500, 500, 500), component 6 of the stored file
+
+# raw files to a directory (original SDRBench names, downloaded in parallel)
+nyx.download("data/")                    # -> data/512x512x512/*.f32
+sdrbench.dataset("nyx", root="data/")    # later: use local files first (also $SDRBENCH_DATA)
+
+print(nyx.citation())                    # BibTeX + provider request + exact data version
+```
+
+Compress with SZ3 through [pysz](https://pypi.org/project/pysz/):
+
+```python
+import numpy as np
+from pysz import sz, szConfig, szErrorBoundMode
+
+x = sdrbench.dataset("scale-letkf")["T"]
+conf = szConfig()
+conf.errorBoundMode = szErrorBoundMode.REL
+conf.relErrorBound = 1e-3
+compressed, ratio = sz.compress(np.ascontiguousarray(x), conf)
+y, _ = sz.decompress(compressed, x.dtype.type, x.shape)
+print(ratio, sz.verify(np.asarray(x), y))  # ratio, (max error, PSNR, NRMSE)
+```
+
+Command line:
+
+```bash
+sdrbench list                                   # every dataset/variant with field count and size
+sdrbench info hurricane-isabel/P                # description, fields, dtype, shape, files
+sdrbench download nyx temperature -o data/      # plain files under data/<repo path>, -j parallel
+sdrbench path cesm-atm CLDHGH                   # local path (downloads if needed), e.g. for `sz3 -i`
+sdrbench cite qmcpack
+```
+
+## How it works
+
+- **Datasets, variants, fields.** A dataset (`nyx`) has variants with descriptive names (`original`,
+  `log`; `cesm-atm`: `2d`, `2d-cleared`, `3d`; `hacc`: `medium`, `big`, `sbig`, `region1`...;
+  `hurricane-isabel`: `snapshot`, `P`, `QCLOUD`, ...). The first is the default. The SDRBench / Hugging
+  Face folder name also works as a variant name (`dataset("nyx", "512x512x512")`).
+- **Field names are physical variables** (`CLDHGH`, `T`, `temperature`), or `var/step` for time series
+  and slabs (`P/07`, `xx/00042`, `density/31`); `ds.variables`, `ds.steps` and `ds.series(var)` expose the
+  structure. The original SDRBench file name also works (`ds["CLDHGH_1_1800_3600.f32"]`), lookups are
+  case-insensitive, and errors suggest close matches. Non-array files are in `ds.extra_files`.
+- **dtype and shape come from the catalog, never from names. Shapes are C order** (slowest first; the
+  convention of the SZ3 test-suite table): `np.fromfile(path, dtype).reshape(field.stored_shape)` reads
+  any stored file. `field.shape_fastest_first` is the order command-line compressors expect.
+- **Views, no copies of the data.** QMCPACK's default variant `preconditioned` (288 x 115 x 69 x 69, the
+  layout of the SDRBench examples) is computed on load by transposing the stored native file (variant
+  `original`); S3D's stacked files are split into per-variable memmap views. `field.save(path)` writes
+  the array exactly as loaded.
+- **Downloads** come from Hugging Face, pinned to the commit this release's catalog was built from and
+  checked against the catalog sha256, and are cached (`SDRBENCH_CACHE` moves the cache). Local copies are
+  used first when `root=` / `$SDRBENCH_DATA` is given (an untarred SDRBench directory works). If Hugging
+  Face cannot deliver, the package falls back to the original archive on Globus (whole archive, md5 and
+  sha256 verified, size shown in a warning; resumes dropped connections). `source="globus"` forces it.
+- Arrays are read-only memmaps; `np.array(x)` (or `load(mmap=False)`) gives a writable copy, e.g. for
+  `torch.from_numpy`.
+
+## Datasets
+
+| Dataset | Hugging Face | Variants | Source |
+|---|---|---|---|
+| CESM-ATM | [sdrbench/cesm-atm](https://huggingface.co/datasets/sdrbench/cesm-atm) | 2d, 2d-cleared, 3d | climate (SNL) |
+| EXAALT | [sdrbench/exaalt](https://huggingface.co/datasets/sdrbench/exaalt) | small, copper-1/2, helium-1/2 | molecular dynamics |
+| Hurricane ISABEL | [sdrbench/hurricane-isabel](https://huggingface.co/datasets/sdrbench/hurricane-isabel) | snapshot, P, U, ..., CLOUD_log10, ... | weather (NCAR, IEEE Vis 2004) |
+| EXAFEL | [sdrbench/exafel](https://huggingface.co/datasets/sdrbench/exafel) | small, large, assembled | LCLS X-ray images |
+| HACC | [sdrbench/hacc](https://huggingface.co/datasets/sdrbench/hacc) | medium, big, sbig, region1-6 | cosmology particles |
+| NYX | [sdrbench/nyx](https://huggingface.co/datasets/sdrbench/nyx) | original, log | cosmology |
+| NWChem | [sdrbench/nwchem](https://huggingface.co/datasets/sdrbench/nwchem) | default, f32 | quantum chemistry |
+| SCALE-LETKF | [sdrbench/scale-letkf](https://huggingface.co/datasets/sdrbench/scale-letkf) | original, log | weather (RIKEN) |
+| QMCPACK | [sdrbench/qmcpack](https://huggingface.co/datasets/sdrbench/qmcpack) | preconditioned, original | quantum Monte Carlo |
+| Miranda | [sdrbench/miranda](https://huggingface.co/datasets/sdrbench/miranda) | small, big | turbulence (LLNL) |
+| S3D | [sdrbench/s3d](https://huggingface.co/datasets/sdrbench/s3d) | default | combustion (SNL) |
+| Basilisk-Turbulence | [sdrbench/basilisk-turbulence](https://huggingface.co/datasets/sdrbench/basilisk-turbulence) | small, large | 2D turbulence |
+| XGC | [sdrbench/xgc](https://huggingface.co/datasets/sdrbench/xgc) | raw, adios | fusion (PPPL) |
+
+`sdrbench list` and `sdrbench info` show fields and sizes. NSTX GPI is not mirrored (its owner asks to
+be contacted before results are published); HACC region 5 is not available on Globus (HTTP 404).
+
+Please cite SDRBench and the data provider named on each dataset card (`sdrbench cite <dataset>`), and
+report the `sdrbench` version you used: a release always reads the same data.
+
+## Keeping the mirror in sync
+
+Globus remains the source of truth. `.github/workflows/release.yml` runs weekly:
+
+1. `sync/sync.py check` HEADs every archive linked from the SDRBench page and compares size, ETag and
+   Last-Modified with `sync/state.json`; archives that appear on, or disappear from, the page open an issue.
+2. Changed archives are re-mirrored in parallel jobs (`sync.py mirror`). Archives are streamed: each file is
+   unpacked, hashed and committed to `sdrbench/<dataset>` in batches and then deleted, so a runner only needs
+   room for the largest single file (~17 GB); the job frees disk like the SZ3 CI. Files removed from an
+   archive are removed from the repo, but only if the whole archive arrived and at most 25% of a variant
+   goes (`--allow-delete` overrides). The same command works on any machine:
+   `HF_TOKEN=... python sync/sync.py mirror --only <dataset>`.
+3. `sync.py catalog` regenerates `src/sdrbench/catalog.json` (pinned to the new Hugging Face revisions),
+   `sync.py verify` checks dtypes, element counts and dimension order against the data, `sync.py cards`
+   refreshes the dataset cards and file tables, and a new patch version is tagged and published to PyPI.
+
+### Adding a dataset
+
+1. Add an entry to `sync/datasets.json` (see its `_comment`): title, provider, science description, and per
+   variant the Hugging Face `folder`, Globus `archive`, optional `select`/`exclude`, and `rules` mapping file
+   globs to dtype, C-order shape and `var`/`step` name templates.
+2. `python sync/sync.py mirror --only <dataset> --dry --workdir /tmp/w` downloads and hashes without uploading
+   (writes `/tmp/w/state.dry.json`); fix rules until `sync.py catalog` builds without errors (it refuses files
+   without a rule or with a size that does not match the shape).
+3. Mirror for real, then `sync.py catalog`, `sync.py verify --only <dataset>`, `sync.py cards --only <dataset>`,
+   and `python sync/selftest.py --only <dataset> --workdir /big/tmp` (every field through the package).
+
+## Development
+
+```bash
+pip install -e ".[test,sz3]"
+pytest                       # offline tests
+pytest -m online             # end-to-end against Hugging Face and Globus
+python sync/sync.py verify   # data-level check of every dtype/shape (network)
+python sync/selftest.py --workdir /big/tmp   # every field of every dataset through the package
+```

sdrbench-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["hatchling>=1.24", "hatch-vcs>=0.4"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sdrbench"
+dynamic = ["version"]
+description = "Download and load SDRBench scientific datasets from Hugging Face or the original Globus archives"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+authors = [{ name = "SDRBench team" }]
+keywords = ["sdrbench", "scientific data", "lossy compression", "benchmark", "hpc", "hugging face"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Intended Audience :: Science/Research",
+  "Topic :: Scientific/Engineering",
+  "Operating System :: OS Independent",
+]
+dependencies = ["numpy>=1.21", "huggingface_hub>=0.23", "filelock>=3.8"]
+
+[project.optional-dependencies]
+sz3 = ["pysz>=1.1"]
+test = ["pytest>=7"]
+
+[project.urls]
+Homepage = "https://sdrbench.github.io/"
+Source = "https://github.com/szcompressor/sdrbench"
+"Hugging Face" = "https://huggingface.co/sdrbench"
+
+[project.scripts]
+sdrbench = "sdrbench.cli:main"
+
+[tool.hatch.version]
+source = "vcs"
+fallback-version = "0.0.0"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/sdrbench/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sdrbench"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/sdrbench", "tests", "README.md", "LICENSE"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = ["online: needs network access to Hugging Face and Globus"]
+addopts = "-m 'not online'"

sdrbench-0.1.0/src/sdrbench/__init__.py

@@ -0,0 +1,23 @@
+"""SDRBench scientific datasets as numpy arrays.
+
+    >>> import sdrbench
+    >>> sdrbench.list()                      # datasets; sdrbench.list(variants=True) for all variants
+    >>> nyx = sdrbench.dataset("nyx")
+    >>> nyx.fields
+    >>> t = nyx["temperature"]               # memmap, shape (512, 512, 512), C order
+    >>> p = sdrbench.dataset("hurricane-isabel", "P").series("P")   # 48 time steps
+
+Files come from the Hugging Face mirror (https://huggingface.co/sdrbench), pinned to the
+revision this release was built from; if Hugging Face cannot deliver, from the original
+SDRBench archives on Globus. Local copies can be used via ``root=`` or ``$SDRBENCH_DATA``.
+"""
+from ._core import Dataset, Field, Series, cache_dir, catalog, dataset, list, load
+
+try:
+    from ._version import __version__
+except ImportError:  # source checkout without a build
+    __version__ = "0.0.0"
+
+# `list` is deliberately not exported by `from sdrbench import *` (it would shadow the builtin);
+# use sdrbench.list().
+__all__ = ["Dataset", "Field", "Series", "cache_dir", "catalog", "dataset", "load"]

sdrbench-0.1.0/src/sdrbench/_archive.py

@@ -0,0 +1,219 @@
+"""Download and unpack SDRBench archives.
+
+The Hugging Face mirror is built with exactly this code, so a file fetched
+from Globus ends up at the same relative path, with the same bytes, as the
+file stored on Hugging Face.
+"""
+from __future__ import annotations
+
+import hashlib
+import http.client
+import os
+import shutil
+import tarfile
+import time
+import urllib.error
+import urllib.request
+import zipfile
+from pathlib import Path
+from typing import Optional, Tuple
+
+CHUNK = 1 << 22
+JUNK = ("._", ".DS_Store", "__MACOSX")
+
+
+def _is_junk(name: str) -> bool:
+    return any(part.startswith(JUNK) for part in Path(name).parts)
+
+
+def sha256_file(path, chunk: int = 1 << 24) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while b := f.read(chunk):
+            h.update(b)
+    return h.hexdigest()
+
+
+class _HashingReader:
+    """File-like wrapper that hashes and counts everything read through it."""
+
+    def __init__(self, raw):
+        self.raw = raw
+        self.md5 = hashlib.md5()
+        self.nbytes = 0
+
+    def read(self, n: int = -1) -> bytes:
+        b = self.raw.read(n)
+        self.md5.update(b)
+        self.nbytes += len(b)
+        return b
+
+    def drain(self) -> None:
+        while self.read(CHUNK):
+            pass
+
+
+class _ResumingResponse:
+    """Readable HTTP body that reconnects with a Range request when the connection drops,
+    so multi-GB downloads survive transient network failures.
+
+    The full size must be known (Globus sends GET bodies chunked, without Content-Length, so it
+    comes from HEAD); without it a dropped connection cannot be told apart from the end of the
+    file, so we refuse to download instead of risking a silently truncated archive."""
+
+    def __init__(self, url: str, retries: int = 10, timeout: float = 120):
+        self.url, self.retries, self.timeout = url, retries, timeout
+        self.pos = 0
+        self.total = self._head_length()
+        self.resp = self._connect()
+
+    def _head_length(self) -> int:
+        last = None
+        for attempt in range(self.retries + 1):
+            req = urllib.request.Request(self.url, method="HEAD", headers={"User-Agent": "sdrbench"})
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as r:
+                    n = r.headers.get("Content-Length")
+                    if n is not None:
+                        return int(n)
+                    last = "no Content-Length in HEAD response"
+            except urllib.error.HTTPError as e:
+                if e.code not in (429, 502, 503, 504):  # only temporary server errors are retried
+                    raise
+                last = e
+            except (OSError, http.client.HTTPException, ValueError) as e:
+                last = e
+            time.sleep(min(60, 2 ** attempt))
+        raise IOError(f"{self.url}: cannot determine the archive size ({last}); refusing an unverifiable download")
+
+    def _connect(self):
+        headers = {"User-Agent": "sdrbench"}
+        if self.pos:
+            headers["Range"] = f"bytes={self.pos}-"
+        resp = urllib.request.urlopen(urllib.request.Request(self.url, headers=headers), timeout=self.timeout)
+        if self.pos:
+            rng = resp.headers.get("Content-Range", "")
+            if resp.status != 206 or not rng.startswith(f"bytes {self.pos}-"):
+                resp.close()
+                raise IOError(f"{self.url}: server did not resume at byte {self.pos} ({resp.status} {rng!r})")
+        return resp
+
+    def read(self, n: int = -1) -> bytes:
+        for attempt in range(self.retries + 1):
+            try:
+                b = self.resp.read(n)
+                if not b and n != 0 and self.pos < self.total:
+                    # body ended early: a dropped connection (with chunked encoding this can look
+                    # like a clean end of stream)
+                    raise http.client.IncompleteRead(b"", self.total - self.pos)
+                self.pos += len(b)
+                if self.pos > self.total:
+                    raise IOError(f"{self.url}: received more than the {self.total} bytes announced")
+                return b
+            except (OSError, http.client.HTTPException) as e:
+                if isinstance(e, IOError) and "announced" in str(e) or attempt == self.retries:
+                    raise
+                time.sleep(min(60, 2 ** attempt))
+                try:
+                    self.resp.close()
+                except Exception:
+                    pass
+                try:
+                    self.resp = self._connect()
+                except (OSError, http.client.HTTPException):
+                    continue  # try again on the next attempt
+        raise IOError(f"{self.url}: download failed at byte {self.pos} of {self.total}")
+
+    def close(self):
+        self.resp.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.close()
+
+
+def _open_url(url: str):
+    if url.startswith(("http://", "https://")):
+        return _ResumingResponse(url)
+    return urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "sdrbench"}), timeout=120)
+
+
+def _safe_target(root: Path, name: str) -> Path:
+    target = (root / name).resolve()
+    if os.path.commonpath([str(root.resolve()), str(target)]) != str(root.resolve()):
+        raise ValueError(f"unsafe path in archive: {name}")
+    return target
+
+
+def _extract_tar_stream(stream, root: Path) -> None:
+    with tarfile.open(fileobj=stream, mode="r|*") as tf:
+        for m in tf:
+            if _is_junk(m.name) or not (m.isfile() or m.isdir()):
+                continue
+            target = _safe_target(root, m.name)
+            if m.isdir():
+                target.mkdir(parents=True, exist_ok=True)
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            src = tf.extractfile(m)
+            with open(target, "wb") as out:
+                shutil.copyfileobj(src, out, CHUNK)
+
+
+def _extract_zip(path: Path, root: Path) -> None:
+    with zipfile.ZipFile(path) as zf:
+        for info in zf.infolist():
+            if _is_junk(info.filename) or info.is_dir():
+                continue
+            target = _safe_target(root, info.filename)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            with zf.open(info) as src, open(target, "wb") as out:
+                shutil.copyfileobj(src, out, CHUNK)
+
+
+def fetch_archive(url: str, dest, expected_md5: Optional[str] = None) -> Tuple[str, int]:
+    """Stream ``url`` (a .tar.gz or .zip) into directory ``dest``.
+
+    If all files of the archive live under a single top-level directory, its contents are
+    placed directly in ``dest`` (the same rule the Hugging Face mirror uses). File contents
+    are never modified. The whole archive must arrive: its size is checked against the
+    server's, and its md5 against ``expected_md5`` when given.
+    Returns ``(md5 of the archive, archive size in bytes)``.
+    """
+    import tempfile
+
+    dest = Path(dest)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = Path(tempfile.mkdtemp(prefix=dest.name + ".", suffix=".partial", dir=dest.parent))
+    try:
+        with _open_url(url) as resp:
+            reader = _HashingReader(resp)
+            if url.endswith(".zip"):
+                zpath = tmp / "_archive.zip"
+                with open(zpath, "wb") as out:
+                    shutil.copyfileobj(reader, out, CHUNK)
+                files = tmp / "files"
+                _extract_zip(zpath, files)
+                zpath.unlink()
+            else:
+                files = tmp / "files"
+                files.mkdir()
+                _extract_tar_stream(reader, files)
+                reader.drain()
+            total = getattr(resp, "total", None)
+        if total is not None and reader.nbytes != total:
+            raise IOError(f"{url}: got {reader.nbytes} of {total} bytes")
+        md5 = reader.md5.hexdigest()
+        if expected_md5 and md5 != expected_md5:
+            raise IOError(f"md5 mismatch for {url}: got {md5}, expected {expected_md5}")
+        tops = {p.relative_to(files).parts[0] for p in files.rglob("*") if p.is_file()}
+        nested = {p.relative_to(files).parts[0] for p in files.rglob("*") if p.is_file()
+                  and len(p.relative_to(files).parts) > 1}
+        src = files / next(iter(tops)) if len(tops) == 1 and tops == nested else files
+        shutil.rmtree(dest, ignore_errors=True)
+        os.replace(src, dest)
+        return md5, reader.nbytes
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)