ncarnate 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ncarnate-2.0.0/.gitattributes +2 -0
- ncarnate-2.0.0/.github/FUNDING.yml +12 -0
- ncarnate-2.0.0/.github/workflows/ci.yml +70 -0
- ncarnate-2.0.0/.github/workflows/publish.yml +79 -0
- ncarnate-2.0.0/.gitignore +21 -0
- ncarnate-2.0.0/CHANGELOG.md +70 -0
- ncarnate-2.0.0/CITATION.cff +25 -0
- ncarnate-2.0.0/LICENSE +21 -0
- ncarnate-2.0.0/PKG-INFO +199 -0
- ncarnate-2.0.0/README.rst +166 -0
- ncarnate-2.0.0/docs/fidelity-notes.md +134 -0
- ncarnate-2.0.0/ncarnate/__init__.py +34 -0
- ncarnate-2.0.0/ncarnate/__main__.py +21 -0
- ncarnate-2.0.0/ncarnate/cli.py +288 -0
- ncarnate-2.0.0/ncarnate/constants.py +19 -0
- ncarnate-2.0.0/ncarnate/core.py +502 -0
- ncarnate-2.0.0/ncarnate/eos/__init__.py +12 -0
- ncarnate-2.0.0/ncarnate/eos/gctp.py +260 -0
- ncarnate-2.0.0/ncarnate/eos/grid.py +159 -0
- ncarnate-2.0.0/ncarnate/eos/structmeta.py +420 -0
- ncarnate-2.0.0/ncarnate/eos/swath.py +172 -0
- ncarnate-2.0.0/ncarnate/errors.py +85 -0
- ncarnate-2.0.0/ncarnate/formats.py +83 -0
- ncarnate-2.0.0/ncarnate/hdf4.py +1109 -0
- ncarnate-2.0.0/ncarnate/limits.py +64 -0
- ncarnate-2.0.0/pyproject.toml +101 -0
- ncarnate-2.0.0/tests/conftest.py +115 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_5daysnow_trim.hdf +0 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_5daysnow_trim.provenance.json +15 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_seaice12km_trim.hdf +0 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_seaice12km_trim.provenance.json +15 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/mod03_trim.hdf +0 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/mod03_trim.provenance.json +19 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/myd05_trim.hdf +0 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/myd05_trim.provenance.json +20 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/raingrid_trim.hdf +0 -0
- ncarnate-2.0.0/tests/fixtures/data/hdfeos2/raingrid_trim.provenance.json +15 -0
- ncarnate-2.0.0/tests/fixtures/data/netcdf/endianness.nc +0 -0
- ncarnate-2.0.0/tests/fixtures/data/netcdf/nested_groups.nc +0 -0
- ncarnate-2.0.0/tests/fixtures/data/netcdf/packed_fill.nc +0 -0
- ncarnate-2.0.0/tests/fixtures/data/netcdf/unlimited_dim.nc +0 -0
- ncarnate-2.0.0/tests/fixtures/make_fixtures.py +136 -0
- ncarnate-2.0.0/tests/fixtures/trim_hdfeos2.py +296 -0
- ncarnate-2.0.0/tests/test_cli.py +72 -0
- ncarnate-2.0.0/tests/test_eos_units.py +316 -0
- ncarnate-2.0.0/tests/test_formats.py +37 -0
- ncarnate-2.0.0/tests/test_hdf4_conversion.py +318 -0
- ncarnate-2.0.0/tests/test_limits.py +57 -0
- ncarnate-2.0.0/tests/test_raw_granules.py +96 -0
- ncarnate-2.0.0/tests/test_recompress_netcdf.py +133 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# These are supported funding model platforms
|
|
2
|
+
|
|
3
|
+
github: [ErickShepherd] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
|
4
|
+
patreon: ErickShepherd # Replace with a single Patreon username
|
|
5
|
+
open_collective: # Replace with a single Open Collective username
|
|
6
|
+
ko_fi: # Replace with a single Ko-fi username
|
|
7
|
+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
|
8
|
+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
|
9
|
+
liberapay: # Replace with a single Liberapay username
|
|
10
|
+
issuehunt: # Replace with a single IssueHunt username
|
|
11
|
+
otechie: # Replace with a single Otechie username
|
|
12
|
+
custom: ['paypal.me/ErickEShepherd', 'https://cash.app/$ErickShepherd', 'https://venmo.com/ErickShepherd', 'https://erickshepherd.com/pay/bitcoin/'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, master]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
# No system HDF libraries are installed anywhere in this workflow on
|
|
10
|
+
# purpose: the netCDF4 wheels bundle libhdf5/libnetcdf, and the pyhdf
|
|
11
|
+
# >= 0.11.6 manylinux wheels bundle libhdf4 (verified against PyPI
|
|
12
|
+
# 2026-07-08 for cp310-cp313). Platforms without a pyhdf wheel (e.g.
|
|
13
|
+
# linux-aarch64) need a system libhdf4 for the sdist build instead.
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
lint:
|
|
17
|
+
name: ruff
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.12"
|
|
24
|
+
- name: Install ruff
|
|
25
|
+
run: python -m pip install ruff
|
|
26
|
+
- name: Run ruff
|
|
27
|
+
run: ruff check .
|
|
28
|
+
|
|
29
|
+
test:
|
|
30
|
+
name: pytest (${{ matrix.os }} py${{ matrix.python-version }})
|
|
31
|
+
runs-on: ${{ matrix.os }}
|
|
32
|
+
strategy:
|
|
33
|
+
fail-fast: false
|
|
34
|
+
matrix:
|
|
35
|
+
os: [ubuntu-latest]
|
|
36
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
37
|
+
# Smoke the self-contained-wheel install story (pyhdf + netCDF4 +
|
|
38
|
+
# pyproj, no system HDF libs) on macOS too. Windows is deliberately
|
|
39
|
+
# excluded: pyhdf's Windows wheel ships no HDF4 runtime, so the
|
|
40
|
+
# HDF4 path can't import from PyPI wheels there (README documents
|
|
41
|
+
# the conda-forge / WSL route). The netCDF path works on Windows.
|
|
42
|
+
include:
|
|
43
|
+
- os: macos-latest
|
|
44
|
+
python-version: "3.12"
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/checkout@v4
|
|
47
|
+
- uses: actions/setup-python@v5
|
|
48
|
+
with:
|
|
49
|
+
python-version: ${{ matrix.python-version }}
|
|
50
|
+
- name: Install package with test extras
|
|
51
|
+
run: |
|
|
52
|
+
python -m pip install --upgrade pip
|
|
53
|
+
python -m pip install -e ".[test]"
|
|
54
|
+
- name: Run tests (offline; raw-granule marks self-skip)
|
|
55
|
+
run: python -m pytest -q
|
|
56
|
+
|
|
57
|
+
build:
|
|
58
|
+
name: build + twine check
|
|
59
|
+
runs-on: ubuntu-latest
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/checkout@v4
|
|
62
|
+
- uses: actions/setup-python@v5
|
|
63
|
+
with:
|
|
64
|
+
python-version: "3.12"
|
|
65
|
+
- name: Install build tooling
|
|
66
|
+
run: python -m pip install build twine
|
|
67
|
+
- name: Build sdist + wheel
|
|
68
|
+
run: python -m build
|
|
69
|
+
- name: Check distributions
|
|
70
|
+
run: twine check dist/*
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishes the package via PyPI Trusted Publishing (OIDC) — no API token is
|
|
4
|
+
# stored anywhere (not in the repo, not in GitHub secrets). PyPI verifies the
|
|
5
|
+
# workflow's short-lived OIDC identity instead.
|
|
6
|
+
#
|
|
7
|
+
# Two ways to run:
|
|
8
|
+
# * A published GitHub Release -> builds and publishes to PyPI.
|
|
9
|
+
# * Manual "Run workflow" -> choose testpypi (dry run) or pypi.
|
|
10
|
+
#
|
|
11
|
+
# The publish jobs run in protected GitHub Environments (`pypi` / `testpypi`).
|
|
12
|
+
# Add yourself as a required reviewer on each (repo Settings -> Environments)
|
|
13
|
+
# so every upload pauses for your one-click approval before it leaves.
|
|
14
|
+
|
|
15
|
+
on:
|
|
16
|
+
release:
|
|
17
|
+
types: [released] # full releases only — pre-releases never auto-publish
|
|
18
|
+
workflow_dispatch:
|
|
19
|
+
inputs:
|
|
20
|
+
target:
|
|
21
|
+
description: "Where to publish"
|
|
22
|
+
type: choice
|
|
23
|
+
options: [testpypi, pypi]
|
|
24
|
+
default: testpypi
|
|
25
|
+
|
|
26
|
+
permissions:
|
|
27
|
+
contents: read
|
|
28
|
+
|
|
29
|
+
jobs:
|
|
30
|
+
build:
|
|
31
|
+
name: Build sdist + wheel
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- name: Build
|
|
39
|
+
run: |
|
|
40
|
+
python -m pip install --upgrade build twine
|
|
41
|
+
python -m build
|
|
42
|
+
python -m twine check dist/*
|
|
43
|
+
- uses: actions/upload-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
|
|
48
|
+
publish-testpypi:
|
|
49
|
+
name: Publish to TestPyPI
|
|
50
|
+
needs: build
|
|
51
|
+
if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
environment: testpypi
|
|
54
|
+
permissions:
|
|
55
|
+
id-token: write
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/download-artifact@v4
|
|
58
|
+
with:
|
|
59
|
+
name: dist
|
|
60
|
+
path: dist/
|
|
61
|
+
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
|
62
|
+
with:
|
|
63
|
+
repository-url: https://test.pypi.org/legacy/
|
|
64
|
+
skip-existing: true # re-dispatching a dry run for an existing version won't fail
|
|
65
|
+
|
|
66
|
+
publish-pypi:
|
|
67
|
+
name: Publish to PyPI
|
|
68
|
+
needs: build
|
|
69
|
+
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
environment: pypi
|
|
72
|
+
permissions:
|
|
73
|
+
id-token: write
|
|
74
|
+
steps:
|
|
75
|
+
- uses: actions/download-artifact@v4
|
|
76
|
+
with:
|
|
77
|
+
name: dist
|
|
78
|
+
path: dist/
|
|
79
|
+
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Python build/runtime artifacts.
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
|
|
8
|
+
# Virtual environments.
|
|
9
|
+
venv/
|
|
10
|
+
.venv/
|
|
11
|
+
|
|
12
|
+
# Tooling caches.
|
|
13
|
+
.pytest_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
|
|
16
|
+
# Local scratch.
|
|
17
|
+
tmp/
|
|
18
|
+
|
|
19
|
+
# Audit orchestration sidecars (ephemeral analysis).
|
|
20
|
+
docs/audits/*.run.log
|
|
21
|
+
docs/audits/*.audit-state.json
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is
|
|
4
|
+
based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and
|
|
5
|
+
this project adheres to [Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [2.0.0] - 2026-07-08
|
|
8
|
+
|
|
9
|
+
A ground-up rebuild of the 2020 utility, renamed from
|
|
10
|
+
`netcdf_recompressor` to **ncarnate**. Everything below is breaking
|
|
11
|
+
relative to the notional 1.x behavior; there are no downstream users
|
|
12
|
+
(1.x was never published).
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- The CLI recompresses again: a debugging `raise ValueError` committed
|
|
17
|
+
in 2020 made the shipped tool process nothing.
|
|
18
|
+
- Variables carrying `_FillValue` no longer crash the copy (`fill_value`
|
|
19
|
+
is declared at variable creation, as netCDF4 requires).
|
|
20
|
+
- Packed-integer data is no longer silently re-quantized through float:
|
|
21
|
+
values are copied raw (`set_auto_maskandscale(False)`), with
|
|
22
|
+
`scale_factor`/`add_offset`/`_FillValue` re-declared, never applied.
|
|
23
|
+
- Overwrite is safe: output is written to a temp file, verified
|
|
24
|
+
value-for-value against the source, and only then atomically moved
|
|
25
|
+
into place. A failed run never destroys the source.
|
|
26
|
+
- `-V`/`--version` works without a path; errors exit non-zero with
|
|
27
|
+
clear messages instead of vanishing into a log file.
|
|
28
|
+
|
|
29
|
+
### Added
|
|
30
|
+
|
|
31
|
+
- **HDF4/HDF-EOS2 ingest** (read-only; output is always netCDF4):
|
|
32
|
+
SDS payloads convert bit-identically, one netCDF4 group per EOS
|
|
33
|
+
structure, EOS metadata preserved verbatim, hostile names sanitized
|
|
34
|
+
with originals recorded in companion attributes.
|
|
35
|
+
- **CF geolocation reconstruction** for HDF-EOS2: GCTP
|
|
36
|
+
polar-stereographic, geographic, and Lambert-azimuthal (EASE-Grid)
|
|
37
|
+
grids become CF grid mappings + `x`/`y` + 2-D `lat`/`lon`; swath
|
|
38
|
+
geolocation is CF-attached; dimension-mapped geolocation (5 km →
|
|
39
|
+
1 km) is interpolated through ECEF space with fill propagation.
|
|
40
|
+
`--no-geolocation` converts payload-only. Unsupported constructs fail
|
|
41
|
+
loud with named errors.
|
|
42
|
+
- File-type dispatch by magic bytes (netCDF3 / netCDF4-HDF5 / HDF4),
|
|
43
|
+
independent of file extension.
|
|
44
|
+
- pytest suite (66 tests) pinning the fidelity contract over committed
|
|
45
|
+
fixtures trimmed from real granules, plus the geolocation verification
|
|
46
|
+
lattice (four of the design's five checks; the `eos2dump` external
|
|
47
|
+
reference is unimplemented — the same-granule THG reference covers
|
|
48
|
+
that grid); CI across CPython 3.10–3.13.
|
|
49
|
+
- Modern packaging: hatchling `pyproject.toml`, console entry point
|
|
50
|
+
`ncarnate`, single-sourced version, `LICENSE` file.
|
|
51
|
+
|
|
52
|
+
### Changed
|
|
53
|
+
|
|
54
|
+
- Project renamed to `ncarnate`; import package `ncarnate`; CLI
|
|
55
|
+
`ncarnate`.
|
|
56
|
+
- Python 3.10+ (was 3.7).
|
|
57
|
+
- Logging goes to stderr only when the CLI runs — importing the library
|
|
58
|
+
no longer creates log files as a side effect.
|
|
59
|
+
|
|
60
|
+
### Removed
|
|
61
|
+
|
|
62
|
+
- `setup.py`, `MANIFEST.in`, `version.json` (and its malformed
|
|
63
|
+
`"1.0.0."` version string), committed `__pycache__`, and the stray
|
|
64
|
+
`LOG_FILE`.
|
|
65
|
+
|
|
66
|
+
## [1.0.0] - 2020-05-28
|
|
67
|
+
|
|
68
|
+
Initial private version (`netcdf_recompressor`), never published:
|
|
69
|
+
a single-purpose netCDF4/HDF5 recompression script with a `setup.py`
|
|
70
|
+
package skeleton.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
title: "ncarnate"
|
|
4
|
+
abstract: >-
|
|
5
|
+
Reincarnate legacy scientific data as modern netCDF4: losslessly
|
|
6
|
+
recompress netCDF/HDF5 files and convert HDF4/HDF-EOS2 granules to
|
|
7
|
+
CF-annotated netCDF4 with reconstructed grid and swath geolocation.
|
|
8
|
+
type: software
|
|
9
|
+
authors:
|
|
10
|
+
- family-names: "Shepherd"
|
|
11
|
+
given-names: "Erick"
|
|
12
|
+
email: "Contact@ErickShepherd.com"
|
|
13
|
+
version: "2.0.0"
|
|
14
|
+
date-released: "2026-07-08"
|
|
15
|
+
license: MIT
|
|
16
|
+
repository-code: "https://github.com/ErickShepherd/ncarnate"
|
|
17
|
+
url: "https://github.com/ErickShepherd/ncarnate"
|
|
18
|
+
keywords:
|
|
19
|
+
- netCDF
|
|
20
|
+
- HDF4
|
|
21
|
+
- HDF-EOS2
|
|
22
|
+
- compression
|
|
23
|
+
- CF conventions
|
|
24
|
+
- atmospheric science
|
|
25
|
+
- remote sensing
|
ncarnate-2.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020-2026 Erick Edward Shepherd
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ncarnate-2.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ncarnate
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Losslessly rewrite netCDF4/HDF5 files with different compression settings.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ErickShepherd/ncarnate
|
|
6
|
+
Project-URL: Source, https://github.com/ErickShepherd/ncarnate
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/ErickShepherd/ncarnate/issues
|
|
8
|
+
Author-email: Erick Shepherd <Contact@ErickShepherd.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: HDF-EOS2,HDF4,HDF5,atmospheric science,compression,netCDF,netCDF4,recompression
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: netcdf4>=1.6
|
|
26
|
+
Requires-Dist: numpy>=1.26
|
|
27
|
+
Requires-Dist: pyhdf>=0.11.6
|
|
28
|
+
Requires-Dist: pyproj>=3.6
|
|
29
|
+
Requires-Dist: tqdm>=4.66
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: pytest; extra == 'test'
|
|
32
|
+
Description-Content-Type: text/x-rst
|
|
33
|
+
|
|
34
|
+
ncarnate
|
|
35
|
+
========
|
|
36
|
+
|
|
37
|
+
|ci| |license| |python|
|
|
38
|
+
|
|
39
|
+
.. |ci| image:: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml/badge.svg
|
|
40
|
+
:target: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml
|
|
41
|
+
:alt: CI status
|
|
42
|
+
|
|
43
|
+
.. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg
|
|
44
|
+
:target: https://github.com/ErickShepherd/ncarnate/blob/master/LICENSE
|
|
45
|
+
:alt: MIT License
|
|
46
|
+
|
|
47
|
+
.. |python| image:: https://img.shields.io/badge/python-3.10%E2%80%933.13-blue.svg
|
|
48
|
+
:target: https://pypi.org/project/ncarnate/
|
|
49
|
+
:alt: Python 3.10-3.13
|
|
50
|
+
|
|
51
|
+
Reincarnate legacy scientific data as modern netCDF4.
|
|
52
|
+
|
|
53
|
+
ncarnate reads netCDF3, netCDF4/HDF5, and HDF4/HDF-EOS2 files and writes
|
|
54
|
+
recompressed, CF-annotated netCDF4. It does two jobs:
|
|
55
|
+
|
|
56
|
+
- **Recompress** netCDF/HDF5 files — change the compression level,
|
|
57
|
+
shuffle filter, or storage layout without changing a single stored
|
|
58
|
+
value.
|
|
59
|
+
- **Convert** HDF4 and HDF-EOS2 granules (AMSR-E, MODIS, and kin) to
|
|
60
|
+
netCDF4, reconstructing the CF coordinates that modern tools (xarray,
|
|
61
|
+
QGIS, Panoply) need: grid projections become CF grid mappings with
|
|
62
|
+
1-D ``x``/``y`` and 2-D ``lat``/``lon`` coordinates, swath geolocation
|
|
63
|
+
is attached as CF coordinates, and dimension-mapped (e.g. 5 km → 1 km)
|
|
64
|
+
geolocation is interpolated through ECEF space.
|
|
65
|
+
|
|
66
|
+
The fidelity contract
|
|
67
|
+
---------------------
|
|
68
|
+
|
|
69
|
+
Converting or recompressing a file changes *storage*, never *science
|
|
70
|
+
data*:
|
|
71
|
+
|
|
72
|
+
- Every variable's stored values are preserved **bit-identically** —
|
|
73
|
+
packed integers stay packed; ``scale_factor``/``add_offset``/
|
|
74
|
+
``_FillValue`` are carried across as declarations, never applied.
|
|
75
|
+
- Every dimension (including unlimited-ness), attribute (including its
|
|
76
|
+
type), and group survives. HDF-EOS2 ``StructMetadata`` is preserved
|
|
77
|
+
verbatim; names netCDF cannot hold are sanitized with the original
|
|
78
|
+
recorded in a companion attribute.
|
|
79
|
+
- Geolocation reconstruction is strictly **additive**: the original
|
|
80
|
+
information always rides along, so the conversion never becomes the
|
|
81
|
+
only copy of the truth.
|
|
82
|
+
- Every output is **verified against the source value-for-value before
|
|
83
|
+
it replaces anything**. A source file is never destroyed by a failed
|
|
84
|
+
run, and HDF4 sources are never replaced at all.
|
|
85
|
+
- Unsupported constructs (user-defined netCDF types, unverified GCTP
|
|
86
|
+
projections, exotic swath layouts) **fail loud** with a named error
|
|
87
|
+
rather than guessing — a wrong coordinate is worse than a refused
|
|
88
|
+
conversion. ``--no-geolocation`` converts the raw payload anyway.
|
|
89
|
+
|
|
90
|
+
The details, the guarantee boundary, and how the test suite pins each
|
|
91
|
+
clause live in ``docs/fidelity-notes.md``.
|
|
92
|
+
|
|
93
|
+
Installation
|
|
94
|
+
------------
|
|
95
|
+
|
|
96
|
+
.. code-block:: console
|
|
97
|
+
|
|
98
|
+
pip install ncarnate
|
|
99
|
+
|
|
100
|
+
On **Linux (x86_64)** and **macOS (arm64)**, every dependency — including
|
|
101
|
+
``pyhdf`` — installs as a self-contained binary wheel with no system
|
|
102
|
+
libraries required (the ``pyhdf`` wheels there bundle the HDF4 C
|
|
103
|
+
library). On platforms without a repaired ``pyhdf`` wheel (e.g. Linux
|
|
104
|
+
aarch64), building from sdist requires the system HDF4 library first
|
|
105
|
+
(Debian/Ubuntu: ``apt install libhdf4-dev``).
|
|
106
|
+
|
|
107
|
+
**Windows:** the netCDF/HDF5 *recompression* path works from PyPI wheels
|
|
108
|
+
out of the box, but the HDF4/HDF-EOS2 *conversion* path does **not** —
|
|
109
|
+
``pyhdf``'s Windows wheel ships no HDF4 runtime, so ``import pyhdf`` fails
|
|
110
|
+
with a DLL-load error. For HDF4 support on Windows, install
|
|
111
|
+
``pyhdf`` from **conda-forge** first (``conda install -c conda-forge
|
|
112
|
+
pyhdf``), which provides a properly linked build with the HDF4 runtime,
|
|
113
|
+
then ``pip install ncarnate`` into that same environment — or use **WSL**
|
|
114
|
+
and follow the Linux instructions.
|
|
115
|
+
|
|
116
|
+
Command line usage
|
|
117
|
+
------------------
|
|
118
|
+
|
|
119
|
+
.. code-block:: console
|
|
120
|
+
|
|
121
|
+
# Recompress a netCDF4 file in place (verified before replacement).
|
|
122
|
+
ncarnate observations.nc --complevel 9
|
|
123
|
+
|
|
124
|
+
# Keep the original; write observations_recompressed.nc beside it.
|
|
125
|
+
ncarnate --no-overwrite observations.nc
|
|
126
|
+
|
|
127
|
+
# Convert an HDF-EOS2 granule -> granule.nc with CF geolocation.
|
|
128
|
+
ncarnate AMSR_E_L3_SeaIce12km_B02_20020619.hdf
|
|
129
|
+
|
|
130
|
+
# Convert the raw SDS payload only (unsupported-projection escape hatch).
|
|
131
|
+
ncarnate --no-geolocation granule.hdf
|
|
132
|
+
|
|
133
|
+
# Recurse over a directory tree.
|
|
134
|
+
ncarnate -r /data/archive
|
|
135
|
+
|
|
136
|
+
Exit codes: ``0`` success, ``1`` one or more files failed, ``2`` bad
|
|
137
|
+
input paths or arguments.
|
|
138
|
+
|
|
139
|
+
Library usage
|
|
140
|
+
-------------
|
|
141
|
+
|
|
142
|
+
.. code-block:: python
|
|
143
|
+
|
|
144
|
+
from ncarnate import recompress
|
|
145
|
+
|
|
146
|
+
# Lossless recompression; returns the output path.
|
|
147
|
+
recompress("observations.nc", complevel=9)
|
|
148
|
+
|
|
149
|
+
# HDF-EOS2 conversion; the .hdf source is never replaced.
|
|
150
|
+
recompress("granule.hdf", dst="granule.nc")
|
|
151
|
+
|
|
152
|
+
Example
|
|
153
|
+
-------
|
|
154
|
+
|
|
155
|
+
The AMSR-E daily 12.5 km sea-ice granule this project grew up around:
|
|
156
|
+
|
|
157
|
+
=============================================== ========== ===========
|
|
158
|
+
File Input Output
|
|
159
|
+
=============================================== ========== ===========
|
|
160
|
+
netCDF4 recompression (``--complevel 9``) 42.6 MB 19.9 MB
|
|
161
|
+
HDF-EOS2 → netCDF4 (+ reconstructed lat/lon) 60.2 MB 35.5 MB
|
|
162
|
+
=============================================== ========== ===========
|
|
163
|
+
|
|
164
|
+
Both outputs re-read bit-identically to their sources; the conversion
|
|
165
|
+
additionally carries CF ``polar_stereographic`` grid mappings and
|
|
166
|
+
coordinates for both hemispheric grids. The northern grid's
|
|
167
|
+
reconstructed latitudes/longitudes agree with The HDF Group's
|
|
168
|
+
independent conversion of the same granule to within 10\ :sup:`-5`
|
|
169
|
+
degrees (about a metre), the tolerance the test suite enforces.
|
|
170
|
+
|
|
171
|
+
Supported inputs
|
|
172
|
+
----------------
|
|
173
|
+
|
|
174
|
+
- **netCDF4 / HDF5** and **netCDF3** — recompressed via the netCDF4
|
|
175
|
+
library.
|
|
176
|
+
- **HDF4 / HDF-EOS2** — read via the pyhdf SD API. GRID structures with
|
|
177
|
+
GCTP polar-stereographic, geographic, and Lambert-azimuthal
|
|
178
|
+
(EASE-Grid) projections; SWATH structures with direct or
|
|
179
|
+
dimension-mapped geolocation. Output is always netCDF4 — HDF4 is
|
|
180
|
+
never written.
|
|
181
|
+
|
|
182
|
+
Development
|
|
183
|
+
-----------
|
|
184
|
+
|
|
185
|
+
.. code-block:: console
|
|
186
|
+
|
|
187
|
+
pip install -e ".[test]"
|
|
188
|
+
ruff check .
|
|
189
|
+
pytest
|
|
190
|
+
|
|
191
|
+
The test suite runs entirely offline against small committed fixtures
|
|
192
|
+
trimmed from real granules (provenance sidecars included); cross-checks
|
|
193
|
+
against the raw multi-MB granules self-skip where the local granule
|
|
194
|
+
store is absent.
|
|
195
|
+
|
|
196
|
+
License
|
|
197
|
+
-------
|
|
198
|
+
|
|
199
|
+
MIT — see ``LICENSE``.
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
ncarnate
|
|
2
|
+
========
|
|
3
|
+
|
|
4
|
+
|ci| |license| |python|
|
|
5
|
+
|
|
6
|
+
.. |ci| image:: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml/badge.svg
|
|
7
|
+
:target: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml
|
|
8
|
+
:alt: CI status
|
|
9
|
+
|
|
10
|
+
.. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg
|
|
11
|
+
:target: https://github.com/ErickShepherd/ncarnate/blob/master/LICENSE
|
|
12
|
+
:alt: MIT License
|
|
13
|
+
|
|
14
|
+
.. |python| image:: https://img.shields.io/badge/python-3.10%E2%80%933.13-blue.svg
|
|
15
|
+
:target: https://pypi.org/project/ncarnate/
|
|
16
|
+
:alt: Python 3.10-3.13
|
|
17
|
+
|
|
18
|
+
Reincarnate legacy scientific data as modern netCDF4.
|
|
19
|
+
|
|
20
|
+
ncarnate reads netCDF3, netCDF4/HDF5, and HDF4/HDF-EOS2 files and writes
|
|
21
|
+
recompressed, CF-annotated netCDF4. It does two jobs:
|
|
22
|
+
|
|
23
|
+
- **Recompress** netCDF/HDF5 files — change the compression level,
|
|
24
|
+
shuffle filter, or storage layout without changing a single stored
|
|
25
|
+
value.
|
|
26
|
+
- **Convert** HDF4 and HDF-EOS2 granules (AMSR-E, MODIS, and kin) to
|
|
27
|
+
netCDF4, reconstructing the CF coordinates that modern tools (xarray,
|
|
28
|
+
QGIS, Panoply) need: grid projections become CF grid mappings with
|
|
29
|
+
1-D ``x``/``y`` and 2-D ``lat``/``lon`` coordinates, swath geolocation
|
|
30
|
+
is attached as CF coordinates, and dimension-mapped (e.g. 5 km → 1 km)
|
|
31
|
+
geolocation is interpolated through ECEF space.
|
|
32
|
+
|
|
33
|
+
The fidelity contract
|
|
34
|
+
---------------------
|
|
35
|
+
|
|
36
|
+
Converting or recompressing a file changes *storage*, never *science
|
|
37
|
+
data*:
|
|
38
|
+
|
|
39
|
+
- Every variable's stored values are preserved **bit-identically** —
|
|
40
|
+
packed integers stay packed; ``scale_factor``/``add_offset``/
|
|
41
|
+
``_FillValue`` are carried across as declarations, never applied.
|
|
42
|
+
- Every dimension (including unlimited-ness), attribute (including its
|
|
43
|
+
type), and group survives. HDF-EOS2 ``StructMetadata`` is preserved
|
|
44
|
+
verbatim; names netCDF cannot hold are sanitized with the original
|
|
45
|
+
recorded in a companion attribute.
|
|
46
|
+
- Geolocation reconstruction is strictly **additive**: the original
|
|
47
|
+
information always rides along, so the conversion never becomes the
|
|
48
|
+
only copy of the truth.
|
|
49
|
+
- Every output is **verified against the source value-for-value before
|
|
50
|
+
it replaces anything**. A source file is never destroyed by a failed
|
|
51
|
+
run, and HDF4 sources are never replaced at all.
|
|
52
|
+
- Unsupported constructs (user-defined netCDF types, unverified GCTP
|
|
53
|
+
projections, exotic swath layouts) **fail loud** with a named error
|
|
54
|
+
rather than guessing — a wrong coordinate is worse than a refused
|
|
55
|
+
conversion. ``--no-geolocation`` converts the raw payload anyway.
|
|
56
|
+
|
|
57
|
+
The details, the guarantee boundary, and how the test suite pins each
|
|
58
|
+
clause live in ``docs/fidelity-notes.md``.
|
|
59
|
+
|
|
60
|
+
Installation
|
|
61
|
+
------------
|
|
62
|
+
|
|
63
|
+
.. code-block:: console
|
|
64
|
+
|
|
65
|
+
pip install ncarnate
|
|
66
|
+
|
|
67
|
+
On **Linux (x86_64)** and **macOS (arm64)**, every dependency — including
|
|
68
|
+
``pyhdf`` — installs as a self-contained binary wheel with no system
|
|
69
|
+
libraries required (the ``pyhdf`` wheels there bundle the HDF4 C
|
|
70
|
+
library). On platforms without a repaired ``pyhdf`` wheel (e.g. Linux
|
|
71
|
+
aarch64), building from sdist requires the system HDF4 library first
|
|
72
|
+
(Debian/Ubuntu: ``apt install libhdf4-dev``).
|
|
73
|
+
|
|
74
|
+
**Windows:** the netCDF/HDF5 *recompression* path works from PyPI wheels
|
|
75
|
+
out of the box, but the HDF4/HDF-EOS2 *conversion* path does **not** —
|
|
76
|
+
``pyhdf``'s Windows wheel ships no HDF4 runtime, so ``import pyhdf`` fails
|
|
77
|
+
with a DLL-load error. For HDF4 support on Windows, install
|
|
78
|
+
``pyhdf`` from **conda-forge** first (``conda install -c conda-forge
|
|
79
|
+
pyhdf``), which provides a properly linked build with the HDF4 runtime,
|
|
80
|
+
then ``pip install ncarnate`` into that same environment — or use **WSL**
|
|
81
|
+
and follow the Linux instructions.
|
|
82
|
+
|
|
83
|
+
Command line usage
|
|
84
|
+
------------------
|
|
85
|
+
|
|
86
|
+
.. code-block:: console
|
|
87
|
+
|
|
88
|
+
# Recompress a netCDF4 file in place (verified before replacement).
|
|
89
|
+
ncarnate observations.nc --complevel 9
|
|
90
|
+
|
|
91
|
+
# Keep the original; write observations_recompressed.nc beside it.
|
|
92
|
+
ncarnate --no-overwrite observations.nc
|
|
93
|
+
|
|
94
|
+
# Convert an HDF-EOS2 granule -> granule.nc with CF geolocation.
|
|
95
|
+
ncarnate AMSR_E_L3_SeaIce12km_B02_20020619.hdf
|
|
96
|
+
|
|
97
|
+
# Convert the raw SDS payload only (unsupported-projection escape hatch).
|
|
98
|
+
ncarnate --no-geolocation granule.hdf
|
|
99
|
+
|
|
100
|
+
# Recurse over a directory tree.
|
|
101
|
+
ncarnate -r /data/archive
|
|
102
|
+
|
|
103
|
+
Exit codes: ``0`` success, ``1`` one or more files failed, ``2`` bad
|
|
104
|
+
input paths or arguments.
|
|
105
|
+
|
|
106
|
+
Library usage
|
|
107
|
+
-------------
|
|
108
|
+
|
|
109
|
+
.. code-block:: python
|
|
110
|
+
|
|
111
|
+
from ncarnate import recompress
|
|
112
|
+
|
|
113
|
+
# Lossless recompression; returns the output path.
|
|
114
|
+
recompress("observations.nc", complevel=9)
|
|
115
|
+
|
|
116
|
+
# HDF-EOS2 conversion; the .hdf source is never replaced.
|
|
117
|
+
recompress("granule.hdf", dst="granule.nc")
|
|
118
|
+
|
|
119
|
+
Example
|
|
120
|
+
-------
|
|
121
|
+
|
|
122
|
+
The AMSR-E daily 12.5 km sea-ice granule this project grew up around:
|
|
123
|
+
|
|
124
|
+
=============================================== ========== ===========
|
|
125
|
+
File Input Output
|
|
126
|
+
=============================================== ========== ===========
|
|
127
|
+
netCDF4 recompression (``--complevel 9``) 42.6 MB 19.9 MB
|
|
128
|
+
HDF-EOS2 → netCDF4 (+ reconstructed lat/lon) 60.2 MB 35.5 MB
|
|
129
|
+
=============================================== ========== ===========
|
|
130
|
+
|
|
131
|
+
Both outputs re-read bit-identically to their sources; the conversion
|
|
132
|
+
additionally carries CF ``polar_stereographic`` grid mappings and
|
|
133
|
+
coordinates for both hemispheric grids. The northern grid's
|
|
134
|
+
reconstructed latitudes/longitudes agree with The HDF Group's
|
|
135
|
+
independent conversion of the same granule to within 10\ :sup:`-5`
|
|
136
|
+
degrees (about a metre), the tolerance the test suite enforces.
|
|
137
|
+
|
|
138
|
+
Supported inputs
|
|
139
|
+
----------------
|
|
140
|
+
|
|
141
|
+
- **netCDF4 / HDF5** and **netCDF3** — recompressed via the netCDF4
|
|
142
|
+
library.
|
|
143
|
+
- **HDF4 / HDF-EOS2** — read via the pyhdf SD API. GRID structures with
|
|
144
|
+
GCTP polar-stereographic, geographic, and Lambert-azimuthal
|
|
145
|
+
(EASE-Grid) projections; SWATH structures with direct or
|
|
146
|
+
dimension-mapped geolocation. Output is always netCDF4 — HDF4 is
|
|
147
|
+
never written.
|
|
148
|
+
|
|
149
|
+
Development
|
|
150
|
+
-----------
|
|
151
|
+
|
|
152
|
+
.. code-block:: console
|
|
153
|
+
|
|
154
|
+
pip install -e ".[test]"
|
|
155
|
+
ruff check .
|
|
156
|
+
pytest
|
|
157
|
+
|
|
158
|
+
The test suite runs entirely offline against small committed fixtures
|
|
159
|
+
trimmed from real granules (provenance sidecars included); cross-checks
|
|
160
|
+
against the raw multi-MB granules self-skip where the local granule
|
|
161
|
+
store is absent.
|
|
162
|
+
|
|
163
|
+
License
|
|
164
|
+
-------
|
|
165
|
+
|
|
166
|
+
MIT — see ``LICENSE``.
|