ncarnate 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. ncarnate-2.0.0/.gitattributes +2 -0
  2. ncarnate-2.0.0/.github/FUNDING.yml +12 -0
  3. ncarnate-2.0.0/.github/workflows/ci.yml +70 -0
  4. ncarnate-2.0.0/.github/workflows/publish.yml +79 -0
  5. ncarnate-2.0.0/.gitignore +21 -0
  6. ncarnate-2.0.0/CHANGELOG.md +70 -0
  7. ncarnate-2.0.0/CITATION.cff +25 -0
  8. ncarnate-2.0.0/LICENSE +21 -0
  9. ncarnate-2.0.0/PKG-INFO +199 -0
  10. ncarnate-2.0.0/README.rst +166 -0
  11. ncarnate-2.0.0/docs/fidelity-notes.md +134 -0
  12. ncarnate-2.0.0/ncarnate/__init__.py +34 -0
  13. ncarnate-2.0.0/ncarnate/__main__.py +21 -0
  14. ncarnate-2.0.0/ncarnate/cli.py +288 -0
  15. ncarnate-2.0.0/ncarnate/constants.py +19 -0
  16. ncarnate-2.0.0/ncarnate/core.py +502 -0
  17. ncarnate-2.0.0/ncarnate/eos/__init__.py +12 -0
  18. ncarnate-2.0.0/ncarnate/eos/gctp.py +260 -0
  19. ncarnate-2.0.0/ncarnate/eos/grid.py +159 -0
  20. ncarnate-2.0.0/ncarnate/eos/structmeta.py +420 -0
  21. ncarnate-2.0.0/ncarnate/eos/swath.py +172 -0
  22. ncarnate-2.0.0/ncarnate/errors.py +85 -0
  23. ncarnate-2.0.0/ncarnate/formats.py +83 -0
  24. ncarnate-2.0.0/ncarnate/hdf4.py +1109 -0
  25. ncarnate-2.0.0/ncarnate/limits.py +64 -0
  26. ncarnate-2.0.0/pyproject.toml +101 -0
  27. ncarnate-2.0.0/tests/conftest.py +115 -0
  28. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_5daysnow_trim.hdf +0 -0
  29. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_5daysnow_trim.provenance.json +15 -0
  30. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_seaice12km_trim.hdf +0 -0
  31. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/amsre_seaice12km_trim.provenance.json +15 -0
  32. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/mod03_trim.hdf +0 -0
  33. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/mod03_trim.provenance.json +19 -0
  34. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/myd05_trim.hdf +0 -0
  35. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/myd05_trim.provenance.json +20 -0
  36. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/raingrid_trim.hdf +0 -0
  37. ncarnate-2.0.0/tests/fixtures/data/hdfeos2/raingrid_trim.provenance.json +15 -0
  38. ncarnate-2.0.0/tests/fixtures/data/netcdf/endianness.nc +0 -0
  39. ncarnate-2.0.0/tests/fixtures/data/netcdf/nested_groups.nc +0 -0
  40. ncarnate-2.0.0/tests/fixtures/data/netcdf/packed_fill.nc +0 -0
  41. ncarnate-2.0.0/tests/fixtures/data/netcdf/unlimited_dim.nc +0 -0
  42. ncarnate-2.0.0/tests/fixtures/make_fixtures.py +136 -0
  43. ncarnate-2.0.0/tests/fixtures/trim_hdfeos2.py +296 -0
  44. ncarnate-2.0.0/tests/test_cli.py +72 -0
  45. ncarnate-2.0.0/tests/test_eos_units.py +316 -0
  46. ncarnate-2.0.0/tests/test_formats.py +37 -0
  47. ncarnate-2.0.0/tests/test_hdf4_conversion.py +318 -0
  48. ncarnate-2.0.0/tests/test_limits.py +57 -0
  49. ncarnate-2.0.0/tests/test_raw_granules.py +96 -0
  50. ncarnate-2.0.0/tests/test_recompress_netcdf.py +133 -0
@@ -0,0 +1,2 @@
1
+ # Auto detect text files and perform LF normalization
2
+ * text=auto
@@ -0,0 +1,12 @@
1
+ # These are supported funding model platforms
2
+
3
+ github: [ErickShepherd] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
4
+ patreon: ErickShepherd # Replace with a single Patreon username
5
+ open_collective: # Replace with a single Open Collective username
6
+ ko_fi: # Replace with a single Ko-fi username
7
+ tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
8
+ community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9
+ liberapay: # Replace with a single Liberapay username
10
+ issuehunt: # Replace with a single IssueHunt username
11
+ otechie: # Replace with a single Otechie username
12
+ custom: ['paypal.me/ErickEShepherd', 'https://cash.app/$ErickShepherd', 'https://venmo.com/ErickShepherd', 'https://erickshepherd.com/pay/bitcoin/'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
@@ -0,0 +1,70 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, master]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ # No system HDF libraries are installed anywhere in this workflow on
10
+ # purpose: the netCDF4 wheels bundle libhdf5/libnetcdf, and the pyhdf
11
+ # >= 0.11.6 manylinux wheels bundle libhdf4 (verified against PyPI
12
+ # 2026-07-08 for cp310-cp313). Platforms without a pyhdf wheel (e.g.
13
+ # linux-aarch64) need a system libhdf4 for the sdist build instead.
14
+
15
+ jobs:
16
+ lint:
17
+ name: ruff
18
+ runs-on: ubuntu-latest
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+ - uses: actions/setup-python@v5
22
+ with:
23
+ python-version: "3.12"
24
+ - name: Install ruff
25
+ run: python -m pip install ruff
26
+ - name: Run ruff
27
+ run: ruff check .
28
+
29
+ test:
30
+ name: pytest (${{ matrix.os }} py${{ matrix.python-version }})
31
+ runs-on: ${{ matrix.os }}
32
+ strategy:
33
+ fail-fast: false
34
+ matrix:
35
+ os: [ubuntu-latest]
36
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
37
+ # Smoke the self-contained-wheel install story (pyhdf + netCDF4 +
38
+ # pyproj, no system HDF libs) on macOS too. Windows is deliberately
39
+ # excluded: pyhdf's Windows wheel ships no HDF4 runtime, so the
40
+ # HDF4 path can't import from PyPI wheels there (README documents
41
+ # the conda-forge / WSL route). The netCDF path works on Windows.
42
+ include:
43
+ - os: macos-latest
44
+ python-version: "3.12"
45
+ steps:
46
+ - uses: actions/checkout@v4
47
+ - uses: actions/setup-python@v5
48
+ with:
49
+ python-version: ${{ matrix.python-version }}
50
+ - name: Install package with test extras
51
+ run: |
52
+ python -m pip install --upgrade pip
53
+ python -m pip install -e ".[test]"
54
+ - name: Run tests (offline; raw-granule marks self-skip)
55
+ run: python -m pytest -q
56
+
57
+ build:
58
+ name: build + twine check
59
+ runs-on: ubuntu-latest
60
+ steps:
61
+ - uses: actions/checkout@v4
62
+ - uses: actions/setup-python@v5
63
+ with:
64
+ python-version: "3.12"
65
+ - name: Install build tooling
66
+ run: python -m pip install build twine
67
+ - name: Build sdist + wheel
68
+ run: python -m build
69
+ - name: Check distributions
70
+ run: twine check dist/*
@@ -0,0 +1,79 @@
1
+ name: Publish to PyPI
2
+
3
+ # Publishes the package via PyPI Trusted Publishing (OIDC) — no API token is
4
+ # stored anywhere (not in the repo, not in GitHub secrets). PyPI verifies the
5
+ # workflow's short-lived OIDC identity instead.
6
+ #
7
+ # Two ways to run:
8
+ # * A published GitHub Release -> builds and publishes to PyPI.
9
+ # * Manual "Run workflow" -> choose testpypi (dry run) or pypi.
10
+ #
11
+ # The publish jobs run in protected GitHub Environments (`pypi` / `testpypi`).
12
+ # Add yourself as a required reviewer on each (repo Settings -> Environments)
13
+ # so every upload pauses for your one-click approval before it leaves.
14
+
15
+ on:
16
+ release:
17
+ types: [released] # full releases only — pre-releases never auto-publish
18
+ workflow_dispatch:
19
+ inputs:
20
+ target:
21
+ description: "Where to publish"
22
+ type: choice
23
+ options: [testpypi, pypi]
24
+ default: testpypi
25
+
26
+ permissions:
27
+ contents: read
28
+
29
+ jobs:
30
+ build:
31
+ name: Build sdist + wheel
32
+ runs-on: ubuntu-latest
33
+ steps:
34
+ - uses: actions/checkout@v4
35
+ - uses: actions/setup-python@v5
36
+ with:
37
+ python-version: "3.12"
38
+ - name: Build
39
+ run: |
40
+ python -m pip install --upgrade build twine
41
+ python -m build
42
+ python -m twine check dist/*
43
+ - uses: actions/upload-artifact@v4
44
+ with:
45
+ name: dist
46
+ path: dist/
47
+
48
+ publish-testpypi:
49
+ name: Publish to TestPyPI
50
+ needs: build
51
+ if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
52
+ runs-on: ubuntu-latest
53
+ environment: testpypi
54
+ permissions:
55
+ id-token: write
56
+ steps:
57
+ - uses: actions/download-artifact@v4
58
+ with:
59
+ name: dist
60
+ path: dist/
61
+ - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
62
+ with:
63
+ repository-url: https://test.pypi.org/legacy/
64
+ skip-existing: true # re-dispatching a dry run for an existing version won't fail
65
+
66
+ publish-pypi:
67
+ name: Publish to PyPI
68
+ needs: build
69
+ if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
70
+ runs-on: ubuntu-latest
71
+ environment: pypi
72
+ permissions:
73
+ id-token: write
74
+ steps:
75
+ - uses: actions/download-artifact@v4
76
+ with:
77
+ name: dist
78
+ path: dist/
79
+ - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
@@ -0,0 +1,21 @@
1
+ # Python build/runtime artifacts.
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ dist/
6
+ build/
7
+
8
+ # Virtual environments.
9
+ venv/
10
+ .venv/
11
+
12
+ # Tooling caches.
13
+ .pytest_cache/
14
+ .ruff_cache/
15
+
16
+ # Local scratch.
17
+ tmp/
18
+
19
+ # Audit orchestration sidecars (ephemeral analysis).
20
+ docs/audits/*.run.log
21
+ docs/audits/*.audit-state.json
@@ -0,0 +1,70 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format is
4
+ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and
5
+ this project adheres to [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [2.0.0] - 2026-07-08
8
+
9
+ A ground-up rebuild of the 2020 utility, renamed from
10
+ `netcdf_recompressor` to **ncarnate**. Everything below is breaking
11
+ relative to the notional 1.x behavior; there are no downstream users
12
+ (1.x was never published).
13
+
14
+ ### Fixed
15
+
16
+ - The CLI recompresses again: a debugging `raise ValueError` committed
17
+ in 2020 made the shipped tool process nothing.
18
+ - Variables carrying `_FillValue` no longer crash the copy (`fill_value`
19
+ is declared at variable creation, as netCDF4 requires).
20
+ - Packed-integer data is no longer silently re-quantized through float:
21
+ values are copied raw (`set_auto_maskandscale(False)`), with
22
+ `scale_factor`/`add_offset`/`_FillValue` re-declared, never applied.
23
+ - Overwrite is safe: output is written to a temp file, verified
24
+ value-for-value against the source, and only then atomically moved
25
+ into place. A failed run never destroys the source.
26
+ - `-V`/`--version` works without a path; errors exit non-zero with
27
+ clear messages instead of vanishing into a log file.
28
+
29
+ ### Added
30
+
31
+ - **HDF4/HDF-EOS2 ingest** (read-only; output is always netCDF4):
32
+ SDS payloads convert bit-identically, one netCDF4 group per EOS
33
+ structure, EOS metadata preserved verbatim, hostile names sanitized
34
+ with originals recorded in companion attributes.
35
+ - **CF geolocation reconstruction** for HDF-EOS2: GCTP
36
+ polar-stereographic, geographic, and Lambert-azimuthal (EASE-Grid)
37
+ grids become CF grid mappings + `x`/`y` + 2-D `lat`/`lon`; swath
38
+ geolocation is CF-attached; dimension-mapped geolocation (5 km →
39
+ 1 km) is interpolated through ECEF space with fill propagation.
40
+ `--no-geolocation` converts payload-only. Unsupported constructs fail
41
+ loud with named errors.
42
+ - File-type dispatch by magic bytes (netCDF3 / netCDF4-HDF5 / HDF4),
43
+ independent of file extension.
44
+ - pytest suite (66 tests) pinning the fidelity contract over committed
45
+ fixtures trimmed from real granules, plus the geolocation verification
46
+ lattice (four of the design's five checks; the `eos2dump` external
47
+ reference is unimplemented — the same-granule THG reference covers
48
+ that grid); CI across CPython 3.10–3.13.
49
+ - Modern packaging: hatchling `pyproject.toml`, console entry point
50
+ `ncarnate`, single-sourced version, `LICENSE` file.
51
+
52
+ ### Changed
53
+
54
+ - Project renamed to `ncarnate`; import package `ncarnate`; CLI
55
+ `ncarnate`.
56
+ - Python 3.10+ (was 3.7).
57
+ - Logging goes to stderr only when the CLI runs — importing the library
58
+ no longer creates log files as a side effect.
59
+
60
+ ### Removed
61
+
62
+ - `setup.py`, `MANIFEST.in`, `version.json` (and its malformed
63
+ `"1.0.0."` version string), committed `__pycache__`, and the stray
64
+ `LOG_FILE`.
65
+
66
+ ## [1.0.0] - 2020-05-28
67
+
68
+ Initial private version (`netcdf_recompressor`), never published:
69
+ a single-purpose netCDF4/HDF5 recompression script with a `setup.py`
70
+ package skeleton.
@@ -0,0 +1,25 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it as below."
3
+ title: "ncarnate"
4
+ abstract: >-
5
+ Reincarnate legacy scientific data as modern netCDF4: losslessly
6
+ recompress netCDF/HDF5 files and convert HDF4/HDF-EOS2 granules to
7
+ CF-annotated netCDF4 with reconstructed grid and swath geolocation.
8
+ type: software
9
+ authors:
10
+ - family-names: "Shepherd"
11
+ given-names: "Erick"
12
+ email: "Contact@ErickShepherd.com"
13
+ version: "2.0.0"
14
+ date-released: "2026-07-08"
15
+ license: MIT
16
+ repository-code: "https://github.com/ErickShepherd/ncarnate"
17
+ url: "https://github.com/ErickShepherd/ncarnate"
18
+ keywords:
19
+ - netCDF
20
+ - HDF4
21
+ - HDF-EOS2
22
+ - compression
23
+ - CF conventions
24
+ - atmospheric science
25
+ - remote sensing
ncarnate-2.0.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020-2026 Erick Edward Shepherd
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,199 @@
1
+ Metadata-Version: 2.4
2
+ Name: ncarnate
3
+ Version: 2.0.0
4
+ Summary: Losslessly rewrite netCDF4/HDF5 files with different compression settings.
5
+ Project-URL: Homepage, https://github.com/ErickShepherd/ncarnate
6
+ Project-URL: Source, https://github.com/ErickShepherd/ncarnate
7
+ Project-URL: Bug Tracker, https://github.com/ErickShepherd/ncarnate/issues
8
+ Author-email: Erick Shepherd <Contact@ErickShepherd.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: HDF-EOS2,HDF4,HDF5,atmospheric science,compression,netCDF,netCDF4,recompression
12
+ Classifier: Development Status :: 5 - Production/Stable
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: MacOS
15
+ Classifier: Operating System :: Microsoft :: Windows
16
+ Classifier: Operating System :: POSIX :: Linux
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
23
+ Classifier: Topic :: Scientific/Engineering :: GIS
24
+ Requires-Python: >=3.10
25
+ Requires-Dist: netcdf4>=1.6
26
+ Requires-Dist: numpy>=1.26
27
+ Requires-Dist: pyhdf>=0.11.6
28
+ Requires-Dist: pyproj>=3.6
29
+ Requires-Dist: tqdm>=4.66
30
+ Provides-Extra: test
31
+ Requires-Dist: pytest; extra == 'test'
32
+ Description-Content-Type: text/x-rst
33
+
34
+ ncarnate
35
+ ========
36
+
37
+ |ci| |license| |python|
38
+
39
+ .. |ci| image:: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml/badge.svg
40
+ :target: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml
41
+ :alt: CI status
42
+
43
+ .. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg
44
+ :target: https://github.com/ErickShepherd/ncarnate/blob/master/LICENSE
45
+ :alt: MIT License
46
+
47
+ .. |python| image:: https://img.shields.io/badge/python-3.10%E2%80%933.13-blue.svg
48
+ :target: https://pypi.org/project/ncarnate/
49
+ :alt: Python 3.10-3.13
50
+
51
+ Reincarnate legacy scientific data as modern netCDF4.
52
+
53
+ ncarnate reads netCDF3, netCDF4/HDF5, and HDF4/HDF-EOS2 files and writes
54
+ recompressed, CF-annotated netCDF4. It does two jobs:
55
+
56
+ - **Recompress** netCDF/HDF5 files — change the compression level,
57
+ shuffle filter, or storage layout without changing a single stored
58
+ value.
59
+ - **Convert** HDF4 and HDF-EOS2 granules (AMSR-E, MODIS, and kin) to
60
+ netCDF4, reconstructing the CF coordinates that modern tools (xarray,
61
+ QGIS, Panoply) need: grid projections become CF grid mappings with
62
+ 1-D ``x``/``y`` and 2-D ``lat``/``lon`` coordinates, swath geolocation
63
+ is attached as CF coordinates, and dimension-mapped (e.g. 5 km → 1 km)
64
+ geolocation is interpolated through ECEF space.
65
+
66
+ The fidelity contract
67
+ ---------------------
68
+
69
+ Converting or recompressing a file changes *storage*, never *science
70
+ data*:
71
+
72
+ - Every variable's stored values are preserved **bit-identically** —
73
+ packed integers stay packed; ``scale_factor``/``add_offset``/
74
+ ``_FillValue`` are carried across as declarations, never applied.
75
+ - Every dimension (including unlimited-ness), attribute (including its
76
+ type), and group survives. HDF-EOS2 ``StructMetadata`` is preserved
77
+ verbatim; names netCDF cannot hold are sanitized with the original
78
+ recorded in a companion attribute.
79
+ - Geolocation reconstruction is strictly **additive**: the original
80
+ information always rides along, so the conversion never becomes the
81
+ only copy of the truth.
82
+ - Every output is **verified against the source value-for-value before
83
+ it replaces anything**. A source file is never destroyed by a failed
84
+ run, and HDF4 sources are never replaced at all.
85
+ - Unsupported constructs (user-defined netCDF types, unverified GCTP
86
+ projections, exotic swath layouts) **fail loud** with a named error
87
+ rather than guessing — a wrong coordinate is worse than a refused
88
+ conversion. ``--no-geolocation`` converts the raw payload anyway.
89
+
90
+ The details, the guarantee boundary, and how the test suite pins each
91
+ clause live in ``docs/fidelity-notes.md``.
92
+
93
+ Installation
94
+ ------------
95
+
96
+ .. code-block:: console
97
+
98
+ pip install ncarnate
99
+
100
+ On **Linux (x86_64)** and **macOS (arm64)**, every dependency — including
101
+ ``pyhdf`` — installs as a self-contained binary wheel with no system
102
+ libraries required (the ``pyhdf`` wheels there bundle the HDF4 C
103
+ library). On platforms without a repaired ``pyhdf`` wheel (e.g. Linux
104
+ aarch64), building from sdist requires the system HDF4 library first
105
+ (Debian/Ubuntu: ``apt install libhdf4-dev``).
106
+
107
+ **Windows:** the netCDF/HDF5 *recompression* path works from PyPI wheels
108
+ out of the box, but the HDF4/HDF-EOS2 *conversion* path does **not** —
109
+ ``pyhdf``'s Windows wheel ships no HDF4 runtime, so ``import pyhdf`` fails
110
+ with a DLL-load error. For HDF4 support on Windows, install
111
+ ``pyhdf`` from **conda-forge** first (``conda install -c conda-forge
112
+ pyhdf``), which provides a properly linked build with the HDF4 runtime,
113
+ then ``pip install ncarnate`` into that same environment — or use **WSL**
114
+ and follow the Linux instructions.
115
+
116
+ Command line usage
117
+ ------------------
118
+
119
+ .. code-block:: console
120
+
121
+ # Recompress a netCDF4 file in place (verified before replacement).
122
+ ncarnate observations.nc --complevel 9
123
+
124
+ # Keep the original; write observations_recompressed.nc beside it.
125
+ ncarnate --no-overwrite observations.nc
126
+
127
+ # Convert an HDF-EOS2 granule -> granule.nc with CF geolocation.
128
+ ncarnate AMSR_E_L3_SeaIce12km_B02_20020619.hdf
129
+
130
+ # Convert the raw SDS payload only (unsupported-projection escape hatch).
131
+ ncarnate --no-geolocation granule.hdf
132
+
133
+ # Recurse over a directory tree.
134
+ ncarnate -r /data/archive
135
+
136
+ Exit codes: ``0`` success, ``1`` one or more files failed, ``2`` bad
137
+ input paths or arguments.
138
+
139
+ Library usage
140
+ -------------
141
+
142
+ .. code-block:: python
143
+
144
+ from ncarnate import recompress
145
+
146
+ # Lossless recompression; returns the output path.
147
+ recompress("observations.nc", complevel=9)
148
+
149
+ # HDF-EOS2 conversion; the .hdf source is never replaced.
150
+ recompress("granule.hdf", dst="granule.nc")
151
+
152
+ Example
153
+ -------
154
+
155
+ The AMSR-E daily 12.5 km sea-ice granule this project grew up around:
156
+
157
+ =============================================== ========== ===========
158
+ File Input Output
159
+ =============================================== ========== ===========
160
+ netCDF4 recompression (``--complevel 9``) 42.6 MB 19.9 MB
161
+ HDF-EOS2 → netCDF4 (+ reconstructed lat/lon) 60.2 MB 35.5 MB
162
+ =============================================== ========== ===========
163
+
164
+ Both outputs re-read bit-identically to their sources; the conversion
165
+ additionally carries CF ``polar_stereographic`` grid mappings and
166
+ coordinates for both hemispheric grids. The northern grid's
167
+ reconstructed latitudes/longitudes agree with The HDF Group's
168
+ independent conversion of the same granule to within 10\ :sup:`-5`
169
+ degrees (about a metre), the tolerance the test suite enforces.
170
+
171
+ Supported inputs
172
+ ----------------
173
+
174
+ - **netCDF4 / HDF5** and **netCDF3** — recompressed via the netCDF4
175
+ library.
176
+ - **HDF4 / HDF-EOS2** — read via the pyhdf SD API. GRID structures with
177
+ GCTP polar-stereographic, geographic, and Lambert-azimuthal
178
+ (EASE-Grid) projections; SWATH structures with direct or
179
+ dimension-mapped geolocation. Output is always netCDF4 — HDF4 is
180
+ never written.
181
+
182
+ Development
183
+ -----------
184
+
185
+ .. code-block:: console
186
+
187
+ pip install -e ".[test]"
188
+ ruff check .
189
+ pytest
190
+
191
+ The test suite runs entirely offline against small committed fixtures
192
+ trimmed from real granules (provenance sidecars included); cross-checks
193
+ against the raw multi-MB granules self-skip where the local granule
194
+ store is absent.
195
+
196
+ License
197
+ -------
198
+
199
+ MIT — see ``LICENSE``.
@@ -0,0 +1,166 @@
1
+ ncarnate
2
+ ========
3
+
4
+ |ci| |license| |python|
5
+
6
+ .. |ci| image:: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml/badge.svg
7
+ :target: https://github.com/ErickShepherd/ncarnate/actions/workflows/ci.yml
8
+ :alt: CI status
9
+
10
+ .. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg
11
+ :target: https://github.com/ErickShepherd/ncarnate/blob/master/LICENSE
12
+ :alt: MIT License
13
+
14
+ .. |python| image:: https://img.shields.io/badge/python-3.10%E2%80%933.13-blue.svg
15
+ :target: https://pypi.org/project/ncarnate/
16
+ :alt: Python 3.10-3.13
17
+
18
+ Reincarnate legacy scientific data as modern netCDF4.
19
+
20
+ ncarnate reads netCDF3, netCDF4/HDF5, and HDF4/HDF-EOS2 files and writes
21
+ recompressed, CF-annotated netCDF4. It does two jobs:
22
+
23
+ - **Recompress** netCDF/HDF5 files — change the compression level,
24
+ shuffle filter, or storage layout without changing a single stored
25
+ value.
26
+ - **Convert** HDF4 and HDF-EOS2 granules (AMSR-E, MODIS, and kin) to
27
+ netCDF4, reconstructing the CF coordinates that modern tools (xarray,
28
+ QGIS, Panoply) need: grid projections become CF grid mappings with
29
+ 1-D ``x``/``y`` and 2-D ``lat``/``lon`` coordinates, swath geolocation
30
+ is attached as CF coordinates, and dimension-mapped (e.g. 5 km → 1 km)
31
+ geolocation is interpolated through ECEF space.
32
+
33
+ The fidelity contract
34
+ ---------------------
35
+
36
+ Converting or recompressing a file changes *storage*, never *science
37
+ data*:
38
+
39
+ - Every variable's stored values are preserved **bit-identically** —
40
+ packed integers stay packed; ``scale_factor``/``add_offset``/
41
+ ``_FillValue`` are carried across as declarations, never applied.
42
+ - Every dimension (including unlimited-ness), attribute (including its
43
+ type), and group survives. HDF-EOS2 ``StructMetadata`` is preserved
44
+ verbatim; names netCDF cannot hold are sanitized with the original
45
+ recorded in a companion attribute.
46
+ - Geolocation reconstruction is strictly **additive**: the original
47
+ information always rides along, so the conversion never becomes the
48
+ only copy of the truth.
49
+ - Every output is **verified against the source value-for-value before
50
+ it replaces anything**. A source file is never destroyed by a failed
51
+ run, and HDF4 sources are never replaced at all.
52
+ - Unsupported constructs (user-defined netCDF types, unverified GCTP
53
+ projections, exotic swath layouts) **fail loud** with a named error
54
+ rather than guessing — a wrong coordinate is worse than a refused
55
+ conversion. ``--no-geolocation`` converts the raw payload anyway.
56
+
57
+ The details, the guarantee boundary, and how the test suite pins each
58
+ clause live in ``docs/fidelity-notes.md``.
59
+
60
+ Installation
61
+ ------------
62
+
63
+ .. code-block:: console
64
+
65
+ pip install ncarnate
66
+
67
+ On **Linux (x86_64)** and **macOS (arm64)**, every dependency — including
68
+ ``pyhdf`` — installs as a self-contained binary wheel with no system
69
+ libraries required (the ``pyhdf`` wheels there bundle the HDF4 C
70
+ library). On platforms without a repaired ``pyhdf`` wheel (e.g. Linux
71
+ aarch64), building from sdist requires the system HDF4 library first
72
+ (Debian/Ubuntu: ``apt install libhdf4-dev``).
73
+
74
+ **Windows:** the netCDF/HDF5 *recompression* path works from PyPI wheels
75
+ out of the box, but the HDF4/HDF-EOS2 *conversion* path does **not** —
76
+ ``pyhdf``'s Windows wheel ships no HDF4 runtime, so ``import pyhdf`` fails
77
+ with a DLL-load error. For HDF4 support on Windows, install
78
+ ``pyhdf`` from **conda-forge** first (``conda install -c conda-forge
79
+ pyhdf``), which provides a properly linked build with the HDF4 runtime,
80
+ then ``pip install ncarnate`` into that same environment — or use **WSL**
81
+ and follow the Linux instructions.
82
+
83
+ Command line usage
84
+ ------------------
85
+
86
+ .. code-block:: console
87
+
88
+ # Recompress a netCDF4 file in place (verified before replacement).
89
+ ncarnate observations.nc --complevel 9
90
+
91
+ # Keep the original; write observations_recompressed.nc beside it.
92
+ ncarnate --no-overwrite observations.nc
93
+
94
+ # Convert an HDF-EOS2 granule -> granule.nc with CF geolocation.
95
+ ncarnate AMSR_E_L3_SeaIce12km_B02_20020619.hdf
96
+
97
+ # Convert the raw SDS payload only (unsupported-projection escape hatch).
98
+ ncarnate --no-geolocation granule.hdf
99
+
100
+ # Recurse over a directory tree.
101
+ ncarnate -r /data/archive
102
+
103
+ Exit codes: ``0`` success, ``1`` one or more files failed, ``2`` bad
104
+ input paths or arguments.
105
+
106
+ Library usage
107
+ -------------
108
+
109
+ .. code-block:: python
110
+
111
+ from ncarnate import recompress
112
+
113
+ # Lossless recompression; returns the output path.
114
+ recompress("observations.nc", complevel=9)
115
+
116
+ # HDF-EOS2 conversion; the .hdf source is never replaced.
117
+ recompress("granule.hdf", dst="granule.nc")
118
+
119
+ Example
120
+ -------
121
+
122
+ The AMSR-E daily 12.5 km sea-ice granule this project grew up around:
123
+
124
+ =============================================== ========== ===========
125
+ File Input Output
126
+ =============================================== ========== ===========
127
+ netCDF4 recompression (``--complevel 9``) 42.6 MB 19.9 MB
128
+ HDF-EOS2 → netCDF4 (+ reconstructed lat/lon) 60.2 MB 35.5 MB
129
+ =============================================== ========== ===========
130
+
131
+ Both outputs re-read bit-identically to their sources; the conversion
132
+ additionally carries CF ``polar_stereographic`` grid mappings and
133
+ coordinates for both hemispheric grids. The northern grid's
134
+ reconstructed latitudes/longitudes agree with The HDF Group's
135
+ independent conversion of the same granule to within 10\ :sup:`-5`
136
+ degrees (about a metre), the tolerance the test suite enforces.
137
+
138
+ Supported inputs
139
+ ----------------
140
+
141
+ - **netCDF4 / HDF5** and **netCDF3** — recompressed via the netCDF4
142
+ library.
143
+ - **HDF4 / HDF-EOS2** — read via the pyhdf SD API. GRID structures with
144
+ GCTP polar-stereographic, geographic, and Lambert-azimuthal
145
+ (EASE-Grid) projections; SWATH structures with direct or
146
+ dimension-mapped geolocation. Output is always netCDF4 — HDF4 is
147
+ never written.
148
+
149
+ Development
150
+ -----------
151
+
152
+ .. code-block:: console
153
+
154
+ pip install -e ".[test]"
155
+ ruff check .
156
+ pytest
157
+
158
+ The test suite runs entirely offline against small committed fixtures
159
+ trimmed from real granules (provenance sidecars included); cross-checks
160
+ against the raw multi-MB granules self-skip where the local granule
161
+ store is absent.
162
+
163
+ License
164
+ -------
165
+
166
+ MIT — see ``LICENSE``.