ncarnate 2.2.2__tar.gz → 2.3.0__tar.gz

ncarnate-2.3.0/.gitattributes

@@ -0,0 +1,2 @@
+# Keep release source bytes consistent in Windows and Linux checkouts.
+* text=auto eol=lf

{ncarnate-2.2.2 → ncarnate-2.3.0}/.github/workflows/ci.yml

@@ -5,6 +5,17 @@ on:
     branches: [main]
   pull_request:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      release_tag:
+        type: string
+        default: ''
+      publish_target:
+        type: string
+        default: ''
+
+permissions:
+  contents: read
 
 # The pip-based jobs (test, test-windows-degraded, build) install no system
 # HDF libraries on purpose: the netCDF4 wheels bundle libhdf5/libnetcdf, and
@@ -16,6 +27,20 @@ on:
 # the *supported* install channel delivers full HDF4 capability (KD-L7, G2).
 
 jobs:
+  docs-and-metadata:
+    name: Documentation and release metadata
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: "3.12"
+      - run: python -m pip install -r docs/requirements.txt PyYAML cffconvert pytest
+      - run: python -m pytest -q tools/tests
+      - run: python tools/validate_release_metadata.py
+      - run: cffconvert --validate
+      - run: python -m sphinx -W --keep-going -b html docs docs/_build/html
+
   lint:
     name: ruff
     runs-on: ubuntu-latest
@@ -41,15 +66,13 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.10", "3.11", "3.12", "3.13"]
         # Smoke the self-contained-wheel install story (pyhdf + netCDF4 +
-        # pyproj, no system HDF libs) on macOS too. Windows is deliberately
-        # excluded from the *full* matrix: pyhdf's Windows wheel ships no
-        # HDF4 runtime, so the HDF4 fixtures can't run there. Windows-pip
-        # instead gets the dedicated degraded-capability job below
-        # (KD-L4/KD-L7), which pins exactly what must keep working — and
-        # exactly how the HDF4 path must refuse — without a usable pyhdf.
+        # pyproj, no system HDF libs) on macOS and Windows. A separate
+        # injected-missing-runtime job retains degraded-capability coverage.
         include:
           - os: macos-latest
             python-version: "3.12"
+          - os: windows-latest
+            python-version: "3.11"
     steps:
       - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09  # v5.1.0
       - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
@@ -71,9 +94,8 @@ jobs:
         with:
           python-version: "3.12"
       - name: Install package with test extras (pip wheels only)
-        # pyhdf's Windows wheel *installs* but cannot *import* (it ships no
-        # HDF4 runtime) — which is precisely the degraded environment this
-        # job exists to pin (KD-L4/KD-L7, gate G2).
+        # Tests inject an unavailable runtime, independently of whether the
+        # installed pyhdf wheel works on this particular Windows runner.
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e ".[test]"
@@ -85,8 +107,12 @@ jobs:
         run: python -m pytest tests/test_optional_hdf4.py -q
 
   test-conda-forge:
-    name: pytest full-capability (conda-forge)
-    runs-on: ubuntu-latest
+    name: pytest full-capability (conda-forge, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
     # conda-forge is the *supported* HDF4 install channel (the refusal and
     # the README point Windows/aarch64 users at `conda install -c
     # conda-forge ncarnate`). This lane proves that channel actually
@@ -120,12 +146,15 @@ jobs:
             pyproj
             tqdm
             pytest
+            pip
       - name: Install ncarnate from source without replacing native deps
         # --no-deps: install only ncarnate; the native stack (pyhdf,
         # netCDF4, numpy, pyproj, tqdm) stays the conda-forge build. Build
         # isolation still fetches the hatchling backend into a throwaway
         # env, which never touches the runtime dependencies above.
-        run: python -m pip install -e . --no-deps
+        run: |
+          python -m pip install build "hatchling>=1.27"
+          python -m pip install -e . --no-deps
       - name: Prove the conda-forge HDF4 runtime imports
         run: |
           python -c "from ncarnate.hdf4_runtime import require_hdf4_runtime; require_hdf4_runtime(); print('conda-forge HDF4 runtime OK')"
@@ -144,14 +173,75 @@ jobs:
         with:
           python-version: "3.12"
       - name: Install build tooling
-        run: python -m pip install build twine
+        run: python -m pip install build twine PyYAML cffconvert
+      - name: Validate release identity before building
+        env:
+          NCARNATE_RELEASE_TAG: ${{ inputs.release_tag || '' }}
+          NCARNATE_PUBLISH_TARGET: ${{ inputs.publish_target || '' }}
+        run: |
+          python tools/validate_release_metadata.py
+          cffconvert --validate
       - name: Build sdist + wheel
         run: python -m build
       - name: Check distributions
-        run: twine check dist/*
+        run: twine check --strict dist/*
       - name: Validate distributions
         run: >-
           python tools/validate_distributions.py
           --allowed-email dev@erickshepherd.com
           --allowed-email ca@trufo.ai
           dist/*
+      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  installed-packages:
+    name: Installed ${{ matrix.kind }} (${{ matrix.os }})
+    needs: build
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        kind: [wheel, sdist]
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: "3.12"
+      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
+        with:
+          name: dist
+          path: dist/
+      - name: Test installed package outside the source checkout
+        run: python tools/test_distribution.py dist --kind ${{ matrix.kind }}
+
+  zarr-companion:
+    name: Installed Zarr companion
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: "3.12"
+      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
+        with:
+          name: dist
+          path: dist/
+      - name: Build and install the companion with the candidate wheel
+        run: |
+          python -m pip install build twine
+          python -m build companions/zarr-demo --outdir companion-dist
+          python -m twine check --strict companion-dist/*
+          python -m pip install dist/*.whl companion-dist/*.whl pytest
+          python -m pip check
+      - name: Run companion tests outside both source packages
+        run: |
+          cp -R companions/zarr-demo/tests "$RUNNER_TEMP/zarr-tests"
+          cd "$RUNNER_TEMP"
+          python -m pytest -q zarr-tests

ncarnate-2.3.0/.github/workflows/full-granules.yml

@@ -0,0 +1,39 @@
+name: Full-granule validation
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  full-granules:
+    name: Exact originals and independent reference
+    if: github.event_name == 'workflow_dispatch' || startsWith(github.head_ref, 'release/')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: "3.12"
+      - run: python -m pip install -e ".[test]"
+      - name: Set the corpus location outside the checkout
+        run: echo "NCARNATE_GRANULE_DIR=$RUNNER_TEMP/ncarnate-granules" >> "$GITHUB_ENV"
+      - name: Fetch and verify every required file before running tests
+        run: >-
+          python tools/corpus.py "$NCARNATE_GRANULE_DIR" --fetch --include-references
+          > corpus-verification.json
+      - name: Run full-size conversions and reference comparisons
+        run: python tools/test_full_corpus.py --junitxml full-granules.xml
+      - name: Retain verification and test results, including failures
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: full-granule-results
+          path: |
+            corpus-verification.json
+            full-granules.xml
+          if-no-files-found: warn

{ncarnate-2.2.2 → ncarnate-2.3.0}/.github/workflows/publish.yml

@@ -27,28 +27,19 @@ permissions:
   contents: read
 
 jobs:
-  build:
-    name: Build sdist + wheel
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09  # v5.1.0
-      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
-        with:
-          python-version: "3.12"
-      - name: Build
-        run: |
-          python -m pip install --upgrade build twine
-          python -m build
-          python -m twine check dist/*
-          python tools/validate_distributions.py --allowed-email dev@erickshepherd.com --allowed-email ca@trufo.ai dist/*
-      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
-        with:
-          name: dist
-          path: dist/
+  verify:
+    name: Test and build the exact release revision
+    # A local reusable workflow uses the same revision as this caller.
+    # Its dist artifact is installed by the package tests and then published
+    # unchanged below. Failure of ANY CI job prevents either upload.
+    uses: ./.github/workflows/ci.yml
+    with:
+      release_tag: ${{ github.event_name == 'release' && github.event.release.tag_name || (github.ref_type == 'tag' && github.ref_name || '') }}
+      publish_target: ${{ github.event_name == 'release' && 'pypi' || inputs.target }}
 
   publish-testpypi:
     name: Publish to TestPyPI
-    needs: build
+    needs: verify
     if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
     runs-on: ubuntu-latest
     environment: testpypi
@@ -62,11 +53,11 @@ jobs:
       - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247  # v1.14.1
         with:
           repository-url: https://test.pypi.org/legacy/
-          skip-existing: true  # re-dispatching a dry run for an existing version won't fail
+          # Duplicate versions fail: success must mean these files were uploaded.
 
   publish-pypi:
     name: Publish to PyPI
-    needs: build
+    needs: verify
     if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
     runs-on: ubuntu-latest
     environment: pypi

{ncarnate-2.2.2 → ncarnate-2.3.0}/.gitignore

@@ -23,3 +23,9 @@ docs/audits/*.audit-state.json
 
 # Sphinx documentation build output.
 docs/_build/
+
+# Machine-local publication policy, requests and private reports.
+.publication-private/
+.publication-reports/
+.publication-cache/
+full-granules.xml

{ncarnate-2.2.2 → ncarnate-2.3.0}/.readthedocs.yaml

@@ -3,7 +3,7 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
     python: "3.12"
 

ncarnate-2.3.0/.zenodo.json

@@ -0,0 +1,26 @@
+{
+  "upload_type": "software",
+  "version": "2.3.0",
+  "title": "ncarnate: Convert HDF4/HDF-EOS2 (MODIS, AMSR-E) to CF-annotated netCDF4 and losslessly recompress netCDF/HDF5",
+  "description": "A Python library and command-line tool for converting HDF4/HDF-EOS2 granules to CF-annotated netCDF4 and losslessly recompressing netCDF/HDF5. Version 2.3.0 adds explicit-output prepared jobs, digest-verified resume, streamed copying and verification, tiled swath interpolation, and optional isolated conversion workers with OS-enforced memory limits on Windows and Linux. Outputs are verified value-for-value before publication. Generated coordinate fields can still require whole arrays, and the worker limit excludes the calling process.",
+  "access_right": "open",
+  "license": "MIT",
+  "keywords": [
+    "netCDF4",
+    "HDF4",
+    "HDF-EOS2",
+    "HDF5",
+    "CF conventions",
+    "MODIS",
+    "AMSR-E",
+    "remote sensing",
+    "atmospheric science",
+    "Python"
+  ],
+  "creators": [
+    {
+      "name": "Shepherd, Erick Edward",
+      "orcid": "0000-0002-4750-6100"
+    }
+  ]
+}

{ncarnate-2.2.2 → ncarnate-2.3.0}/CHANGELOG.md

@@ -4,6 +4,51 @@ All notable changes to this project are documented here. The format is
 based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and
 this project adheres to [Semantic Versioning](https://semver.org/).
 
+## Unreleased
+
+## [2.3.0] - 2026-10-08
+
+- Add explicit-output prepared plans, shared finite-batch collision checks and
+  digest-verified resume journals. Existing one-shot defaults remain compatible.
+- Store completion checkpoints separately so large batches remain resumable;
+  resolve directory aliases and check every output parent before execution.
+- Refuse unknown inputs before output-directory creation; reserve CLI result
+  journals before conversion and distinguish later reporting failures.
+- Bound hostile handoff records, refuse invalid text and audit modes, freeze
+  encoding options, and guard unsynchronized attribute inquiries.
+- Stream netCDF and HDF4 SDS copying and verification in slices; offer an
+  OS-enforced conversion-worker memory cap on Windows and Linux. Generated
+  HDF-EOS coordinates still materialize whole arrays within that worker cap,
+  while tiled swath interpolation avoids large full-grid temporaries.
+- Preserve reads of multidimensional variables named like dimensions when
+  configuring netCDF native caches (found with the complete MOD03 granule).
+- Finish large storage chunks before advancing and coalesce small chunks to
+  avoid repeated compression and excessive per-chunk Python calls.
+- Add exact-granule retrieval and verification, Windows test portability fixes,
+  and a separately packaged numeric Zarr/xarray demonstration.
+- Document the public staged, prepared, handoff and bounded-worker APIs on Read
+  the Docs, and synchronize package, citation, Zenodo and conda recipe metadata.
+- Gate publication on the complete CI suite, test built packages outside the
+  checkout on Linux, Windows and macOS, and test the separate Zarr companion.
+  Add release-PR and manual full-granule validation that refuses incomplete evidence; require
+  matching production tags and fail duplicate TestPyPI uploads.
+- Reconcile MYD05 fixture regeneration with its recorded trimmed metadata and
+  emit canonical source-archive gzip headers for publication inspection.
+
+### Compatibility notes
+
+- Existing one-shot source-replacement defaults remain unchanged. The new
+  prepared and bounded APIs always require a separate explicit output.
+- Encoding options are now immutable, enforcing the documented immutable-plan
+  contract. Use `dataclasses.replace(options, complevel=...)` to derive options;
+  assigning to a field now raises `FrozenInstanceError`.
+- Unknown audit modes and invalid UTF-8 result text are now deliberately refused.
+  Metadata-only audit is the only implemented audit depth.
+- The handoff schema stays at version 1. New diagnostic codes are append-only;
+  the classification ruleset advances independently to version 9.
+- Hard worker limits are available on Windows and Linux; macOS is explicitly
+  unsupported by `execute_bounded`. Ordinary conversion remains portable.
+
 ## [2.2.2] - 2026-07-21
 
 Maintenance release with packaging and CI updates. Conversion, recompression,

{ncarnate-2.2.2 → ncarnate-2.3.0}/CITATION.cff

@@ -5,14 +5,17 @@ abstract: >-
   Reincarnate legacy scientific data as modern netCDF4: losslessly
   recompress netCDF/HDF5 files and convert HDF4/HDF-EOS2 granules to
   CF-annotated netCDF4 with reconstructed grid and swath geolocation.
+  Supports verified resumable conversions, streamed data transfer, and
+  isolated conversion workers with memory limits on Windows and Linux.
 type: software
 authors:
   - family-names: "Shepherd"
     given-names: "Erick Edward"
     email: "dev@erickshepherd.com"
     orcid: "https://orcid.org/0000-0002-4750-6100"
-version: "2.2.2"
-date-released: "2026-07-21"
+version: "2.3.0"
+date-released: "2026-10-08"
+# This is the verified concept DOI, which identifies all versions.
 doi: "10.5281/zenodo.21288802"
 license: MIT
 repository-code: "https://github.com/ErickShepherd/ncarnate"

{ncarnate-2.2.2 → ncarnate-2.3.0}/CONTRIBUTING.md

@@ -30,12 +30,15 @@ or an issue labelled `question`. The README covers installation (including the
 ```console
 git clone https://github.com/ErickShepherd/ncarnate
 cd ncarnate
-pip install -e ".[test]"
+python -m pip install -e ".[test,release]" "ruff==0.15.*"
+python -m pip install -r docs/requirements.txt
 ```
 
-On Linux (x86_64) and macOS (arm64) every dependency — including `pyhdf` —
-installs as a binary wheel. On other platforms the HDF4 path may need a system
-HDF4 library first (see the README).
+Compatible pip wheels can supply the native dependencies. If a wheel is not
+available for your platform and Python version, use conda-forge or install the
+native libraries and build tools described in the README. Ordinary conversion
+and the hard memory-limit worker have different platform coverage: the worker
+supports Windows and Linux and deliberately refuses macOS.
 
 ## Making a change
 
@@ -47,12 +50,37 @@ HDF4 library first (see the README).
 4. Run the checks locally:
    ```console
    ruff check .
-   pytest
+   python -m pytest -q
+   python -m pytest -q tools/tests
+   python tools/validate_release_metadata.py
+   cffconvert --validate
+   python -m sphinx -W --keep-going -b html docs docs/_build/html
    ```
 5. Open a pull request describing the change and, for a conversion change, how
    it preserves the fidelity contract (stored values unchanged; output verified
    against the source before it replaces anything).
 
+For full-size regression evidence, fetch and verify the catalogued originals
+and independent reference with
+`python tools/corpus.py DATA_DIRECTORY --fetch --include-references`.
+Set `NCARNATE_GRANULE_DIR` to that directory and
+run `python tools/test_full_corpus.py --junitxml full-granules.xml`. That check
+fails on missing files or skipped/missing tests; ordinary pytest remains usable
+without the external corpus. Keep downloaded inputs and reports outside the
+tracked source tree. The manual full-granule GitHub workflow retains its reports.
+
+The Zarr demonstration requires Python 3.11 or newer and is a separate package:
+install `companions/zarr-demo` after the candidate ncarnate package, then run
+`python -m pytest -q companions/zarr-demo/tests`. Its CI job builds and tests the
+installed wheel outside both source packages.
+
+To exercise release installations locally, build to a directory outside the
+checkout with `python -m build --outdir PATH_TO_ARTIFACTS`, then run
+`python tools/test_distribution.py PATH_TO_ARTIFACTS --kind wheel` and repeat
+with `--kind sdist`. Each run creates a clean environment and tests the installed
+package. PyPI and TestPyPI wait for these jobs and the full source CI. Publication
+is a separate maintainer action; see `docs/releasing.rst`.
+
 ## Scope and design
 
 ncarnate deliberately **fails loud** on constructs it cannot convert correctly