umfive 0.2.0__tar.gz

umfive-0.2.0/.flake8

@@ -0,0 +1,29 @@
+# Configure the flake8 code style checker. Note this must be in INI style
+# (see https://flake8.pycqa.org/en/latest/). To run the flake8 checker with
+# this configuration, simply execute 'flake8' under the root of the repo.
+
+[flake8]
+# Ignores should include listing of pycodestyle ignores as in test_style.py.
+# Note W503 is incompatible with PEP8 which we want to abide by
+ignore =
+    W605,
+    E272,
+    E402,
+    E501,
+    E203,
+    W503
+# Exclude these directories/files, otherwise check all *.py files
+exclude =
+    .git,
+    __pycache__,
+    cfdm.egg-info,
+    docs/build/
+
+# Ignore certain rules for some files (can't do this via comments in the files)
+per-file-ignores =
+    # Ignore unused imports in __init__ modules, as they are there to provide
+    # the project namespace (decide not to use __all__ to contain these, as
+    # suggested in some flake8 discussions e.g. by the core maintainer at
+    # https://stackoverflow.com/questions/59167405/)
+    __init__.py: F401
+    */__init__.py: F401

umfive-0.2.0/.github/workflows/build-and-deploy-wheels-on-pypi.yml

@@ -0,0 +1,101 @@
+name: Build and Publish Wheels
+
+on:
+  release:
+    types: [published]
+  # use this for testing
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  id-token: write   # required for PyPI trusted publishing
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]  # windows-latest fails
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.2.1
+        env:
+          # Build CPython 3.11-3.14
+          CIBW_BUILD: "cp311-* cp312-* cp313-* cp314-*"
+          CIBW_SKIP: "*-musllinux_* pp*"
+          # Run tests after build (optional)
+          # CIBW_TEST_COMMAND: "python -c \"import umfive; print('import ok')\""
+
+      - name: Upload wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build backend
+        run: python -m pip install --upgrade build
+
+      - name: Build sdist
+        run: python -m build --sdist
+
+      - name: Upload sdist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to PyPI
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    # publish only on version tags like v0.1.0
+    if: startsWith(github.ref, 'refs/tags')
+    environment:
+      name: pypi
+      url: https://pypi.org/project/umfive/
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: distfiles
+
+      - name: Flatten artifacts
+        run: |
+          mkdir upload
+          find distfiles -type f \( -name "*.whl" -o -name "*.tar.gz" \) -exec cp {} upload/ \;
+          ls -la upload
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: upload

umfive-0.2.0/.github/workflows/pytest.yml

@@ -0,0 +1,57 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Umfive Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: '0 0 * * *'  # nightly
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  build:
+
+    name: Test Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: umfive
+          environment-file: environment.yml
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          use-mamba: true
+      - name: Install Umfive Test Mode
+        run: |
+          mamba install -c conda-forge pip
+          pip install -e .[test]
+      - name: Run pip check
+        run: pip check
+      # - name: Run pre-commit with mypy
+      #   run: |
+      #     pre-commit install
+      #     pre-commit run -a
+      - name: Check environment contents
+        run: |
+          conda list
+          pip list
+      - name: Test with pytest
+        run: pytest -n 2

umfive-0.2.0/.gitignore

@@ -0,0 +1,48 @@
+# macOS
+.DS_Store
+
+# Python bytecode/cache
+__pycache__/
+*.py[cod]
+*.pyo
+
+# Packaging/build artifacts
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# Virtual/env files
+.venv/
+venv/
+env/
+.python-version
+
+# Test/cache/tool outputs
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+
+# Notebook/checkpoint noise
+.ipynb_checkpoints/
+
+# IDE/editor
+.vscode/settings.json
+
+# Local logs/temp
+*.log
+*.tmp
+
+# Do not commit original source snapshot
+orig/
+
+# SO object
+umfive/_wgdos.cpython-314-x86_64-linux-gnu.so
+umfive/_wgdos.cpython-312-darwin.so
+umfive/_wgdos.cpython*
+
+# Local large test data (do not commit)
+tests/data/dk922*

umfive-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,79 @@
+# Configure pre-commit hooks to check for Python code validity, formatting and
+# style issues and prompt for these to be corrected before committing.
+
+# Excludes for *all* pre-commit hooks as listed below:
+# (these are documentation files, either old ones or aut-generated ones)
+exclude: docs\/1\..*\d\/|docs\/_downloads\/|docs\/.*\/tutorial\.py
+
+repos:
+
+  # Use specific format-enforcing pre-commit hooks from the core library
+  # with the default configuration (see pre-commit.com for documentation)
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-ast
+      - id: debug-statements
+      - id: end-of-file-fixer
+        types: [python]
+      - id: trailing-whitespace
+        types: [python]
+
+  # Also validate against 'black' for unambiguous Python formatting
+  # (see https://black.readthedocs.io/en/stable/ for documentation and see
+  # the cfdm pyproject.toml file for our custom black configuration)
+  - repo: https://github.com/ambv/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+        language_version: python3
+
+  # Also format docstrings in a consistent way, on top of 'black' requirements
+  # with the aim to auto-correct towards the Google docstring format (see
+  # https://google.github.io/styleguide/pyguide.html#381-docstrings) as much
+  # as possible (see https://github.com/myint/docformatter for docs, with
+  # configuration as below)
+#  - repo: https://github.com/myint/docformatter
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.6
+    hooks:
+      - id: docformatter
+        # These arguments act as the configuration for docformatter. They:
+        # * make docformatter auto-format in-place if not compliant;
+        # * ensure there is a blank line after the final non-empty line;
+        # * wraps both summary and description at 72 characters (the latter
+        #   by default).
+        args: [--in-place, --blank, --wrap-summaries=72]
+
+  # Check the docstrings conforms to Google docstring format as much as can
+  # be detected by the 'pydocstyle' checker
+  # (see http://www.pydocstyle.org/en/stable/index.html for
+  # docs and see the cfdm .pydocstyle.ini file for our custom configuration)
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+
+  # Additionally validate againt flake8 for other Python style enforcement
+  # (see https://flake8.pycqa.org/en/latest/ for documentation and see
+  # the cfdm .flake8 file for our custom flake8 configuration)
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.1.0
+    hooks:
+      - id: flake8
+
+  # Apply the 'isort' tool to sort Python import statements systematically
+  # (see https://pycqa.github.io/isort/ for documentation). It is fully
+  # compatible with 'black' with the lines set to ensure so in the repo's
+  # pyproject.toml. Other than that and the below, no extra config is required.
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+      - id: isort
+        name: isort (cython)
+        types: [cython]
+      - id: isort
+        name: isort (pyi)
+        types: [pyi]

umfive-0.2.0/.pydocstyle.ini

@@ -0,0 +1,10 @@
+[pydocstyle]
+inherit = false
+match = .*\.py
+convention = google
+
+# Don't require a docstring for every module and package (i.e. in any __init__)
+#
+# Ignore D415: First line should end with a period, question mark, or
+# exclamation point
+add-ignore=D100,D104,D105,D107,D415

umfive-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 NCAS CMS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

umfive-0.2.0/MANIFEST.in

@@ -0,0 +1 @@
+recursive-exclude doc *

umfive-0.2.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: umfive
+Version: 0.2.0
+Summary: PP/Fields reader with pyfive-style high-level API
+Author: David Hassell, Alan Iwi, Bryan Lawrence, Valeriu Predoi - package
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: cftime
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyfive>=1.1.2
+Provides-Extra: io
+Requires-Dist: fsspec>=2024.3.0; extra == "io"
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Requires-Dist: pytest-cov>=5; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+Dynamic: license-file
+
+# umfive
+
+A PP and Fields file reader that mimics relevant parts of the `pyfive` high-level API,
+with support for lazy metadata loading and parallel data loading when necessary.
+
+This is NOT an alternative to pyfive, it is a package which makes pp and fields files
+look like and taste ("quack like") chunked HDF5 files written by a NetCDF library.
+
+## API Contract (umfive vs pyfive)
+
+`umfive` is designed to be **pyfive-compatible** as far as likely users of pyfive
+would need such compatability. That is, it produces a chunk index (for kerchunk)
+if you really want that, and it supports all the information needed by CF/CFDM
+workflows to make pp-data as CF-compliant as possible.
+
+
+For the canonical pyfive interface, see the pyfive docs:
+
+- https://pyfive.readthedocs.io/
+
+### What maps directly to pyfive
+
+- `umfive.File` is registered as a `umfive.File` virtual type (when `umfive` is installed).
+- `umfive.Variable` is registered as a `umfive.Dataset` virtual type.
+- File/root-level members expected by pyfive-style callers are present:
+	- `attrs`, `groups`, `variables`, `dimensions`, `name`, `path`, `parent`
+- Mapping-style access works:
+	- `f["var"]`, iteration over variables, `items()`, `len(f)`
+- Dataset-like variable members needed by downstream consumers are present:
+	- `shape`, `dtype`, `ndim`, `size`, `attrs`, `chunks`/`chunk_shape`, `file`, `parent`
+	- slicing/indexing via `__getitem__` (for example `f["var"][:]`)
+	- `read_direct`, `astype(...)`, `iter_chunks(...)`
+
+Like `pyfive`, `umfive` is read-only:
+
+	- `File(..., mode="r")` is supported; write/update modes are not.
+
+### Important differences from pyfive
+
+
+- No nested groups in PP/Fields model:
+	- `f["group/var"]` is not supported.
+	- `groups` exists for compatibility but is empty for current PP/Fields inputs.
+- `get_lazy_view(...)` fallback behavior:
+	- `pyfive`-style API entry exists, but returns the normal variable view with an info log.
+- Some Dataset properties are intentionally placeholders (`None`) because they are
+	not meaningful for PP/Fields records:
+	- `compression`, `compression_opts`, `shuffle`, `fletcher32`, `maxshape`,
+		`fillvalue`, `dims`, `scaleoffset`.
+- Compatibility metadata is synthesized for CF/cfdm bridging:
+	- dimension-scale datasets and `DIMENSION_LIST` are created where needed.
+	- rotated-grid helper variables (for example `latitude`, `longitude`,
+		`rotated_latitude_longitude`) may be exposed when implied by UM headers.
+- umfive-specific extension API:
+	- `File.set_parallelism(thread_count=..., cat_range_allowed=...)`
+		is provided by `umfive` and is not part of `pyfive`.
+
+
+### Practical expectation
+
+If your code treats `umfive.File` as  a `pyfive`-like read object,
+common analysis workflows should work, including chunk-level access (PP records
+are treated as chunks).  
+
+## Authorship
+
+Key authors of this package are:
+
+- David Hassell (@davidhassell)
+- Alan Iwi (@alaniwi)
+- Bryan Lawrence (@bnlawrence)

umfive-0.2.0/README.md

@@ -0,0 +1,71 @@
+# umfive
+
+A PP and Fields file reader that mimics relevant parts of the `pyfive` high-level API,
+with support for lazy metadata loading and parallel data loading when necessary.
+
+This is NOT an alternative to pyfive, it is a package which makes pp and fields files
+look like and taste ("quack like") chunked HDF5 files written by a NetCDF library.
+
+## API Contract (umfive vs pyfive)
+
+`umfive` is designed to be **pyfive-compatible** as far as likely users of pyfive
+would need such compatability. That is, it produces a chunk index (for kerchunk)
+if you really want that, and it supports all the information needed by CF/CFDM
+workflows to make pp-data as CF-compliant as possible.
+
+
+For the canonical pyfive interface, see the pyfive docs:
+
+- https://pyfive.readthedocs.io/
+
+### What maps directly to pyfive
+
+- `umfive.File` is registered as a `umfive.File` virtual type (when `umfive` is installed).
+- `umfive.Variable` is registered as a `umfive.Dataset` virtual type.
+- File/root-level members expected by pyfive-style callers are present:
+	- `attrs`, `groups`, `variables`, `dimensions`, `name`, `path`, `parent`
+- Mapping-style access works:
+	- `f["var"]`, iteration over variables, `items()`, `len(f)`
+- Dataset-like variable members needed by downstream consumers are present:
+	- `shape`, `dtype`, `ndim`, `size`, `attrs`, `chunks`/`chunk_shape`, `file`, `parent`
+	- slicing/indexing via `__getitem__` (for example `f["var"][:]`)
+	- `read_direct`, `astype(...)`, `iter_chunks(...)`
+
+Like `pyfive`, `umfive` is read-only:
+
+	- `File(..., mode="r")` is supported; write/update modes are not.
+
+### Important differences from pyfive
+
+
+- No nested groups in PP/Fields model:
+	- `f["group/var"]` is not supported.
+	- `groups` exists for compatibility but is empty for current PP/Fields inputs.
+- `get_lazy_view(...)` fallback behavior:
+	- `pyfive`-style API entry exists, but returns the normal variable view with an info log.
+- Some Dataset properties are intentionally placeholders (`None`) because they are
+	not meaningful for PP/Fields records:
+	- `compression`, `compression_opts`, `shuffle`, `fletcher32`, `maxshape`,
+		`fillvalue`, `dims`, `scaleoffset`.
+- Compatibility metadata is synthesized for CF/cfdm bridging:
+	- dimension-scale datasets and `DIMENSION_LIST` are created where needed.
+	- rotated-grid helper variables (for example `latitude`, `longitude`,
+		`rotated_latitude_longitude`) may be exposed when implied by UM headers.
+- umfive-specific extension API:
+	- `File.set_parallelism(thread_count=..., cat_range_allowed=...)`
+		is provided by `umfive` and is not part of `pyfive`.
+
+
+### Practical expectation
+
+If your code treats `umfive.File` as  a `pyfive`-like read object,
+common analysis workflows should work, including chunk-level access (PP records
+are treated as chunks).  
+
+## Authorship
+
+Key authors of this package are:
+
+- David Hassell (@davidhassell)
+- Alan Iwi (@alaniwi)
+- Bryan Lawrence (@bnlawrence)

umfive-0.2.0/environment.yml

@@ -0,0 +1,17 @@
+name: umfive-dev
+channels:
+  - conda-forge
+  - nodefaults  # stay away from Anaconda like the plague
+
+dependencies:
+  - python >=3.11
+  - numpy >=1.24
+  - fsspec >=2024.3.0
+  - pyfive >=1.1.2
+  - pytest >=8
+  - pytest-xdist
+  - udunits2  # cf-python needs it
+  - pip
+  - pip:
+      - cf-python @ git+https://github.com/davidhassell/cf-python.git@kerchunk-read
+      - cfdm @ git+https://github.com/davidhassell/cfdm.git@pyfive-netcdf

umfive-0.2.0/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = [
+  "setuptools>=80",
+  "setuptools_scm>=8",
+  "numpy>=1.24",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "umfive"
+version = "0.2.0"
+description = "PP/Fields reader with pyfive-style high-level API"
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+	{name = "David Hassell"},
+	{name = "Alan Iwi"},
+	{name = "Bryan Lawrence"},
+        {name = "Valeriu Predoi - package"},
+]
+dependencies = [
+	"cftime",
+	"numpy>=1.24",
+	"pyfive>=1.1.2",
+]
+
+[project.optional-dependencies]
+io = ["fsspec>=2024.3.0"]
+test = [
+	"pytest>=8",
+	"pytest-cov>=5",
+        "pytest-xdist",
+]
+
+[project.scripts]
+ppdump = "umfive.ppdump:main"
+
+[tool.setuptools]
+include-package-data = true
+license-files = ["LICENSE"]
+
+[tool.setuptools.packages.find]
+include = ["umfive*"]
+
+[tool.setuptools.package-data]
+umfive = ["data/*.txt"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.isort]
+profile = "black"
+# ... and since we set this against the black default line length:
+line_length=79
+# Prevent isort from auto-formatting '__init__.py' file imports because
+# they require a specific non-aphabetical (etc.) ordering else they will
+# cause errors due to bad or circular importing across the modules.
+extend_skip_glob = [
+  "**/__init__.py",
+]
+
+[tool.black]
+line-length = 79

umfive-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

umfive-0.2.0/setup.py

@@ -0,0 +1,13 @@
+import numpy as np
+from setuptools import Extension, setup
+
+print(np.get_include())
+setup(
+    ext_modules=[
+        Extension(
+            "umfive._wgdos",
+            sources=["umfive/c_ext/wgdos_module.c"],
+            include_dirs=[np.get_include(), "umfive/c_ext"],
+        )
+    ]
+)