h5netcdf 1.6.0__tar.gz → 1.6.2__tar.gz

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/CHANGELOG.rst

@@ -1,6 +1,26 @@
 Change Log
 ----------
 
+Version 1.6.2 (June 26th, 2025):
+
+- Codespell fixes ({pull}`261`).
+  By `Kurt Schwehr <https://github.com/schwehr>`_
+- Fix hsds/h5pyd test fixture spinup issues ({pull}`265`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+- Fix and add circular referrer tests for Python 3.14 and update CI matrix ({pull}`264`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+- Avoid opening h5pyd file to check if there is a preexisting file,
+  instead remap mode "a" -> "r+", resort to "w" if file doesn't exist ({issue}`262`, {pull}`266`).
+  By `Jonas Grönberg <https://github.com/JonasGronberg>`_ and `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+- Reduce CI time by installing available scientific-python-nightly-wheels and using pip cache ({pull}`267`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+
+Version 1.6.1 (March 7th, 2025):
+
+- Let Variable.chunks return None for scalar variables, independent of what the underlying
+  h5ds object returns ({pull}`259`).
+  By `Rickard Holmberg <https://github.com/rho-novatron>`_
+
 Version 1.6.0 (March 7th, 2025):
 
 - Allow specifying `h5netcdf.File(driver="h5pyd")` to force the use of h5pyd ({issue}`255`, {pull}`256`).

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: h5netcdf
-Version: 1.6.0
+Version: 1.6.2
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>
@@ -56,6 +56,7 @@ Requires-Dist: packaging
 Provides-Extra: test
 Requires-Dist: netCDF4; extra == "test"
 Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
 
 h5netcdf
 ========
@@ -318,7 +319,7 @@ The following describes the behavior of h5netcdf with respect to order tracking
 for a few key versions:
 
 - Version 0.12.0 and earlier, the ``track_order`` parameter`order was missing
-  and thus order tracking was implicitely set to ``False``.
+  and thus order tracking was implicitly set to ``False``.
 - Version 0.13.0 enabled order tracking by setting the parameter
   ``track_order`` to ``True`` by default without deprecation.
 - Versions 0.13.1 to 1.0.2 set ``track_order`` to ``False`` due to a bug in a

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/README.rst

@@ -259,7 +259,7 @@ The following describes the behavior of h5netcdf with respect to order tracking
 for a few key versions:
 
 - Version 0.12.0 and earlier, the ``track_order`` parameter`order was missing
-  and thus order tracking was implicitely set to ``False``.
+  and thus order tracking was implicitly set to ``False``.
 - Version 0.13.0 enabled order tracking by setting the parameter
   ``track_order`` to ``True`` by default without deprecation.
 - Versions 0.13.1 to 1.0.2 set ``track_order`` to ``False`` due to a bug in a

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/doc/conf.py

@@ -117,7 +117,7 @@ napoleon_type_aliases = {
     "Path": "~~pathlib.Path",
 }
 
-# handle release substition
+# handle release substitution
 url = "https://github.com/h5netcdf"
 
 # get version

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/doc/devguide.rst

@@ -18,10 +18,14 @@ Contributors
 - `Ezequiel Cimadevilla Alvarez <https://github.com/zequihg50>`_
 - `Frédéric Laliberté <https://github.com/laliberte>`_
 - `Ghislain Vaillant <https://github.com/ghisvail>`_
+- `John Readey <https://github.com/jreadey>`_
+- `Jonas Grönberg <https://github.com/JonasGronberg>`_
+- `Kurt Schwehr <https://github.com/schwehr>`_
 - `Lion Krischer <https://github.com/krischer>`_
 - `Mark Harfouche <https://github.com/hmaarrfk>`_
 - `Martin Raspaud <https://github.com/mraspaud>`_
 - `Pierre Augier <https://github.com/paugier>`_
+- `Rickard Holmberg <https://github.com/rho-novatron>`_
 - `Ryan Grout <https://github.com/groutr>`_
 - `Scott Henderson <https://github.com/scottyhq>`_
 - `Thomas Kluyver <https://github.com/takluyver>`_

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/doc/index.rst

@@ -50,7 +50,7 @@ by Stephan Hoyer. The first `official` ``h5netcdf`` announcement was made by Ste
 `xarray issue tracker`_ only one day later.
 
 The library evolved constantly over the years (fixing bugs and adding enhancements)
-and gained contributions from 17 other :ref:`contributors` so far. The library is widely used,
+and gained contributions from 21 other :ref:`contributors` so far. The library is widely used,
 especially as backend within `xarray`_.
 
 Early 2020 Kai Mühlbauer started to add contributions and after some time he volunteered

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/h5netcdf/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.0'
-__version_tuple__ = version_tuple = (1, 6, 0)
+__version__ = version = '1.6.2'
+__version_tuple__ = version_tuple = (1, 6, 2)

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/h5netcdf/core.py

@@ -608,6 +608,11 @@ class BaseVariable(BaseObject):
 class Variable(BaseVariable):
     @property
     def chunks(self):
+        if self.shape == ():
+            # In HSDS, the layout can be chunked even for scalar datasets, but with only a single chunk.
+            # Return None for scalar datasets since they shall be handled as non-chunked.
+            assert self._h5ds.chunks in (None, (), (1,))
+            return None
         return self._h5ds.chunks
 
     @property
@@ -1514,16 +1519,35 @@ class File(Group):
                             "No module named 'h5pyd'. h5pyd is required for "
                             f"opening urls: {path}"
                         )
+                    self._preexisting_file = mode in {"r", "r+", "a"}
+                    # remap "a" -> "r+" to check file existence
+                    # fallback to "w" if not
+                    _mode = mode
+                    if mode == "a":
+                        mode = "r+"
+                    self._h5py = h5pyd
                     try:
-                        with h5pyd.File(path, "r", **kwargs) as f:  # noqa
-                            pass
-                        self._preexisting_file = True
+                        self._h5file = self._h5py.File(
+                            path, mode, track_order=track_order, **kwargs
+                        )
+                        self._preexisting_file = mode != "w"
                     except OSError:
-                        self._preexisting_file = False
-                    self._h5py = h5pyd
-                    self._h5file = self._h5py.File(
-                        path, mode, track_order=track_order, **kwargs
-                    )
+                        # if file does not exist, create it
+                        if _mode == "a":
+                            mode = "w"
+                            self._h5file = self._h5py.File(
+                                path, mode, track_order=track_order, **kwargs
+                            )
+                            self._preexisting_file = False
+                            msg = (
+                                "Append mode for h5pyd now probes with 'r+' first and "
+                                "only falls back to 'w' if the file is missing.\n"
+                                "To silence this warning use 'r+' (open-existing) or 'w' "
+                                "(create-new) directly."
+                            )
+                            warnings.warn(msg, UserWarning, stacklevel=2)
+                        else:
+                            raise
                 else:
                     self._preexisting_file = os.path.exists(path) and mode != "w"
                     self._h5py = h5py

h5netcdf-1.6.2/h5netcdf/tests/conftest.py

@@ -0,0 +1,83 @@
+import os
+import tempfile
+import time
+from pathlib import Path
+from shutil import rmtree
+
+import pytest
+
+try:
+    from h5pyd import Folder
+    from hsds.hsds_app import HsdsApp
+
+    with_reqd_pkgs = True
+except ImportError:
+    with_reqd_pkgs = False
+
+
+@pytest.fixture(scope="session")
+def hsds_up():
+    """Provide HDF Highly Scalable Data Service (HSDS) for h5pyd testing."""
+    if not with_reqd_pkgs:
+        pytest.skip("Required packages h5pyd and hsds not available")
+
+    root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
+    bucket_name = "pytest"
+    os.environ["BUCKET_NAME"] = bucket_name
+    # need to create a directory for our bucket
+    (root_dir / bucket_name).mkdir()
+
+    kwargs = {
+        "username": "h5netcdf-pytest",
+        "password": "TestEarlyTestEverything",
+        "root_dir": str(root_dir),
+        "logfile": str(root_dir / "hsds.log"),
+        "log_level": "DEBUG",
+        "host": "localhost",
+        "sn_port": 5101,
+    }
+
+    os.environ.update(
+        {
+            "BUCKET_NAME": bucket_name,
+            "HS_USERNAME": kwargs["username"],
+            "HS_PASSWORD": kwargs["password"],
+            "HS_USE_HTTPS": "False",
+        }
+    )
+
+    hsds = HsdsApp(**kwargs)
+
+    try:
+        hsds.run()
+        timeout = time.time() + 60
+        while not hsds.ready:
+            if time.time() > timeout:
+                raise TimeoutError("HSDS server did not become ready in time")
+            time.sleep(1)
+
+        os.environ["HS_ENDPOINT"] = hsds.endpoint
+        # make folders expected by pytest
+        Folder("/home/", mode="w")
+        Folder("/home/h5netcdf-pytest/", mode="w")
+
+        yield True
+
+    except Exception as err:
+        log_path = kwargs["logfile"]
+        if os.path.exists(log_path):
+            with open(log_path) as f:
+                print("\n=== HSDS Log ===")
+                print(f.read())
+        else:
+            print(f"HSDS log not found at: {log_path}")
+        raise err
+
+    finally:
+        try:
+            hsds.check_processes()
+            hsds.stop()
+        except Exception:
+            pass
+
+    rmtree(root_dir, ignore_errors=True)

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/h5netcdf/tests/test_h5netcdf.py

@@ -3,7 +3,9 @@ import io
 import random
 import re
 import string
+import sys
 import tempfile
+import weakref
 from os import environ as env
 
 import h5py
@@ -30,6 +32,7 @@ except ImportError:
 
 
 remote_h5 = ("http:", "hdf5:")
+python_version = version.parse(".".join(map(str, sys.version_info[:3])))
 
 
 @pytest.fixture
@@ -37,26 +40,30 @@ def tmp_local_netcdf(tmpdir):
     return str(tmpdir.join("testfile.nc"))
 
 
+@pytest.fixture()
+def setup_h5pyd_config(hsds_up):
+    env["HS_ENDPOINT"] = "http://127.0.0.1:5101"
+    env["HS_USERNAME"] = "h5netcdf-pytest"
+    env["HS_PASSWORD"] = "TestEarlyTestEverything"
+    env["HS_USE_HTTPS"] = "False"
+
+
 @pytest.fixture(params=["testfile.nc", "hdf5://testfile"])
-def tmp_local_or_remote_netcdf(request, tmpdir, hsds_up):
-    if request.param.startswith(remote_h5):
-        if without_h5pyd:
-            pytest.skip("h5pyd package not available")
-        elif not hsds_up:
-            pytest.skip("HSDS service not running")
-        rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
-        return (
-            "hdf5://"
-            + "home"
-            + "/"
-            + env["HS_USERNAME"]
-            + "/"
-            + "testfile"
-            + rnd
-            + ".nc"
-        )
+def tmp_local_or_remote_netcdf(request, tmpdir):
+    param = request.param
+    if param.startswith(remote_h5):
+        try:
+            hsds_up = request.getfixturevalue("hsds_up")
+        except pytest.skip.Exception:
+            pytest.skip("HSDS not available")
+
+        if not hsds_up:
+            pytest.skip("HSDS fixture returned False (not running)")
+
+        rnd = "".join(random.choices(string.ascii_uppercase, k=5))
+        return f"hdf5://home/{env['HS_USERNAME']}/testfile{rnd}.nc"
     else:
-        return str(tmpdir.join(request.param))
+        return str(tmpdir.join(param))
 
 
 @pytest.fixture(params=[True, False])
@@ -1300,6 +1307,24 @@ def test_overwrite_existing_file(tmp_local_netcdf):
         assert ds.attrs._h5attrs.get("_NCProperties", False)
 
 
+def test_overwrite_existing_remote_file(tmp_local_or_remote_netcdf):
+    # create file with legacyapi
+    with legacyapi.Dataset(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.createDimension("x", 10)
+
+    # check attribute
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        assert ds.attrs._h5attrs.get("_NCProperties", False)
+
+    # overwrite file with new api
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.dimensions["x"] = 10
+
+    # check attribute
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        assert ds.attrs._h5attrs.get("_NCProperties", False)
+
+
 def test_scales_on_append(tmp_local_netcdf):
     # create file with _NCProperties attribute
     with netCDF4.Dataset(tmp_local_netcdf, "w") as ds:
@@ -1555,7 +1580,38 @@ def test_no_circular_references(tmp_local_or_remote_netcdf):
         refs = gc.get_referrers(ds)
         for ref in refs:
             print(ref)
-        assert len(refs) == 1
+        if python_version >= version.parse("3.14"):
+            assert len(refs) == 0
+        else:
+            assert len(refs) == 1
+
+
+def test_no_circular_references_py314(tmp_local_or_remote_netcdf):
+    # https://github.com/h5py/h5py/issues/2019
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.dimensions["x"] = 2
+        ds.dimensions["y"] = 2
+
+    # clean up everything
+    gc.collect()
+    gc.garbage.clear()
+
+    # use weakref to hold on object
+    file_ref = None
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        file_ref = weakref.ref(ds)
+
+    # clean up
+    gc.collect()
+
+    # check garbage list
+    if file_ref() is not None:
+        print("Uncollectable object:", file_ref())
+        print("Potential GC garbage:")
+        for obj in gc.garbage:
+            print(repr(obj))
+
+    assert file_ref() is None or "<Closed h5netcdf.File>"
 
 
 def test_expanded_variables_netcdf4(tmp_local_netcdf, netcdf_write_module):
@@ -1693,7 +1749,7 @@ def test_track_order_specification(tmp_local_netcdf):
     # While netcdf4-c has historically only allowed track_order to be True
     # There doesn't seem to be a good reason for this
     # https://github.com/Unidata/netcdf-c/issues/2054 historically, h5netcdf
-    # has not specified this parameter (leaving it implicitely as False)
+    # has not specified this parameter (leaving it implicitly as False)
     # We want to make sure we allow both here
     with h5netcdf.File(tmp_local_netcdf, "w", track_order=False):
         pass
@@ -2757,3 +2813,35 @@ def test_h5pyd_driver(hsds_up):
         with h5netcdf.File(fname, "w", driver="h5pyd") as ds:
             assert ds._h5py == h5pyd
             assert isinstance(ds._h5file, h5pyd.File)
+
+
+def test_h5pyd_nonchunked_scalars(hsds_up):
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    fname = f"hdf5://testfile{rnd}.nc"
+    with h5pyd.File(fname, "w") as ds:
+        ds.create_dataset("foo", data=b"1234")
+    with h5netcdf.File(fname, "r", driver="h5pyd") as ds:
+        # HSDS stores this as a chunked dataset, but only with a single chunk
+        assert ds["foo"]._h5ds.chunks == (1,)
+        # However, since it is a scalar dataset, we should not expose the chunking
+        assert ds["foo"].chunks is None
+
+
+def test_h5pyd_append(hsds_up):
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    fname = f"hdf5://testfile{rnd}.nc"
+
+    with pytest.warns(UserWarning, match="Append mode for h5pyd"):
+        with h5netcdf.File(fname, "a", driver="h5pyd") as ds:
+            assert not ds._preexisting_file
+
+    with h5netcdf.File(fname, "a", driver="h5pyd") as ds:
+        assert ds._preexisting_file

{h5netcdf-1.6.0 → h5netcdf-1.6.2}/h5netcdf.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: h5netcdf
-Version: 1.6.0
+Version: 1.6.2
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>
@@ -56,6 +56,7 @@ Requires-Dist: packaging
 Provides-Extra: test
 Requires-Dist: netCDF4; extra == "test"
 Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
 
 h5netcdf
 ========
@@ -318,7 +319,7 @@ The following describes the behavior of h5netcdf with respect to order tracking
 for a few key versions:
 
 - Version 0.12.0 and earlier, the ``track_order`` parameter`order was missing
-  and thus order tracking was implicitely set to ``False``.
+  and thus order tracking was implicitly set to ``False``.
 - Version 0.13.0 enabled order tracking by setting the parameter
   ``track_order`` to ``True`` by default without deprecation.
 - Versions 0.13.1 to 1.0.2 set ``track_order`` to ``False`` due to a bug in a

h5netcdf-1.6.0/h5netcdf/tests/conftest.py

@@ -1,65 +0,0 @@
-import os
-import tempfile
-from pathlib import Path
-from shutil import rmtree
-
-import pytest
-
-try:
-    from h5pyd import Folder
-    from hsds.hsds_app import HsdsApp
-
-    with_reqd_pkgs = True
-except ImportError:
-    with_reqd_pkgs = False
-
-
-@pytest.fixture(scope="session")
-def hsds_up():
-    """Provide HDF Highly Scalabale Data Service (HSDS) for h5pyd testing."""
-    if with_reqd_pkgs:
-        root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
-        bucket_name = "pytest"
-        os.environ["BUCKET_NAME"] = bucket_name
-        os.mkdir(
-            f"{root_dir}/{bucket_name}"
-        )  # need to create a directory for our bucket
-
-        hs_username = "h5netcdf-pytest"
-        hs_password = "TestEarlyTestEverything"
-
-        kwargs = {}
-        kwargs["username"] = hs_username
-        kwargs["password"] = hs_password
-        kwargs["root_dir"] = str(root_dir)
-        kwargs["logfile"] = f"{root_dir}/hsds.log"
-        kwargs["log_level"] = "DEBUG"
-        kwargs["host"] = "localhost"
-        kwargs["sn_port"] = 5101
-
-        try:
-            hsds = HsdsApp(**kwargs)
-
-            hsds.run()
-            is_up = hsds.ready
-
-            if is_up:
-                os.environ["HS_ENDPOINT"] = hsds.endpoint
-                os.environ["HS_USERNAME"] = hs_username
-                os.environ["HS_PASSWORD"] = hs_password
-                # make folders expected by pytest
-                # pytest/home/h5netcdf-pytest
-                # Folder("/pytest/", mode='w')
-                Folder("/home/", mode="w")
-                Folder("/home/h5netcdf-pytest/", mode="w")
-        except Exception:
-            is_up = False
-
-        yield is_up
-        hsds.check_processes()  # this will capture hsds log output
-        hsds.stop()
-
-        rmtree(root_dir, ignore_errors=True)
-
-    else:
-        yield False