h5netcdf 1.6.0__py3-none-any.whl → 1.6.2__py3-none-any.whl

h5netcdf/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.6.0'
-__version_tuple__ = version_tuple = (1, 6, 0)
+__version__ = version = '1.6.2'
+__version_tuple__ = version_tuple = (1, 6, 2)

h5netcdf/core.py

@@ -608,6 +608,11 @@ class BaseVariable(BaseObject):
 class Variable(BaseVariable):
     @property
     def chunks(self):
+        if self.shape == ():
+            # In HSDS, the layout can be chunked even for scalar datasets, but with only a single chunk.
+            # Return None for scalar datasets since they shall be handled as non-chunked.
+            assert self._h5ds.chunks in (None, (), (1,))
+            return None
         return self._h5ds.chunks
 
     @property
@@ -1514,16 +1519,35 @@ class File(Group):
                             "No module named 'h5pyd'. h5pyd is required for "
                             f"opening urls: {path}"
                         )
+                    self._preexisting_file = mode in {"r", "r+", "a"}
+                    # remap "a" -> "r+" to check file existence
+                    # fallback to "w" if not
+                    _mode = mode
+                    if mode == "a":
+                        mode = "r+"
+                    self._h5py = h5pyd
                     try:
-                        with h5pyd.File(path, "r", **kwargs) as f:  # noqa
-                            pass
-                        self._preexisting_file = True
+                        self._h5file = self._h5py.File(
+                            path, mode, track_order=track_order, **kwargs
+                        )
+                        self._preexisting_file = mode != "w"
                     except OSError:
-                        self._preexisting_file = False
-                    self._h5py = h5pyd
-                    self._h5file = self._h5py.File(
-                        path, mode, track_order=track_order, **kwargs
-                    )
+                        # if file does not exist, create it
+                        if _mode == "a":
+                            mode = "w"
+                            self._h5file = self._h5py.File(
+                                path, mode, track_order=track_order, **kwargs
+                            )
+                            self._preexisting_file = False
+                            msg = (
+                                "Append mode for h5pyd now probes with 'r+' first and "
+                                "only falls back to 'w' if the file is missing.\n"
+                                "To silence this warning use 'r+' (open-existing) or 'w' "
+                                "(create-new) directly."
+                            )
+                            warnings.warn(msg, UserWarning, stacklevel=2)
+                        else:
+                            raise
                 else:
                     self._preexisting_file = os.path.exists(path) and mode != "w"
                     self._h5py = h5py

h5netcdf/tests/conftest.py

@@ -1,5 +1,6 @@
 import os
 import tempfile
+import time
 from pathlib import Path
 from shutil import rmtree
 
@@ -16,50 +17,67 @@ except ImportError:
 
 @pytest.fixture(scope="session")
 def hsds_up():
-    """Provide HDF Highly Scalabale Data Service (HSDS) for h5pyd testing."""
-    if with_reqd_pkgs:
-        root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
-        bucket_name = "pytest"
-        os.environ["BUCKET_NAME"] = bucket_name
-        os.mkdir(
-            f"{root_dir}/{bucket_name}"
-        )  # need to create a directory for our bucket
+    """Provide HDF Highly Scalable Data Service (HSDS) for h5pyd testing."""
+    if not with_reqd_pkgs:
+        pytest.skip("Required packages h5pyd and hsds not available")
 
-        hs_username = "h5netcdf-pytest"
-        hs_password = "TestEarlyTestEverything"
+    root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
+    bucket_name = "pytest"
+    os.environ["BUCKET_NAME"] = bucket_name
+    # need to create a directory for our bucket
+    (root_dir / bucket_name).mkdir()
 
-        kwargs = {}
-        kwargs["username"] = hs_username
-        kwargs["password"] = hs_password
-        kwargs["root_dir"] = str(root_dir)
-        kwargs["logfile"] = f"{root_dir}/hsds.log"
-        kwargs["log_level"] = "DEBUG"
-        kwargs["host"] = "localhost"
-        kwargs["sn_port"] = 5101
+    kwargs = {
+        "username": "h5netcdf-pytest",
+        "password": "TestEarlyTestEverything",
+        "root_dir": str(root_dir),
+        "logfile": str(root_dir / "hsds.log"),
+        "log_level": "DEBUG",
+        "host": "localhost",
+        "sn_port": 5101,
+    }
 
-        try:
-            hsds = HsdsApp(**kwargs)
+    os.environ.update(
+        {
+            "BUCKET_NAME": bucket_name,
+            "HS_USERNAME": kwargs["username"],
+            "HS_PASSWORD": kwargs["password"],
+            "HS_USE_HTTPS": "False",
+        }
+    )
 
-            hsds.run()
-            is_up = hsds.ready
+    hsds = HsdsApp(**kwargs)
 
-            if is_up:
-                os.environ["HS_ENDPOINT"] = hsds.endpoint
-                os.environ["HS_USERNAME"] = hs_username
-                os.environ["HS_PASSWORD"] = hs_password
-                # make folders expected by pytest
-                # pytest/home/h5netcdf-pytest
-                # Folder("/pytest/", mode='w')
-                Folder("/home/", mode="w")
-                Folder("/home/h5netcdf-pytest/", mode="w")
-        except Exception:
-            is_up = False
+    try:
+        hsds.run()
+        timeout = time.time() + 60
+        while not hsds.ready:
+            if time.time() > timeout:
+                raise TimeoutError("HSDS server did not become ready in time")
+            time.sleep(1)
+
+        os.environ["HS_ENDPOINT"] = hsds.endpoint
+        # make folders expected by pytest
+        Folder("/home/", mode="w")
+        Folder("/home/h5netcdf-pytest/", mode="w")
 
-        yield is_up
-        hsds.check_processes()  # this will capture hsds log output
-        hsds.stop()
+        yield True
 
-        rmtree(root_dir, ignore_errors=True)
+    except Exception as err:
+        log_path = kwargs["logfile"]
+        if os.path.exists(log_path):
+            with open(log_path) as f:
+                print("\n=== HSDS Log ===")
+                print(f.read())
+        else:
+            print(f"HSDS log not found at: {log_path}")
+        raise err
+
+    finally:
+        try:
+            hsds.check_processes()
+            hsds.stop()
+        except Exception:
+            pass
 
-    else:
-        yield False
+    rmtree(root_dir, ignore_errors=True)

h5netcdf/tests/test_h5netcdf.py

@@ -3,7 +3,9 @@ import io
 import random
 import re
 import string
+import sys
 import tempfile
+import weakref
 from os import environ as env
 
 import h5py
@@ -30,6 +32,7 @@ except ImportError:
 
 
 remote_h5 = ("http:", "hdf5:")
+python_version = version.parse(".".join(map(str, sys.version_info[:3])))
 
 
 @pytest.fixture
@@ -37,26 +40,30 @@ def tmp_local_netcdf(tmpdir):
     return str(tmpdir.join("testfile.nc"))
 
 
+@pytest.fixture()
+def setup_h5pyd_config(hsds_up):
+    env["HS_ENDPOINT"] = "http://127.0.0.1:5101"
+    env["HS_USERNAME"] = "h5netcdf-pytest"
+    env["HS_PASSWORD"] = "TestEarlyTestEverything"
+    env["HS_USE_HTTPS"] = "False"
+
+
 @pytest.fixture(params=["testfile.nc", "hdf5://testfile"])
-def tmp_local_or_remote_netcdf(request, tmpdir, hsds_up):
-    if request.param.startswith(remote_h5):
-        if without_h5pyd:
-            pytest.skip("h5pyd package not available")
-        elif not hsds_up:
-            pytest.skip("HSDS service not running")
-        rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
-        return (
-            "hdf5://"
-            + "home"
-            + "/"
-            + env["HS_USERNAME"]
-            + "/"
-            + "testfile"
-            + rnd
-            + ".nc"
-        )
+def tmp_local_or_remote_netcdf(request, tmpdir):
+    param = request.param
+    if param.startswith(remote_h5):
+        try:
+            hsds_up = request.getfixturevalue("hsds_up")
+        except pytest.skip.Exception:
+            pytest.skip("HSDS not available")
+
+        if not hsds_up:
+            pytest.skip("HSDS fixture returned False (not running)")
+
+        rnd = "".join(random.choices(string.ascii_uppercase, k=5))
+        return f"hdf5://home/{env['HS_USERNAME']}/testfile{rnd}.nc"
     else:
-        return str(tmpdir.join(request.param))
+        return str(tmpdir.join(param))
 
 
 @pytest.fixture(params=[True, False])
@@ -1300,6 +1307,24 @@ def test_overwrite_existing_file(tmp_local_netcdf):
         assert ds.attrs._h5attrs.get("_NCProperties", False)
 
 
+def test_overwrite_existing_remote_file(tmp_local_or_remote_netcdf):
+    # create file with legacyapi
+    with legacyapi.Dataset(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.createDimension("x", 10)
+
+    # check attribute
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        assert ds.attrs._h5attrs.get("_NCProperties", False)
+
+    # overwrite file with new api
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.dimensions["x"] = 10
+
+    # check attribute
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        assert ds.attrs._h5attrs.get("_NCProperties", False)
+
+
 def test_scales_on_append(tmp_local_netcdf):
     # create file with _NCProperties attribute
     with netCDF4.Dataset(tmp_local_netcdf, "w") as ds:
@@ -1555,7 +1580,38 @@ def test_no_circular_references(tmp_local_or_remote_netcdf):
         refs = gc.get_referrers(ds)
         for ref in refs:
             print(ref)
-        assert len(refs) == 1
+        if python_version >= version.parse("3.14"):
+            assert len(refs) == 0
+        else:
+            assert len(refs) == 1
+
+
+def test_no_circular_references_py314(tmp_local_or_remote_netcdf):
+    # https://github.com/h5py/h5py/issues/2019
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "w") as ds:
+        ds.dimensions["x"] = 2
+        ds.dimensions["y"] = 2
+
+    # clean up everything
+    gc.collect()
+    gc.garbage.clear()
+
+    # use weakref to hold on object
+    file_ref = None
+    with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
+        file_ref = weakref.ref(ds)
+
+    # clean up
+    gc.collect()
+
+    # check garbage list
+    if file_ref() is not None:
+        print("Uncollectable object:", file_ref())
+        print("Potential GC garbage:")
+        for obj in gc.garbage:
+            print(repr(obj))
+
+    assert file_ref() is None or "<Closed h5netcdf.File>"
 
 
 def test_expanded_variables_netcdf4(tmp_local_netcdf, netcdf_write_module):
@@ -1693,7 +1749,7 @@ def test_track_order_specification(tmp_local_netcdf):
     # While netcdf4-c has historically only allowed track_order to be True
     # There doesn't seem to be a good reason for this
     # https://github.com/Unidata/netcdf-c/issues/2054 historically, h5netcdf
-    # has not specified this parameter (leaving it implicitely as False)
+    # has not specified this parameter (leaving it implicitly as False)
     # We want to make sure we allow both here
     with h5netcdf.File(tmp_local_netcdf, "w", track_order=False):
         pass
@@ -2757,3 +2813,35 @@ def test_h5pyd_driver(hsds_up):
         with h5netcdf.File(fname, "w", driver="h5pyd") as ds:
             assert ds._h5py == h5pyd
             assert isinstance(ds._h5file, h5pyd.File)
+
+
+def test_h5pyd_nonchunked_scalars(hsds_up):
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    fname = f"hdf5://testfile{rnd}.nc"
+    with h5pyd.File(fname, "w") as ds:
+        ds.create_dataset("foo", data=b"1234")
+    with h5netcdf.File(fname, "r", driver="h5pyd") as ds:
+        # HSDS stores this as a chunked dataset, but only with a single chunk
+        assert ds["foo"]._h5ds.chunks == (1,)
+        # However, since it is a scalar dataset, we should not expose the chunking
+        assert ds["foo"].chunks is None
+
+
+def test_h5pyd_append(hsds_up):
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    fname = f"hdf5://testfile{rnd}.nc"
+
+    with pytest.warns(UserWarning, match="Append mode for h5pyd"):
+        with h5netcdf.File(fname, "a", driver="h5pyd") as ds:
+            assert not ds._preexisting_file
+
+    with h5netcdf.File(fname, "a", driver="h5pyd") as ds:
+        assert ds._preexisting_file

{h5netcdf-1.6.0.dist-info → h5netcdf-1.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: h5netcdf
-Version: 1.6.0
+Version: 1.6.2
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>
@@ -56,6 +56,7 @@ Requires-Dist: packaging
 Provides-Extra: test
 Requires-Dist: netCDF4; extra == "test"
 Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
 
 h5netcdf
 ========
@@ -318,7 +319,7 @@ The following describes the behavior of h5netcdf with respect to order tracking
 for a few key versions:
 
 - Version 0.12.0 and earlier, the ``track_order`` parameter`order was missing
-  and thus order tracking was implicitely set to ``False``.
+  and thus order tracking was implicitly set to ``False``.
 - Version 0.13.0 enabled order tracking by setting the parameter
   ``track_order`` to ``True`` by default without deprecation.
 - Versions 0.13.1 to 1.0.2 set ``track_order`` to ``False`` due to a bug in a

h5netcdf-1.6.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+h5netcdf/__init__.py,sha256=Y0EBCcmlJctwl1kCmj7yLijTVy9AioBTr2091vInAtw,456
+h5netcdf/_version.py,sha256=0l_x32-zDcY9aQqQSs0LtsRx42EjkEfyXdUn9HtFgXU,511
+h5netcdf/attrs.py,sha256=4IvV4ULLWkz4igFsvu9S2LB745wgUKrIdIuSeO5kpX8,3581
+h5netcdf/core.py,sha256=AnPeV5XE_04QZWPSM7WjrddaKfkkdPtvbIMEKf3cSIs,64154
+h5netcdf/dimensions.py,sha256=2g0p9DOAC0hhQ94spIAjWeKC1qyhzzO0s15xCFYSscM,7803
+h5netcdf/legacyapi.py,sha256=MIZlht5Ad4hDFF1Slz2vXmKkgbv7Fhhf2YwNIe16Lfk,7682
+h5netcdf/utils.py,sha256=6E-HAIE0ONMyL4SxI3oUyQvrDgDWifR5EPde91V9rT0,674
+h5netcdf/tests/conftest.py,sha256=4fLa2qoB8br2UpokaOn1-mjHgqVUgVV0G3QLIUzfbZo,2133
+h5netcdf/tests/pytest.ini,sha256=ruJxrLdCIA4bCPVuPQjxsLSlvVxuIsIakK6iQOmz-ak,107
+h5netcdf/tests/test_h5netcdf.py,sha256=Pap1X7YQNRGyU5wH17ZDmHZaPLdRkQwB7BrKQeUjImE,110737
+h5netcdf-1.6.2.dist-info/licenses/AUTHORS.txt,sha256=LTKzUh9o4Wc_oT3aFC48cyDCCP6tdm6VEV_6RrNy4uo,272
+h5netcdf-1.6.2.dist-info/licenses/LICENSE,sha256=Xer1Jg8iL_n9Da0xt0S99blk6tsg9tee_JdgH1rWTjs,1505
+h5netcdf-1.6.2.dist-info/METADATA,sha256=-FuPQxJcnu21jRLDgd6Dqx1Gu39smy8UrAoDImCGAn4,13387
+h5netcdf-1.6.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+h5netcdf-1.6.2.dist-info/top_level.txt,sha256=Fb_KIpOE6MBqjSvxV1Ay7oYce1mdmQ1pO9JQJPDeGqg,9
+h5netcdf-1.6.2.dist-info/RECORD,,

{h5netcdf-1.6.0.dist-info → h5netcdf-1.6.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

h5netcdf-1.6.0.dist-info/RECORD

@@ -1,16 +0,0 @@
-h5netcdf/__init__.py,sha256=Y0EBCcmlJctwl1kCmj7yLijTVy9AioBTr2091vInAtw,456
-h5netcdf/_version.py,sha256=5FGJNp9Lkk9uOxeCjXpoCGBF79Ar6LGPOR7-atBqb_4,511
-h5netcdf/attrs.py,sha256=4IvV4ULLWkz4igFsvu9S2LB745wgUKrIdIuSeO5kpX8,3581
-h5netcdf/core.py,sha256=fU4uOfrGf0n-C6QmtNw-SJXv1WFmmue0dekba_6HUW4,62819
-h5netcdf/dimensions.py,sha256=2g0p9DOAC0hhQ94spIAjWeKC1qyhzzO0s15xCFYSscM,7803
-h5netcdf/legacyapi.py,sha256=MIZlht5Ad4hDFF1Slz2vXmKkgbv7Fhhf2YwNIe16Lfk,7682
-h5netcdf/utils.py,sha256=6E-HAIE0ONMyL4SxI3oUyQvrDgDWifR5EPde91V9rT0,674
-h5netcdf/tests/conftest.py,sha256=l6bOQyqe4gcdKHSNNijeWlFYTpEZse4QEUWbUntAIf4,1825
-h5netcdf/tests/pytest.ini,sha256=ruJxrLdCIA4bCPVuPQjxsLSlvVxuIsIakK6iQOmz-ak,107
-h5netcdf/tests/test_h5netcdf.py,sha256=fB-4-FxhuvqibPBFQzrS-niAZDLzXr7oIauRGqYQWSU,107713
-h5netcdf-1.6.0.dist-info/AUTHORS.txt,sha256=LTKzUh9o4Wc_oT3aFC48cyDCCP6tdm6VEV_6RrNy4uo,272
-h5netcdf-1.6.0.dist-info/LICENSE,sha256=Xer1Jg8iL_n9Da0xt0S99blk6tsg9tee_JdgH1rWTjs,1505
-h5netcdf-1.6.0.dist-info/METADATA,sha256=Lnov_mrz4uS67oVyS16m1zXhVz4nO55NqLOPaIQDGzY,13366
-h5netcdf-1.6.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-h5netcdf-1.6.0.dist-info/top_level.txt,sha256=Fb_KIpOE6MBqjSvxV1Ay7oYce1mdmQ1pO9JQJPDeGqg,9
-h5netcdf-1.6.0.dist-info/RECORD,,