h5netcdf 1.4.1__tar.gz → 1.6.0__tar.gz

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/CHANGELOG.rst

@@ -1,6 +1,22 @@
 Change Log
 ----------
 
+Version 1.6.0 (March 7th, 2025):
+
+- Allow specifying `h5netcdf.File(driver="h5pyd")` to force the use of h5pyd ({issue}`255`, {pull}`256`).
+  By `Rickard Holmberg <https://github.com/rho-novatron>`_
+- Add pytest-mypy-plugins for xarray nightly test ({pull}`257`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+
+Version 1.5.0 (January 26th, 2025):
+
+- Update CI to new versions (Python 3.13, 3.14 alpha), remove numpy 1 from h5pyd runs ({pull}`250`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_
+- Update CI and reinstate h5pyd/hsds test runs ({pull}`247`).
+  By `John Readey  <https://github.com/jreadey>`_
+- Allow ``zlib`` to be used as an alias for ``gzip`` for enhanced compatibility with h5netcdf's API and xarray.
+  By `Mark Harfouche <https://github.com/hmaarrfk>`_
+
 Version 1.4.1 (November 13th, 2024):
 
 - Add CI run for hdf5 1.10.6, fix complex tests, fix enum/user type tests ({pull}`244`).

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: h5netcdf
-Version: 1.4.1
+Version: 1.6.0
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/h5netcdf/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.1'
-__version_tuple__ = version_tuple = (1, 4, 1)
+__version__ = version = '1.6.0'
+__version_tuple__ = version_tuple = (1, 6, 0)

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/h5netcdf/core.py

@@ -140,6 +140,9 @@ class BaseObject:
 
 
 _h5type_mapping = {
+    "H5T_INTEGER": 0,
+    "H5T_FLOAT": 1,
+    "H5T_STRING": 3,
     "H5T_COMPOUND": 6,
     "H5T_ENUM": 8,
     "H5T_VLEN": 9,
@@ -985,8 +988,7 @@ class Group(Mapping):
         if name in self:
             raise ValueError(f"unable to create group {name:!r} (name already exists)")
         kwargs = {}
-        if self._root._h5py.__name__ == "h5py":
-            kwargs.update(track_order=self._track_order)
+        kwargs.update(track_order=self._track_order)
 
         self._h5group.create_group(name, **kwargs)
         self._groups[name] = self._group_cls(self, name)
@@ -1102,8 +1104,7 @@ class Group(Mapping):
             self._dimensions[name]._detach_scale()
             del self._h5group[name]
 
-        if self._root._h5py.__name__ == "h5py":
-            kwargs.update(dict(track_order=self._parent._track_order))
+        kwargs.update(dict(track_order=self._parent._track_order))
 
         # fill value handling
         fillvalue, h5fillvalue = _check_fillvalue(self, fillvalue, dtype)
@@ -1182,9 +1183,9 @@ class Group(Mapping):
             ``h5netcdf``. Discussion on ``h5netcdf`` chunking can be found in (:issue:`52`)
             and (:pull:`127`).
         compression : str, optional
-            Compression filter to apply, defaults to ``gzip``
+            Compression filter to apply, defaults to ``gzip``. ``zlib`` is an alias for ``gzip``.
         compression_opts : int
-            Parameter for compression filter. For ``compression="gzip"`` Integer from 1 to 9 specifying
+            Parameter for compression filter. For ``compression="gzip"``/``compression="zlib"`` Integer from 1 to 9 specifying
             the compression level. Defaults to 4.
         fletcher32 : bool
             If ``True``, HDF5 Fletcher32 checksum algorithm is applied. Defaults to ``False``.
@@ -1231,6 +1232,13 @@ class Group(Mapping):
         group = self
         for k in keys[:-1]:
             group = group._require_child_group(k)
+
+        # Allow zlib to be an alias for gzip
+        # but use getters and setters so as not to change the behavior
+        # of the default h5py functions
+        if kwargs.get("compression", None) == "zlib":
+            kwargs["compression"] = "gzip"
+
         return group._create_child_variable(
             keys[-1],
             dimensions,
@@ -1497,7 +1505,7 @@ class File(Group):
         self._close_h5file = True
         try:
             if isinstance(path, str):
-                if (
+                if kwargs.get("driver") == "h5pyd" or (
                     path.startswith(("http://", "https://", "hdf5://"))
                     and "driver" not in kwargs
                 ):

h5netcdf-1.6.0/h5netcdf/tests/conftest.py

@@ -0,0 +1,65 @@
+import os
+import tempfile
+from pathlib import Path
+from shutil import rmtree
+
+import pytest
+
+try:
+    from h5pyd import Folder
+    from hsds.hsds_app import HsdsApp
+
+    with_reqd_pkgs = True
+except ImportError:
+    with_reqd_pkgs = False
+
+
+@pytest.fixture(scope="session")
+def hsds_up():
+    """Provide HDF Highly Scalabale Data Service (HSDS) for h5pyd testing."""
+    if with_reqd_pkgs:
+        root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
+        bucket_name = "pytest"
+        os.environ["BUCKET_NAME"] = bucket_name
+        os.mkdir(
+            f"{root_dir}/{bucket_name}"
+        )  # need to create a directory for our bucket
+
+        hs_username = "h5netcdf-pytest"
+        hs_password = "TestEarlyTestEverything"
+
+        kwargs = {}
+        kwargs["username"] = hs_username
+        kwargs["password"] = hs_password
+        kwargs["root_dir"] = str(root_dir)
+        kwargs["logfile"] = f"{root_dir}/hsds.log"
+        kwargs["log_level"] = "DEBUG"
+        kwargs["host"] = "localhost"
+        kwargs["sn_port"] = 5101
+
+        try:
+            hsds = HsdsApp(**kwargs)
+
+            hsds.run()
+            is_up = hsds.ready
+
+            if is_up:
+                os.environ["HS_ENDPOINT"] = hsds.endpoint
+                os.environ["HS_USERNAME"] = hs_username
+                os.environ["HS_PASSWORD"] = hs_password
+                # make folders expected by pytest
+                # pytest/home/h5netcdf-pytest
+                # Folder("/pytest/", mode='w')
+                Folder("/home/", mode="w")
+                Folder("/home/h5netcdf-pytest/", mode="w")
+        except Exception:
+            is_up = False
+
+        yield is_up
+        hsds.check_processes()  # this will capture hsds log output
+        hsds.stop()
+
+        rmtree(root_dir, ignore_errors=True)
+
+    else:
+        yield False

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/h5netcdf/tests/test_h5netcdf.py

@@ -107,7 +107,9 @@ _vlen_string = "foo"
 
 
 def is_h5py_char_working(tmp_netcdf, name):
-    if not isinstance(tmp_netcdf, h5py.File):
+    if not isinstance(tmp_netcdf, h5py.File) and (
+        without_h5pyd or not isinstance(tmp_netcdf, h5pyd.File)
+    ):
         h5 = get_hdf5_module(tmp_netcdf)
         # https://github.com/Unidata/netcdf-c/issues/298
         with h5.File(tmp_netcdf, "r") as ds:
@@ -184,14 +186,14 @@ def write_legacy_netcdf(tmp_netcdf, write_module):
     ds.close()
 
 
-def write_h5netcdf(tmp_netcdf):
+def write_h5netcdf(tmp_netcdf, compression="gzip"):
     ds = h5netcdf.File(tmp_netcdf, "w")
     ds.attrs["global"] = 42
     ds.attrs["other_attr"] = "yes"
     ds.dimensions = {"x": 4, "y": 5, "z": 6, "empty": 0, "unlimited": None}
 
     v = ds.create_variable(
-        "foo", ("x", "y"), float, chunks=(4, 5), compression="gzip", shuffle=True
+        "foo", ("x", "y"), float, chunks=(4, 5), compression=compression, shuffle=True
     )
     v[...] = 1
     v.attrs["units"] = "meters"
@@ -515,6 +517,11 @@ def test_roundtrip_h5netcdf(tmp_local_or_remote_netcdf, decode_vlen_strings):
     read_h5netcdf(tmp_local_or_remote_netcdf, h5netcdf, decode_vlen_strings)
 
 
+def test_write_compression_as_zlib(tmp_local_netcdf):
+    write_h5netcdf(tmp_local_netcdf, compression="zlib")
+    read_legacy_netcdf(tmp_local_netcdf, netCDF4, h5netcdf)
+
+
 def test_write_netCDF4_read_h5netcdf(tmp_local_netcdf, decode_vlen_strings):
     write_legacy_netcdf(tmp_local_netcdf, netCDF4)
     read_h5netcdf(tmp_local_netcdf, netCDF4, decode_vlen_strings)
@@ -2649,6 +2656,8 @@ def test_compoundtype_creation(tmp_local_or_remote_netcdf, netcdf_write_module):
     reason="does not work before netCDF4 v1.7.0",
 )
 def test_nc_complex_compatibility(tmp_local_or_remote_netcdf, netcdf_write_module):
+    if tmp_local_or_remote_netcdf.startswith(remote_h5):
+        pytest.skip("not yet implemented in h5pyd/hsds")
     # native complex
     complex_array = np.array([0 + 0j, 1 + 0j, 0 + 1j, 1 + 1j, 0.25 + 0.75j])
     # compound complex
@@ -2726,9 +2735,7 @@ def test_hsds(hsds_up):
     elif not hsds_up:
         pytest.skip("HSDS service not running")
     rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
-    fname = (
-        "hdf5://" + "home" + "/" + env["HS_USERNAME"] + "/" + "testfile" + rnd + ".nc"
-    )
+    fname = f"hdf5://testfile{rnd}.nc"
     with h5netcdf.File(fname, "w") as ds:
         g = ds.create_group("test")
         g.dimensions["x"] = None
@@ -2736,3 +2743,17 @@ def test_hsds(hsds_up):
 
     with h5netcdf.File(fname, "r") as ds:
         print(ds["test"]["var1"])
+
+
+def test_h5pyd_driver(hsds_up):
+    # test that specifying driver='h5pyd' forces use of h5pyd
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    for prefix in ("/", "hdf5://"):
+        fname = f"{prefix}testfile{rnd}.nc"
+        with h5netcdf.File(fname, "w", driver="h5pyd") as ds:
+            assert ds._h5py == h5pyd
+            assert isinstance(ds._h5file, h5pyd.File)

{h5netcdf-1.4.1 → h5netcdf-1.6.0}/h5netcdf.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: h5netcdf
-Version: 1.4.1
+Version: 1.6.0
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>

h5netcdf-1.4.1/h5netcdf/tests/conftest.py

@@ -1,181 +0,0 @@
-import os
-import sys
-import tempfile
-from pathlib import Path
-from shutil import rmtree
-
-import pytest
-
-try:
-    from h5pyd._apps.hstouch import main as hstouch
-    from hsds.hsds_app import HsdsApp
-
-    with_reqd_pkgs = True
-except ImportError:
-    with_reqd_pkgs = False
-
-
-def set_hsds_root():
-    """Make required HSDS root directory."""
-    hsds_root = Path(os.environ["ROOT_DIR"]) / os.environ["BUCKET_NAME"] / "home"
-    if hsds_root.exists():
-        rmtree(hsds_root)
-
-    old_sysargv = sys.argv
-    sys.argv = [""]
-    sys.argv.extend(["-e", os.environ["HS_ENDPOINT"]])
-    sys.argv.extend(["-u", "admin"])
-    sys.argv.extend(["-p", "admin"])
-    sys.argv.extend(["--bucket", os.environ["BUCKET_NAME"]])
-    sys.argv.append("/home/")
-    hstouch()
-
-    sys.argv = [""]
-    sys.argv.extend(["-e", os.environ["HS_ENDPOINT"]])
-    sys.argv.extend(["-u", "admin"])
-    sys.argv.extend(["-p", "admin"])
-    sys.argv.extend(["--bucket", os.environ["BUCKET_NAME"]])
-    sys.argv.extend(["-o", os.environ["HS_USERNAME"]])
-    sys.argv.append(f'/home/{os.environ["HS_USERNAME"]}/')
-    hstouch()
-    sys.argv = old_sysargv
-
-
-@pytest.fixture(scope="session")
-def hsds_up():
-    """Provide HDF Highly Scalabale Data Service (HSDS) for h5pyd testing."""
-    if with_reqd_pkgs:
-        root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
-        os.environ["BUCKET_NAME"] = "data"
-        (root_dir / os.getenv("BUCKET_NAME")).mkdir(parents=True, exist_ok=True)
-        os.environ["ROOT_DIR"] = str(root_dir)
-        os.environ["HS_USERNAME"] = "h5netcdf-pytest"
-        os.environ["HS_PASSWORD"] = "TestEarlyTestEverything"
-
-        config = """allow_noauth: true
-auth_expiration: -1
-default_public: False
-aws_access_key_id: xxx
-aws_secret_access_key: xxx
-aws_iam_role: hsds_role
-aws_region: us-east-1
-hsds_endpoint: http://hsds.hdf.test
-aws_s3_gateway: null
-aws_dynamodb_gateway: null
-aws_dynamodb_users_table: null
-azure_connection_string: null
-azure_resource_id: null
-azure_storage_account: null
-azure_resource_group: null
-root_dir: null
-password_salt: null
-bucket_name: hsdstest
-head_port: 5100
-head_ram: 512m
-dn_port: 6101
-dn_ram: 3g
-sn_port: 5101
-sn_ram: 1g
-rangeget_port: 6900
-rangeget_ram: 2g
-target_sn_count: 0
-target_dn_count: 0
-log_level: INFO
-log_timestamps: false
-log_prefix: null
-max_tcp_connections: 100
-head_sleep_time: 10
-node_sleep_time: 10
-async_sleep_time: 10
-s3_sync_interval: 1
-s3_sync_task_timeout: 10
-store_read_timeout: 1
-store_read_sleep_interval: 0.1
-max_pending_write_requests: 20
-flush_sleep_interval: 1
-max_chunks_per_request: 1000
-min_chunk_size: 1m
-max_chunk_size: 4m
-max_request_size: 100m
-max_chunks_per_folder: 0
-max_task_count: 100
-max_tasks_per_node_per_request: 16
-aio_max_pool_connections: 64
-metadata_mem_cache_size: 128m
-metadata_mem_cache_expire: 3600
-chunk_mem_cache_size: 128m
-chunk_mem_cache_expire: 3600
-data_cache_size: 128m
-data_cache_max_req_size: 128k
-data_cache_expire_time: 3600
-data_cache_page_size: 4m
-data_cache_max_concurrent_read: 16
-timeout: 30
-password_file: /config/passwd.txt
-groups_file: /config/groups.txt
-server_name: Highly Scalable Data Service (HSDS)
-greeting: Welcome to HSDS!
-about: HSDS is a webservice for HDF data
-top_level_domains: []
-cors_domain: "*"
-admin_user: admin
-admin_group: null
-openid_provider: azure
-openid_url: null
-openid_audience: null
-openid_claims: unique_name,appid,roles
-chaos_die: 0
-standalone_app: false
-blosc_nthreads: 2
-http_compression: false
-http_max_url_length: 512
-k8s_app_label: hsds
-k8s_namespace: null
-restart_policy: on-failure
-domain_req_max_objects_limit: 500
-"""
-        tmp_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-"))
-        config_file = tmp_dir / "config.yml"
-        config_file.write_text(config)
-        passwd_file = tmp_dir / "passwd.txt"
-        passwd_file.write_text(
-            f'admin:admin\n{os.environ["HS_USERNAME"]}:{os.environ["HS_PASSWORD"]}\n'
-        )
-        log_file = str(tmp_dir / "hsds.log")
-        tmp_dir = str(tmp_dir)
-        if sys.platform == "darwin":
-            # macOS temp directory paths can be very long and break low-level
-            # socket comms code...
-            socket_dir = "/tmp/hsds"
-        else:
-            socket_dir = tmp_dir
-
-        try:
-            hsds = HsdsApp(
-                username=os.environ["HS_USERNAME"],
-                password=os.environ["HS_PASSWORD"],
-                password_file=str(passwd_file),
-                log_level=os.getenv("LOG_LEVEL", "DEBUG"),
-                logfile=log_file,
-                socket_dir=socket_dir,
-                config_dir=tmp_dir,
-                dn_count=2,
-            )
-            hsds.run()
-            is_up = hsds.ready
-
-            if is_up:
-                os.environ["HS_ENDPOINT"] = hsds.endpoint
-                set_hsds_root()
-        except Exception:
-            is_up = False
-
-        yield is_up
-
-        hsds.stop()
-        rmtree(tmp_dir, ignore_errors=True)
-        rmtree(socket_dir, ignore_errors=True)
-        rmtree(root_dir, ignore_errors=True)
-
-    else:
-        yield False