h5netcdf 1.4.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

h5netcdf/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.1'
-__version_tuple__ = version_tuple = (1, 4, 1)
+__version__ = version = '1.6.0'
+__version_tuple__ = version_tuple = (1, 6, 0)

h5netcdf/core.py

@@ -140,6 +140,9 @@ class BaseObject:
 
 
 _h5type_mapping = {
+    "H5T_INTEGER": 0,
+    "H5T_FLOAT": 1,
+    "H5T_STRING": 3,
     "H5T_COMPOUND": 6,
     "H5T_ENUM": 8,
     "H5T_VLEN": 9,
@@ -985,8 +988,7 @@ class Group(Mapping):
         if name in self:
             raise ValueError(f"unable to create group {name:!r} (name already exists)")
         kwargs = {}
-        if self._root._h5py.__name__ == "h5py":
-            kwargs.update(track_order=self._track_order)
+        kwargs.update(track_order=self._track_order)
 
         self._h5group.create_group(name, **kwargs)
         self._groups[name] = self._group_cls(self, name)
@@ -1102,8 +1104,7 @@ class Group(Mapping):
             self._dimensions[name]._detach_scale()
             del self._h5group[name]
 
-        if self._root._h5py.__name__ == "h5py":
-            kwargs.update(dict(track_order=self._parent._track_order))
+        kwargs.update(dict(track_order=self._parent._track_order))
 
         # fill value handling
         fillvalue, h5fillvalue = _check_fillvalue(self, fillvalue, dtype)
@@ -1182,9 +1183,9 @@ class Group(Mapping):
             ``h5netcdf``. Discussion on ``h5netcdf`` chunking can be found in (:issue:`52`)
             and (:pull:`127`).
         compression : str, optional
-            Compression filter to apply, defaults to ``gzip``
+            Compression filter to apply, defaults to ``gzip``. ``zlib`` is an alias for ``gzip``.
         compression_opts : int
-            Parameter for compression filter. For ``compression="gzip"`` Integer from 1 to 9 specifying
+            Parameter for compression filter. For ``compression="gzip"``/``compression="zlib"`` Integer from 1 to 9 specifying
             the compression level. Defaults to 4.
         fletcher32 : bool
             If ``True``, HDF5 Fletcher32 checksum algorithm is applied. Defaults to ``False``.
@@ -1231,6 +1232,13 @@ class Group(Mapping):
         group = self
         for k in keys[:-1]:
             group = group._require_child_group(k)
+
+        # Allow zlib to be an alias for gzip
+        # but use getters and setters so as not to change the behavior
+        # of the default h5py functions
+        if kwargs.get("compression", None) == "zlib":
+            kwargs["compression"] = "gzip"
+
         return group._create_child_variable(
             keys[-1],
             dimensions,
@@ -1497,7 +1505,7 @@ class File(Group):
         self._close_h5file = True
         try:
             if isinstance(path, str):
-                if (
+                if kwargs.get("driver") == "h5pyd" or (
                     path.startswith(("http://", "https://", "hdf5://"))
                     and "driver" not in kwargs
                 ):

h5netcdf/tests/conftest.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import tempfile
 from pathlib import Path
 from shutil import rmtree
@@ -7,7 +6,7 @@ from shutil import rmtree
 import pytest
 
 try:
-    from h5pyd._apps.hstouch import main as hstouch
+    from h5pyd import Folder
     from hsds.hsds_app import HsdsApp
 
     with_reqd_pkgs = True
@@ -15,166 +14,51 @@ except ImportError:
     with_reqd_pkgs = False
 
 
-def set_hsds_root():
-    """Make required HSDS root directory."""
-    hsds_root = Path(os.environ["ROOT_DIR"]) / os.environ["BUCKET_NAME"] / "home"
-    if hsds_root.exists():
-        rmtree(hsds_root)
-
-    old_sysargv = sys.argv
-    sys.argv = [""]
-    sys.argv.extend(["-e", os.environ["HS_ENDPOINT"]])
-    sys.argv.extend(["-u", "admin"])
-    sys.argv.extend(["-p", "admin"])
-    sys.argv.extend(["--bucket", os.environ["BUCKET_NAME"]])
-    sys.argv.append("/home/")
-    hstouch()
-
-    sys.argv = [""]
-    sys.argv.extend(["-e", os.environ["HS_ENDPOINT"]])
-    sys.argv.extend(["-u", "admin"])
-    sys.argv.extend(["-p", "admin"])
-    sys.argv.extend(["--bucket", os.environ["BUCKET_NAME"]])
-    sys.argv.extend(["-o", os.environ["HS_USERNAME"]])
-    sys.argv.append(f'/home/{os.environ["HS_USERNAME"]}/')
-    hstouch()
-    sys.argv = old_sysargv
-
-
 @pytest.fixture(scope="session")
 def hsds_up():
     """Provide HDF Highly Scalabale Data Service (HSDS) for h5pyd testing."""
     if with_reqd_pkgs:
         root_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-root-"))
-        os.environ["BUCKET_NAME"] = "data"
-        (root_dir / os.getenv("BUCKET_NAME")).mkdir(parents=True, exist_ok=True)
-        os.environ["ROOT_DIR"] = str(root_dir)
-        os.environ["HS_USERNAME"] = "h5netcdf-pytest"
-        os.environ["HS_PASSWORD"] = "TestEarlyTestEverything"
+        bucket_name = "pytest"
+        os.environ["BUCKET_NAME"] = bucket_name
+        os.mkdir(
+            f"{root_dir}/{bucket_name}"
+        )  # need to create a directory for our bucket
 
-        config = """allow_noauth: true
-auth_expiration: -1
-default_public: False
-aws_access_key_id: xxx
-aws_secret_access_key: xxx
-aws_iam_role: hsds_role
-aws_region: us-east-1
-hsds_endpoint: http://hsds.hdf.test
-aws_s3_gateway: null
-aws_dynamodb_gateway: null
-aws_dynamodb_users_table: null
-azure_connection_string: null
-azure_resource_id: null
-azure_storage_account: null
-azure_resource_group: null
-root_dir: null
-password_salt: null
-bucket_name: hsdstest
-head_port: 5100
-head_ram: 512m
-dn_port: 6101
-dn_ram: 3g
-sn_port: 5101
-sn_ram: 1g
-rangeget_port: 6900
-rangeget_ram: 2g
-target_sn_count: 0
-target_dn_count: 0
-log_level: INFO
-log_timestamps: false
-log_prefix: null
-max_tcp_connections: 100
-head_sleep_time: 10
-node_sleep_time: 10
-async_sleep_time: 10
-s3_sync_interval: 1
-s3_sync_task_timeout: 10
-store_read_timeout: 1
-store_read_sleep_interval: 0.1
-max_pending_write_requests: 20
-flush_sleep_interval: 1
-max_chunks_per_request: 1000
-min_chunk_size: 1m
-max_chunk_size: 4m
-max_request_size: 100m
-max_chunks_per_folder: 0
-max_task_count: 100
-max_tasks_per_node_per_request: 16
-aio_max_pool_connections: 64
-metadata_mem_cache_size: 128m
-metadata_mem_cache_expire: 3600
-chunk_mem_cache_size: 128m
-chunk_mem_cache_expire: 3600
-data_cache_size: 128m
-data_cache_max_req_size: 128k
-data_cache_expire_time: 3600
-data_cache_page_size: 4m
-data_cache_max_concurrent_read: 16
-timeout: 30
-password_file: /config/passwd.txt
-groups_file: /config/groups.txt
-server_name: Highly Scalable Data Service (HSDS)
-greeting: Welcome to HSDS!
-about: HSDS is a webservice for HDF data
-top_level_domains: []
-cors_domain: "*"
-admin_user: admin
-admin_group: null
-openid_provider: azure
-openid_url: null
-openid_audience: null
-openid_claims: unique_name,appid,roles
-chaos_die: 0
-standalone_app: false
-blosc_nthreads: 2
-http_compression: false
-http_max_url_length: 512
-k8s_app_label: hsds
-k8s_namespace: null
-restart_policy: on-failure
-domain_req_max_objects_limit: 500
-"""
-        tmp_dir = Path(tempfile.mkdtemp(prefix="tmp-hsds-"))
-        config_file = tmp_dir / "config.yml"
-        config_file.write_text(config)
-        passwd_file = tmp_dir / "passwd.txt"
-        passwd_file.write_text(
-            f'admin:admin\n{os.environ["HS_USERNAME"]}:{os.environ["HS_PASSWORD"]}\n'
-        )
-        log_file = str(tmp_dir / "hsds.log")
-        tmp_dir = str(tmp_dir)
-        if sys.platform == "darwin":
-            # macOS temp directory paths can be very long and break low-level
-            # socket comms code...
-            socket_dir = "/tmp/hsds"
-        else:
-            socket_dir = tmp_dir
+        hs_username = "h5netcdf-pytest"
+        hs_password = "TestEarlyTestEverything"
+
+        kwargs = {}
+        kwargs["username"] = hs_username
+        kwargs["password"] = hs_password
+        kwargs["root_dir"] = str(root_dir)
+        kwargs["logfile"] = f"{root_dir}/hsds.log"
+        kwargs["log_level"] = "DEBUG"
+        kwargs["host"] = "localhost"
+        kwargs["sn_port"] = 5101
 
         try:
-            hsds = HsdsApp(
-                username=os.environ["HS_USERNAME"],
-                password=os.environ["HS_PASSWORD"],
-                password_file=str(passwd_file),
-                log_level=os.getenv("LOG_LEVEL", "DEBUG"),
-                logfile=log_file,
-                socket_dir=socket_dir,
-                config_dir=tmp_dir,
-                dn_count=2,
-            )
+            hsds = HsdsApp(**kwargs)
+
             hsds.run()
             is_up = hsds.ready
 
             if is_up:
                 os.environ["HS_ENDPOINT"] = hsds.endpoint
-                set_hsds_root()
+                os.environ["HS_USERNAME"] = hs_username
+                os.environ["HS_PASSWORD"] = hs_password
+                # make folders expected by pytest
+                # pytest/home/h5netcdf-pytest
+                # Folder("/pytest/", mode='w')
+                Folder("/home/", mode="w")
+                Folder("/home/h5netcdf-pytest/", mode="w")
         except Exception:
             is_up = False
 
         yield is_up
-
+        hsds.check_processes()  # this will capture hsds log output
         hsds.stop()
-        rmtree(tmp_dir, ignore_errors=True)
-        rmtree(socket_dir, ignore_errors=True)
+
         rmtree(root_dir, ignore_errors=True)
 
     else:

h5netcdf/tests/test_h5netcdf.py

@@ -107,7 +107,9 @@ _vlen_string = "foo"
 
 
 def is_h5py_char_working(tmp_netcdf, name):
-    if not isinstance(tmp_netcdf, h5py.File):
+    if not isinstance(tmp_netcdf, h5py.File) and (
+        without_h5pyd or not isinstance(tmp_netcdf, h5pyd.File)
+    ):
         h5 = get_hdf5_module(tmp_netcdf)
         # https://github.com/Unidata/netcdf-c/issues/298
         with h5.File(tmp_netcdf, "r") as ds:
@@ -184,14 +186,14 @@ def write_legacy_netcdf(tmp_netcdf, write_module):
     ds.close()
 
 
-def write_h5netcdf(tmp_netcdf):
+def write_h5netcdf(tmp_netcdf, compression="gzip"):
     ds = h5netcdf.File(tmp_netcdf, "w")
     ds.attrs["global"] = 42
     ds.attrs["other_attr"] = "yes"
     ds.dimensions = {"x": 4, "y": 5, "z": 6, "empty": 0, "unlimited": None}
 
     v = ds.create_variable(
-        "foo", ("x", "y"), float, chunks=(4, 5), compression="gzip", shuffle=True
+        "foo", ("x", "y"), float, chunks=(4, 5), compression=compression, shuffle=True
     )
     v[...] = 1
     v.attrs["units"] = "meters"
@@ -515,6 +517,11 @@ def test_roundtrip_h5netcdf(tmp_local_or_remote_netcdf, decode_vlen_strings):
     read_h5netcdf(tmp_local_or_remote_netcdf, h5netcdf, decode_vlen_strings)
 
 
+def test_write_compression_as_zlib(tmp_local_netcdf):
+    write_h5netcdf(tmp_local_netcdf, compression="zlib")
+    read_legacy_netcdf(tmp_local_netcdf, netCDF4, h5netcdf)
+
+
 def test_write_netCDF4_read_h5netcdf(tmp_local_netcdf, decode_vlen_strings):
     write_legacy_netcdf(tmp_local_netcdf, netCDF4)
     read_h5netcdf(tmp_local_netcdf, netCDF4, decode_vlen_strings)
@@ -2649,6 +2656,8 @@ def test_compoundtype_creation(tmp_local_or_remote_netcdf, netcdf_write_module):
     reason="does not work before netCDF4 v1.7.0",
 )
 def test_nc_complex_compatibility(tmp_local_or_remote_netcdf, netcdf_write_module):
+    if tmp_local_or_remote_netcdf.startswith(remote_h5):
+        pytest.skip("not yet implemented in h5pyd/hsds")
     # native complex
     complex_array = np.array([0 + 0j, 1 + 0j, 0 + 1j, 1 + 1j, 0.25 + 0.75j])
     # compound complex
@@ -2726,9 +2735,7 @@ def test_hsds(hsds_up):
     elif not hsds_up:
         pytest.skip("HSDS service not running")
     rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
-    fname = (
-        "hdf5://" + "home" + "/" + env["HS_USERNAME"] + "/" + "testfile" + rnd + ".nc"
-    )
+    fname = f"hdf5://testfile{rnd}.nc"
     with h5netcdf.File(fname, "w") as ds:
         g = ds.create_group("test")
         g.dimensions["x"] = None
@@ -2736,3 +2743,17 @@ def test_hsds(hsds_up):
 
     with h5netcdf.File(fname, "r") as ds:
         print(ds["test"]["var1"])
+
+
+def test_h5pyd_driver(hsds_up):
+    # test that specifying driver='h5pyd' forces use of h5pyd
+    if without_h5pyd:
+        pytest.skip("h5pyd package not available")
+    elif not hsds_up:
+        pytest.skip("HSDS service not running")
+    rnd = "".join(random.choice(string.ascii_uppercase) for _ in range(5))
+    for prefix in ("/", "hdf5://"):
+        fname = f"{prefix}testfile{rnd}.nc"
+        with h5netcdf.File(fname, "w", driver="h5pyd") as ds:
+            assert ds._h5py == h5pyd
+            assert isinstance(ds._h5file, h5pyd.File)

{h5netcdf-1.4.1.dist-info → h5netcdf-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: h5netcdf
-Version: 1.4.1
+Version: 1.6.0
 Summary: netCDF4 via h5py
 Author-email: Stephan Hoyer <shoyer@gmail.com>, Kai Mühlbauer <kmuehlbauer@wradlib.org>
 Maintainer-email: h5netcdf developers <devteam@h5netcdf.org>

h5netcdf-1.6.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+h5netcdf/__init__.py,sha256=Y0EBCcmlJctwl1kCmj7yLijTVy9AioBTr2091vInAtw,456
+h5netcdf/_version.py,sha256=5FGJNp9Lkk9uOxeCjXpoCGBF79Ar6LGPOR7-atBqb_4,511
+h5netcdf/attrs.py,sha256=4IvV4ULLWkz4igFsvu9S2LB745wgUKrIdIuSeO5kpX8,3581
+h5netcdf/core.py,sha256=fU4uOfrGf0n-C6QmtNw-SJXv1WFmmue0dekba_6HUW4,62819
+h5netcdf/dimensions.py,sha256=2g0p9DOAC0hhQ94spIAjWeKC1qyhzzO0s15xCFYSscM,7803
+h5netcdf/legacyapi.py,sha256=MIZlht5Ad4hDFF1Slz2vXmKkgbv7Fhhf2YwNIe16Lfk,7682
+h5netcdf/utils.py,sha256=6E-HAIE0ONMyL4SxI3oUyQvrDgDWifR5EPde91V9rT0,674
+h5netcdf/tests/conftest.py,sha256=l6bOQyqe4gcdKHSNNijeWlFYTpEZse4QEUWbUntAIf4,1825
+h5netcdf/tests/pytest.ini,sha256=ruJxrLdCIA4bCPVuPQjxsLSlvVxuIsIakK6iQOmz-ak,107
+h5netcdf/tests/test_h5netcdf.py,sha256=fB-4-FxhuvqibPBFQzrS-niAZDLzXr7oIauRGqYQWSU,107713
+h5netcdf-1.6.0.dist-info/AUTHORS.txt,sha256=LTKzUh9o4Wc_oT3aFC48cyDCCP6tdm6VEV_6RrNy4uo,272
+h5netcdf-1.6.0.dist-info/LICENSE,sha256=Xer1Jg8iL_n9Da0xt0S99blk6tsg9tee_JdgH1rWTjs,1505
+h5netcdf-1.6.0.dist-info/METADATA,sha256=Lnov_mrz4uS67oVyS16m1zXhVz4nO55NqLOPaIQDGzY,13366
+h5netcdf-1.6.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+h5netcdf-1.6.0.dist-info/top_level.txt,sha256=Fb_KIpOE6MBqjSvxV1Ay7oYce1mdmQ1pO9JQJPDeGqg,9
+h5netcdf-1.6.0.dist-info/RECORD,,

{h5netcdf-1.4.1.dist-info → h5netcdf-1.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

h5netcdf-1.4.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-h5netcdf/__init__.py,sha256=Y0EBCcmlJctwl1kCmj7yLijTVy9AioBTr2091vInAtw,456
-h5netcdf/_version.py,sha256=oFZsPxoSsCY6D2DiWMSueNvMDRRQN5ssWrPdQtlLJ_o,411
-h5netcdf/attrs.py,sha256=4IvV4ULLWkz4igFsvu9S2LB745wgUKrIdIuSeO5kpX8,3581
-h5netcdf/core.py,sha256=5rjGhCTIHZKJ_yMHyvZHIPndPr1Em9CVzyiV5F9H3E8,62511
-h5netcdf/dimensions.py,sha256=2g0p9DOAC0hhQ94spIAjWeKC1qyhzzO0s15xCFYSscM,7803
-h5netcdf/legacyapi.py,sha256=MIZlht5Ad4hDFF1Slz2vXmKkgbv7Fhhf2YwNIe16Lfk,7682
-h5netcdf/utils.py,sha256=6E-HAIE0ONMyL4SxI3oUyQvrDgDWifR5EPde91V9rT0,674
-h5netcdf/tests/conftest.py,sha256=cI0BXKM_LRdsQ8vAl3vJ0r1ShGpNUfj2xOH6KmgfZHw,5034
-h5netcdf/tests/pytest.ini,sha256=ruJxrLdCIA4bCPVuPQjxsLSlvVxuIsIakK6iQOmz-ak,107
-h5netcdf/tests/test_h5netcdf.py,sha256=PIm6AW59QzQ3e8v4yzV3qpr5auR0g7CoNrAhbpxNxMA,106846
-h5netcdf-1.4.1.dist-info/AUTHORS.txt,sha256=LTKzUh9o4Wc_oT3aFC48cyDCCP6tdm6VEV_6RrNy4uo,272
-h5netcdf-1.4.1.dist-info/LICENSE,sha256=Xer1Jg8iL_n9Da0xt0S99blk6tsg9tee_JdgH1rWTjs,1505
-h5netcdf-1.4.1.dist-info/METADATA,sha256=V281CIA-dE3OPxxAfzaE7NeYP9C9Lzwr_RyJv2OIxQM,13366
-h5netcdf-1.4.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-h5netcdf-1.4.1.dist-info/top_level.txt,sha256=Fb_KIpOE6MBqjSvxV1Ay7oYce1mdmQ1pO9JQJPDeGqg,9
-h5netcdf-1.4.1.dist-info/RECORD,,