tensogram-xarray 0.20.0__tar.gz → 0.21.0__tar.gz

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensogram-xarray
-Version: 0.20.0
+Version: 0.21.0
 Summary: xarray backend engine for tensogram .tgm files
 Project-URL: Homepage, https://sites.ecmwf.int/docs/tensogram/main
 Project-URL: Repository, https://github.com/ecmwf/tensogram
@@ -14,7 +14,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
 Requires-Python: >=3.11
 Requires-Dist: numpy
-Requires-Dist: tensogram<0.21,>=0.20.0
+Requires-Dist: tensogram<0.22,>=0.21.0
 Requires-Dist: xarray>=2022.06
 Provides-Extra: dask
 Requires-Dist: dask[array]; extra == 'dask'

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "tensogram-xarray"
-version = "0.20.0"
+version = "0.21.0"
 description = "xarray backend engine for tensogram .tgm files"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -18,7 +18,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Atmospheric Science",
 ]
 dependencies = [
-    "tensogram>=0.20.0,<0.21",
+    "tensogram>=0.21.0,<0.22",
     "xarray>=2022.06",
     "numpy",
 ]

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/src/tensogram_xarray/array.py

@@ -200,11 +200,11 @@ class TensogramBackendArray(BackendArray):
         dtype: np.dtype,
         supports_range: bool,
         *,
-        verify_hash: bool = False,
         range_threshold: float = DEFAULT_RANGE_THRESHOLD,
         lock: threading.Lock | None = None,
         storage_options: dict[str, Any] | None = None,
         shared_file: Any | None = None,
+        verify_hash: bool = False,
     ):
         import tensogram
 
@@ -215,10 +215,10 @@ class TensogramBackendArray(BackendArray):
         self.shape = shape
         self.dtype = dtype
         self.supports_range = supports_range
-        self.verify_hash = verify_hash
         self.range_threshold = range_threshold
         self.storage_options = storage_options
         self._shared_file = shared_file
+        self.verify_hash = verify_hash
 
     # -- pickle support (no open handles stored) ----------------------------
 
@@ -267,12 +267,25 @@ class TensogramBackendArray(BackendArray):
                 total_elements = math.prod(self.shape)
 
                 if total_elements > 0 and total_requested / total_elements <= self.range_threshold:
+                    # Per the decode-time verification contract
+                    # (PLAN_DECODE_HASH_VERIFICATION §6, Q6): the
+                    # range-decode fast path does *not* verify
+                    # hashes — verifying a whole-frame hash would
+                    # require reading every byte the optimisation
+                    # is designed to avoid.  When `verify_hash` is
+                    # True, callers who care about integrity should
+                    # set ``range_threshold=0`` to force every read
+                    # through the full-decode path below.  We
+                    # *do* allow this fast path even under
+                    # ``verify_hash=True`` so the user keeps the
+                    # remote-fetch cost characteristics they
+                    # opted in to via ``range_threshold``; the
+                    # verification simply does not apply.
                     arr = f.file_decode_range(
                         self.msg_index,
                         obj_index=self.obj_index,
                         ranges=flat_ranges,
                         join=True,
-                        verify_hash=self.verify_hash,
                         native_byte_order=True,
                     )
                     return np.asarray(arr).reshape(out_shape)

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/src/tensogram_xarray/backend.py

@@ -55,9 +55,9 @@ class TensogramBackendEntrypoint(BackendEntrypoint):
         variable_key: str | None = None,
         message_index: int = 0,
         merge_objects: bool = False,
-        verify_hash: bool = False,
         range_threshold: float = 0.5,
         storage_options: dict[str, Any] | None = None,
+        verify_hash: bool = False,
     ) -> xr.Dataset:
         """Open a single tensogram message as an :class:`xr.Dataset`.
 
@@ -79,8 +79,6 @@ class TensogramBackendEntrypoint(BackendEntrypoint):
             If *True*, attempt to merge objects across messages by stacking
             along metadata dimensions that vary.  When *False* (default),
             only the single message at *message_index* is opened.
-        verify_hash
-            Whether to verify xxh3 hashes during decode.
         range_threshold
             Maximum fraction of total array elements (0.0-1.0) for which
             partial ``decode_range()`` is used instead of a full
@@ -88,6 +86,26 @@ class TensogramBackendEntrypoint(BackendEntrypoint):
         storage_options
             Key-value pairs forwarded to the object store backend when
             the path is a remote URL.  Ignored for local files.
+        verify_hash
+            When *True*, every full ``decode_object`` (or
+            ``file_decode_object``) call materialised by this Dataset's
+            lazy backing arrays is verified against its inline xxh3
+            hash; ``MissingHashError`` / ``HashMismatchError`` from
+            the underlying tensogram bindings propagate to the
+            caller's first read.
+
+            **Caveat — partial-range fast path is unverified.**  Per
+            the decode-time verification contract (see
+            ``plans/DESIGN.md`` §"Integrity Hashing" and
+            ``plans/WIRE_FORMAT.md`` §11.1), ``decode_range`` reads
+            only a slice of the encoded payload and cannot meaningfully
+            verify a whole-frame hash.  When ``verify_hash=True`` and
+            the lazy reader chooses ``file_decode_range`` (because
+            the requested slice is below ``range_threshold``), no
+            verification happens for that read.  Set
+            ``range_threshold=0`` to force every read through the
+            full-decode path if you need consistent integrity
+            coverage.
 
         Returns
         -------
@@ -107,9 +125,9 @@ class TensogramBackendEntrypoint(BackendEntrypoint):
                 file_path,
                 dim_names=dim_names,
                 variable_key=variable_key,
-                verify_hash=verify_hash,
                 range_threshold=range_threshold,
                 storage_options=storage_options,
+                verify_hash=verify_hash,
             )
             if not datasets:
                 return xr.Dataset()
@@ -120,9 +138,9 @@ class TensogramBackendEntrypoint(BackendEntrypoint):
             msg_index=message_index,
             dim_names=dim_names,
             variable_key=variable_key,
-            verify_hash=verify_hash,
             range_threshold=range_threshold,
             storage_options=storage_options,
+            verify_hash=verify_hash,
         )
 
         drop_set = set(drop_variables) if drop_variables else None

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/src/tensogram_xarray/merge.py

@@ -51,8 +51,8 @@ def open_datasets(
     *,
     dim_names: Sequence[str] | None = None,
     variable_key: str | None = None,
-    verify_hash: bool = False,
     range_threshold: float = 0.5,
+    verify_hash: bool = False,
     storage_options: dict[str, Any] | None = None,
 ) -> list[xr.Dataset]:
     """Open a ``.tgm`` file, auto-grouping into compatible Datasets.
@@ -69,8 +69,6 @@ def open_datasets(
         Explicit dimension names for the innermost tensor axes.
     variable_key
         Dotted metadata key path for variable naming.
-    verify_hash
-        Whether to verify hashes on decode.
     range_threshold
         Maximum fraction of total array elements for which partial
         ``decode_range()`` is used.  Default ``0.5``.
@@ -114,8 +112,8 @@ def open_datasets(
             shape=shape,
             dtype=np_dtype,
             supports_range=_supports_range_decode(obj.descriptor),
-            verify_hash=verify_hash,
             range_threshold=range_threshold,
+            verify_hash=verify_hash,
             lock=lock,
             storage_options=storage_options,
             shared_file=shared_file,
@@ -479,8 +477,8 @@ def _single_object_dataset(
         shape=shape,
         dtype=np_dtype,
         supports_range=_supports_range_decode(obj.descriptor),
-        verify_hash=verify_hash,
         range_threshold=range_threshold,
+        verify_hash=verify_hash,
         lock=lock,
         storage_options=storage_options,
         shared_file=shared_file,
@@ -532,8 +530,8 @@ def _flat_group_dataset(
             shape=obj.shape,
             dtype=np_dtype,
             supports_range=_supports_range_decode(obj.descriptor),
-            verify_hash=verify_hash,
             range_threshold=range_threshold,
+            verify_hash=verify_hash,
             lock=lock,
             storage_options=storage_options,
             shared_file=shared_file,
@@ -696,8 +694,8 @@ def _build_multi_variable_dataset(
                 shape=inner_shape,
                 dtype=np_dtype,
                 supports_range=_supports_range_decode(obj.descriptor),
-                verify_hash=verify_hash,
                 range_threshold=range_threshold,
+                verify_hash=verify_hash,
                 lock=lock,
                 storage_options=storage_options,
                 shared_file=shared_file,
@@ -786,8 +784,8 @@ def _build_multi_variable_dataset(
                     shape=inner_shape,
                     dtype=np_dtype,
                     supports_range=_supports_range_decode(obj.descriptor),
-                    verify_hash=verify_hash,
                     range_threshold=range_threshold,
+                    verify_hash=verify_hash,
                     lock=lock,
                     storage_options=storage_options,
                     shared_file=shared_file,
@@ -816,8 +814,8 @@ def _build_multi_variable_dataset(
                 shape=inner_shape,
                 dtype=np_dtype,
                 supports_range=_supports_range_decode(obj.descriptor),
-                verify_hash=verify_hash,
                 range_threshold=range_threshold,
+                verify_hash=verify_hash,
                 lock=lock,
                 storage_options=storage_options,
                 shared_file=shared_file,

{tensogram_xarray-0.20.0 → tensogram_xarray-0.21.0}/src/tensogram_xarray/store.py

@@ -196,8 +196,6 @@ class TensogramDataStore:
         Optional user-specified dimension names for data variables.
     variable_key
         Optional dotted metadata path for variable naming.
-    verify_hash
-        Whether to verify object hashes on decode.
     range_threshold
         Maximum fraction of total array elements (0.0-1.0) for which
         partial ``decode_range()`` is used.  Default ``0.5``.
@@ -214,9 +212,9 @@ class TensogramDataStore:
         msg_index: int = 0,
         dim_names: Sequence[str] | None = None,
         variable_key: str | None = None,
-        verify_hash: bool = False,
         range_threshold: float = 0.5,
         storage_options: dict[str, Any] | None = None,
+        verify_hash: bool = False,
     ):
         import tensogram
 
@@ -225,9 +223,9 @@ class TensogramDataStore:
         self.msg_index = msg_index
         self.dim_names = dim_names
         self.variable_key = variable_key
-        self.verify_hash = verify_hash
         self.range_threshold = range_threshold
         self.storage_options = storage_options
+        self.verify_hash = verify_hash
         self._lock = threading.Lock()
         self._backend_arrays: list[TensogramBackendArray] = []
 
@@ -347,11 +345,11 @@ class TensogramDataStore:
                 shape=shape,
                 dtype=np_dtype,
                 supports_range=_supports_range_decode(desc),
-                verify_hash=self.verify_hash,
                 range_threshold=self.range_threshold,
                 lock=self._lock,
                 storage_options=self.storage_options,
                 shared_file=self._file,
+                verify_hash=self.verify_hash,
             )
             self._backend_arrays.append(backend_array)
             lazy_data = indexing.LazilyIndexedArray(backend_array)
@@ -387,11 +385,11 @@ class TensogramDataStore:
                 shape=shape,
                 dtype=np_dtype,
                 supports_range=_supports_range_decode(desc),
-                verify_hash=self.verify_hash,
                 range_threshold=self.range_threshold,
                 lock=self._lock,
                 storage_options=self.storage_options,
                 shared_file=self._file,
+                verify_hash=self.verify_hash,
             )
             self._backend_arrays.append(backend_array)
 

tensogram_xarray-0.21.0/tests/test_verify_hash.py

@@ -0,0 +1,123 @@
+# (C) Copyright 2026- ECMWF and individual contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+"""verify_hash threading through the xarray backend.
+
+The xarray backend re-exposes the upstream
+``DecodeOptions::verify_hash`` flag via the ``open_dataset(...,
+verify_hash=True)`` keyword.  When the lazy backing arrays
+materialise data via ``file_decode_object`` /
+``decode_object``, the kwarg propagates and integrity errors
+(``MissingHashError`` / ``HashMismatchError``) bubble up to
+the caller's first read.
+
+Per Q6 in ``PLAN_DECODE_HASH_VERIFICATION.md``: the partial-
+range fast path silently does *not* verify (range decode
+does not accept ``verify_hash``).  Set ``range_threshold=0``
+to force every read through the full-decode path if you
+need uniform coverage.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+import tensogram
+import xarray as xr
+
+
+def _build_unhashed_message(tmp_path) -> str:
+    """Encode a 1-object f32 message with hashing off + write to disk.
+
+    Returns the file path.  The unhashed encoding is what makes
+    cell C (`verify_hash=True` → MissingHashError) testable.
+    """
+    meta = {"version": 3}
+    desc = {
+        "type": "ntensor",
+        "ndim": 1,
+        "shape": [4],
+        "strides": [1],
+        "dtype": "float32",
+        "encoding": "none",
+        "filter": "none",
+        "compression": "none",
+    }
+    data = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
+    msg = bytes(tensogram.encode(meta, [(desc, data)], hash=None))
+    path = tmp_path / "unhashed.tgm"
+    path.write_bytes(msg)
+    return str(path)
+
+
+def _build_hashed_message(tmp_path) -> str:
+    """Encode a 1-object f32 message with hashing on + write to disk."""
+    meta = {"version": 3}
+    desc = {
+        "type": "ntensor",
+        "ndim": 1,
+        "shape": [4],
+        "strides": [1],
+        "dtype": "float32",
+        "encoding": "none",
+        "filter": "none",
+        "compression": "none",
+    }
+    data = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
+    msg = bytes(tensogram.encode(meta, [(desc, data)], hash="xxh3"))
+    path = tmp_path / "hashed.tgm"
+    path.write_bytes(msg)
+    return str(path)
+
+
+class TestOpenDatasetVerifyHash:
+    def test_verify_hash_default_is_false(self, tmp_path):
+        """Default ``verify_hash=False`` decodes both hashed and
+        unhashed fixtures cleanly."""
+        ds_hashed = xr.open_dataset(_build_hashed_message(tmp_path), engine="tensogram")
+        # Force materialisation.
+        np.asarray(ds_hashed[next(iter(ds_hashed.data_vars))].values)
+        ds_hashed.close()
+
+        ds_unhashed = xr.open_dataset(_build_unhashed_message(tmp_path), engine="tensogram")
+        np.asarray(ds_unhashed[next(iter(ds_unhashed.data_vars))].values)
+        ds_unhashed.close()
+
+    def test_verify_hash_true_succeeds_on_hashed_dataset(self, tmp_path):
+        """Cell B equivalent on the xarray surface: opening a
+        hashed file with ``verify_hash=True`` and pulling data
+        materialises cleanly."""
+        ds = xr.open_dataset(
+            _build_hashed_message(tmp_path),
+            engine="tensogram",
+            verify_hash=True,
+            # Force the full-decode path so the verification fires.
+            range_threshold=0.0,
+        )
+        arr = np.asarray(ds[next(iter(ds.data_vars))].values)
+        np.testing.assert_array_equal(arr, [1.0, 2.0, 3.0, 4.0])
+        ds.close()
+
+    def test_verify_hash_true_raises_missing_hash_on_unhashed(self, tmp_path):
+        """Cell C on xarray: open a hashless file with
+        ``verify_hash=True`` and the first read raises
+        ``MissingHashError`` from the underlying
+        :mod:`tensogram` bindings."""
+        ds = xr.open_dataset(
+            _build_unhashed_message(tmp_path),
+            engine="tensogram",
+            verify_hash=True,
+            # Force the full-decode path so the verification fires.
+            range_threshold=0.0,
+        )
+        try:
+            with pytest.raises(tensogram.MissingHashError) as excinfo:
+                _ = np.asarray(ds[next(iter(ds.data_vars))].values)
+            assert excinfo.value.object_index == 0
+        finally:
+            ds.close()