ComDAS 0.1.0__py3-none-any.whl

comdas/io/h5_container.py

@@ -0,0 +1,429 @@
+"""
+FiberIO plugin for the ComDAS HDF5 container format.
+
+The ComDAS HDF5 container format is designed to store patches
+in a codec-agnostic manner
+
+Registers the ``COMDAS`` format with DASCore's FiberIO manager. Once
+this module is imported (or the package is pip-installed, via the
+``dascore.fiber_io`` entry point in ``pyproject.toml``), ``dc.spool(path)``
+and ``dc.write(patches, path, file_format="COMDAS")`` work
+like any other DASCore format regardless of which
+:class:`~comdas.codecs.base.Codec` was used to compress each patch.
+
+File layout
+-----------
+::
+
+    /                                       (root)
+      attrs:
+        __format__ = "COMDAS"
+        __COMDAS_version__ = "1"
+    /patches
+      /patch_0000
+        attrs:
+          codec_name         (str)   -- e.g. "SVD"; looked up in the codec registry
+          codec_version      (str)   -- the codec's own version string
+          compression_params (dict)  -- parameters used for compression
+          original_shape     (tuple[int, ...])
+          dtype              (str)   -- original array dtype
+          dims               (str)   -- comma-separated dim names, in order
+          attrs_json         (str)   -- PatchAttrs.model_dump_json()
+        <codec-specific datasets/attrs, written by codec.payload_to_group>
+        /coords
+          <dim_name>              -- one dataset per dim, full coord array
+            attrs: dtype (str), units (str, empty if none)
+      /patch_0001
+        ...
+
+Everything above the codec-specific datasets is written and read by
+this module. Only what's inside a patch's own group (beyond
+``coords``) is delegated to ``codec.payload_to_group`` /
+``codec.payload_from_group``.
+
+Patches are appended, not overwritten: calling ``write`` again on an
+existing file adds new ``patch_XXXX`` groups after the existing ones
+(``H5Writer`` opens in ``"a"`` mode).
+
+Once ``comdas`` is installed, plain ``dc.spool(path)`` finds and uses this
+format automatically. Writing has one convenience function,
+:func:`write_compressed`.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+import dascore as dc
+from dascore.io import FiberIO
+from dascore.utils.hdf5 import H5Reader, H5Writer
+
+from comdas.arrays.duck_array import DuckArray
+from comdas.codecs.base import Codec
+
+FORMAT_NAME = "COMDAS"
+FORMAT_VERSION = "1"
+_PATCH_GROUP_PREFIX = "patch_"
+
+
+def _patch_group_names(h5file) -> list[str]:
+    """
+    List patch group names in a COMDAS file, in write order.
+
+    :param h5file: An open ``h5py.File`` for a COMDAS container.
+    :returns: Sorted patch group names (empty if the file has none yet).
+    :rtype: list[str]
+    """
+    if "patches" not in h5file:
+        return []
+    return sorted(h5file["patches"].keys())
+
+
+def _next_patch_index(h5file) -> int:
+    """
+    The next unused ``patch_XXXX`` index for appending to ``h5file``.
+
+    :param h5file: An open ``h5py.File`` for a COMDAS container.
+    :returns: The next patch index to use.
+    :rtype: int
+    """
+    names = _patch_group_names(h5file)
+    if not names:
+        return 0
+    return int(names[-1][len(_PATCH_GROUP_PREFIX) :]) + 1
+
+
+def _dump_coord_array(arr: np.ndarray) -> tuple[np.ndarray, str]:
+    """
+    Convert a coordinate array to something h5py can store directly.
+
+    h5py can't store ``datetime64``/``timedelta64`` directly, so those
+    are viewed as ``int64`` for storage; the exact original dtype
+    string is returned so :func:`_load_coord_array` can cast back.
+
+    :param arr: The coordinate array to prepare for storage.
+    :type arr: numpy.ndarray
+    :returns: A ``(storable_array, dtype_str)`` pair.
+    :rtype: tuple[numpy.ndarray, str]
+    """
+    dtype_str = str(arr.dtype)
+    if np.issubdtype(arr.dtype, np.datetime64) or np.issubdtype(
+        arr.dtype, np.timedelta64
+    ):
+        return arr.astype("int64"), dtype_str
+    return arr, dtype_str
+
+
+def _load_coord_array(arr: np.ndarray, dtype_str: str) -> np.ndarray:
+    """
+    Invert :func:`_dump_coord_array`.
+
+    :param arr: The array as read from the HDF5 dataset.
+    :type arr: numpy.ndarray
+    :param dtype_str: The original dtype string, as stored on write.
+    :type dtype_str: str
+    :returns: The coordinate array cast back to its original dtype.
+    :rtype: numpy.ndarray
+    """
+    return arr.astype(dtype_str)
+
+
+def _range_overlaps(requested, actual_min, actual_max) -> bool:
+    """
+    Whether a ``(lo, hi)`` request overlaps an ``[actual_min, actual_max]`` range.
+
+    :param requested: A ``(lo, hi)`` tuple (either bound may be
+        ``None`` for "unbounded"), or ``None`` for "no constraint".
+    :param actual_min: The range's actual minimum, or ``None``.
+    :param actual_max: The range's actual maximum, or ``None``.
+    :returns: True if the two ranges overlap (or either is unbounded).
+    :rtype: bool
+    """
+    if requested is None:
+        return True
+    lo, hi = requested
+    if lo is not None and actual_max is not None and lo > actual_max:
+        return False
+    if hi is not None and actual_min is not None and hi < actual_min:
+        return False
+    return True
+
+
+def _patch_matches_kwargs(attrs: dc.PatchAttrs, kwargs: dict) -> bool:
+    """
+    Whether a stored patch's coord ranges match DASCore's selection kwargs.
+
+    :param attrs: The candidate patch's attrs (with a ``coords``
+        summary mapping dim name to a min/max-bearing object).
+    :type attrs: dascore.PatchAttrs
+    :param kwargs: Keyword arguments as received by
+        :meth:`ComdasV1.read`; only keys matching a dim name in
+        ``attrs.coords`` are consulted.
+    :returns: True if every dim-range constraint in ``kwargs`` overlaps
+        this patch's own range for that dim.
+    :rtype: bool
+    """
+    for dim, summary in attrs.coords.items():
+        if dim not in kwargs or kwargs[dim] is None:
+            continue
+        if not _range_overlaps(kwargs[dim], summary.min, summary.max):
+            return False
+    return True
+
+
+def _write_patch(
+    patches_group, name: str, patch, codec: Codec, encode_kwargs: dict
+) -> None:
+    """
+    Compress and write one patch into ``patches_group[name]``.
+
+    :param patches_group: The open ``/patches`` HDF5 group.
+    :type patches_group: h5py.Group
+    :param name: The new patch group's name (e.g. ``"patch_0003"``).
+    :type name: str
+    :param patch: The (uncompressed) patch to write.
+    :type patch: dascore.Patch
+    :param codec: The codec to compress ``patch.data`` with.
+    :type codec: Codec
+    :param encode_kwargs: Forwarded to ``codec.encode``.
+    :type encode_kwargs: dict
+    :raises ValueError: If ``patch.data`` is not 2D (the only shape
+        ComDAS codecs currently support).
+    """
+    dense = np.asarray(patch.data)
+    if dense.ndim != 2:
+        raise ValueError(
+            "ComDAS codecs currently only support 2D patches (e.g. "
+            f"channel x time), got shape {dense.shape} for dims {patch.dims}"
+        )
+    payload = codec.encode(dense, **encode_kwargs)
+
+    grp = patches_group.create_group(name)
+    grp.attrs["codec_name"] = codec.name
+    grp.attrs["codec_version"] = codec.version
+    grp.attrs["original_shape"] = payload.original_shape
+    grp.attrs["dtype"] = str(payload.dtype)
+    grp.attrs["dims"] = ",".join(patch.dims)
+    grp.attrs["attrs_json"] = patch.attrs.model_dump_json()
+
+    codec.payload_to_group(payload, grp)
+
+    coord_grp = grp.create_group("coords")
+    for dim in patch.dims:
+        arr = patch.coords.get_array(dim)
+        storable, dtype_str = _dump_coord_array(arr)
+        cds = coord_grp.create_dataset(dim, data=storable)
+        cds.attrs["dtype"] = dtype_str
+        coord = patch.coords.get_coord(dim)
+        units = getattr(coord, "units", None)
+        cds.attrs["units"] = str(units) if units is not None else ""
+
+
+def _read_patch(patch_group) -> dc.Patch:
+    """
+    Read one patch group back into an ordinary compressed Patch.
+
+    :param patch_group: An open patch group (e.g. ``f["patches"]["patch_0000"]``).
+    :type patch_group: h5py.Group
+    :returns: The reconstructed Patch, backed by a
+        :class:`~comdas.arrays.duck_array.DuckArray`.
+    :rtype: dascore.Patch
+    """
+    codec_cls = Codec.get_registered(patch_group.attrs["codec_name"])
+    codec = codec_cls()
+    original_shape = tuple(patch_group.attrs["original_shape"])
+    dtype = np.dtype(patch_group.attrs["dtype"])
+    payload = codec.payload_from_group(
+        patch_group, original_shape=original_shape, dtype=dtype
+    )
+    compressed_array = DuckArray(payload, codec)
+
+    dims = tuple(patch_group.attrs["dims"].split(","))
+    coord_grp = patch_group["coords"]
+    coords = {}
+    for dim in dims:
+        ds = coord_grp[dim]
+        raw = ds[:]
+        dtype_str = ds.attrs.get("dtype", str(raw.dtype))
+        coords[dim] = _load_coord_array(raw, dtype_str)
+
+    attrs = dc.PatchAttrs.model_validate_json(patch_group.attrs["attrs_json"])
+
+    return dc.Patch(
+        data=compressed_array, coords=coords, dims=dims, attrs=attrs
+    )
+
+
+class ComdasV1(FiberIO):
+    """
+    FiberIO support for the codec-agnostic ComDAS HDF5 container, v1.
+
+    Each patch stores which :class:`~comdas.codecs.base.Codec` it was
+    compressed with. This means a single file can hold patches
+    compressed with different codecs.
+    """
+
+    name = FORMAT_NAME
+    version = FORMAT_VERSION
+    preferred_extensions = ("h5", "hdf5")
+
+    def get_format(
+        self, resource: H5Reader, **kwargs
+    ) -> tuple[str, str] | bool:
+        """
+        Identify a COMDAS file via its root-level magic attrs.
+
+        :param resource: An open, readable HDF5 file (auto-opened by
+            DASCore from a path via the ``H5Reader`` type hint).
+        :type resource: h5py.File
+        :param kwargs: Unused; accepted for FiberIO API compatibility.
+        :returns: ``(format_name, format_version)`` if this file is a
+            COMDAS container, else ``False``.
+        :rtype: tuple[str, str] or bool
+        """
+        fmt = resource.attrs.get("__format__")
+        if fmt == FORMAT_NAME:
+            version = resource.attrs.get(
+                f"__{FORMAT_NAME}_version__", FORMAT_VERSION
+            )
+            return FORMAT_NAME, str(version)
+        return False
+
+    def scan(self, resource: H5Reader, **kwargs) -> list[dc.PatchAttrs]:
+        """
+        Read every patch's attrs without touching datasets.
+
+        :param resource: An open, readable COMDAS HDF5 file.
+        :type resource: h5py.File
+        :param kwargs: Unused; accepted for FiberIO API compatibility.
+        :returns: One :class:`dascore.PatchAttrs` per stored patch.
+        :rtype: list[dascore.PatchAttrs]
+        """
+        out = []
+        for name in _patch_group_names(resource):
+            grp = resource["patches"][name]
+            attrs = dc.PatchAttrs.model_validate_json(grp.attrs["attrs_json"])
+            attrs = attrs.update(
+                file_format=self.name,
+                file_version=self.version,
+                path=str(resource.filename),
+            )
+            out.append(attrs)
+        return out
+
+    def read(self, resource: H5Reader, **kwargs) -> dc.BaseSpool:
+        """
+        Read patches.
+
+        Honors dimension-range kwargs.
+
+        :param resource: An open, readable COMDAS HDF5 file.
+        :type resource: h5py.File
+        :param kwargs: Optional dimension-range constraints (e.g.
+            ``time=(t1, t2)``); any key not matching a coord's dim
+            name is ignored.
+        :returns: A spool of the matching patches.
+        :rtype: dascore.BaseSpool
+        """
+        patches = []
+        for name in _patch_group_names(resource):
+            grp = resource["patches"][name]
+            attrs = dc.PatchAttrs.model_validate_json(grp.attrs["attrs_json"])
+            if _patch_matches_kwargs(attrs, kwargs):
+                patches.append(_read_patch(grp))
+        return dc.spool(patches)
+
+    def write(
+        self,
+        spool,
+        resource: H5Writer,
+        *,
+        codec: Codec | None = None,
+        **encode_kwargs,
+    ) -> None:
+        """
+        Compress and write a Patch/spool of Patches to the container.
+
+        :param spool: A single Patch, or any iterable of Patches, to
+            write.
+        :type spool: dascore.Patch or dascore.BaseSpool or Sequence[dascore.Patch]
+        :param resource: An open, writable HDF5 file (auto-opened by
+            DASCore from a path via the ``H5Writer`` type hint, in
+            append mode).
+        :type resource: h5py.File
+        :param codec: The codec to compress every patch in ``spool``
+            with. (e.g. ``SVDCodec(rank=20)``.)
+        :type codec: Codec
+        :param encode_kwargs: Forwarded to ``codec.encode`` for every
+            patch.
+        :raises ValueError: If ``codec`` is not given.
+        """
+        if codec is None:
+            raise ValueError(
+                "A `codec` must be provided to write a COMDAS container "
+                "(e.g. codec=SVDCodec(rank=20))."
+            )
+        resource.attrs["__format__"] = FORMAT_NAME
+        resource.attrs[f"__{FORMAT_NAME}_version__"] = FORMAT_VERSION
+
+        if "patches" not in resource:
+            resource.create_group("patches")
+        patches_group = resource["patches"]
+
+        patches = [spool] if isinstance(spool, dc.Patch) else list(spool)
+        idx = _next_patch_index(resource)
+        for patch in patches:
+            name = f"{_PATCH_GROUP_PREFIX}{idx:04d}"
+            _write_patch(patches_group, name, patch, codec, encode_kwargs)
+            idx += 1
+
+
+def write_compressed(source, path, codec: Codec, **encode_kwargs) -> None:
+    """
+    Compress ``source`` and write it to ``path`` as a COMDAS container.
+
+    This is the intended way to turn uncompressed DAS data
+    into a compressed COMDAS file.
+
+    :param source: Where the uncompressed data comes from (a file or
+        directory path, a single Patch, a sequence of Patches, or an
+        existing Spool).
+    :type source: str or pathlib.Path or dascore.Patch or dascore.BaseSpool or Sequence[dascore.Patch]
+    :param path: Destination path for the COMDAS container. If it
+        already exists as a COMDAS file, patches are appended rather
+        than overwriting existing ones.
+    :type path: str or pathlib.Path
+    :param codec: The codec to compress every patch with, e.g.
+        ``SVDCodec(rank=20)``.
+    :type codec: Codec
+    :param encode_kwargs: Forwarded to ``codec.encode`` for every
+        patch (e.g. ``rank=``/``energy=`` for
+        :class:`~comdas.codecs.svd.SVDCodec`, overriding the codec's
+        own defaults for this call).
+
+    :Example:
+
+    .. code-block:: python
+
+        import dascore as dc
+        from comdas import write_compressed, SVDCodec
+
+        # from an existing uncompressed file
+        write_compressed("raw.h5", "compressed.h5", SVDCodec(rank=20))
+
+        # from a patch already in memory
+        patch = dc.get_example_patch()
+        write_compressed(patch, "compressed.h5", SVDCodec(rank=20))
+
+        # reading back needs nothing ComDAS-specific
+        read_back = dc.spool("compressed.h5")
+    """
+    spool = dc.spool(source)
+    dc.write(
+        spool,
+        path,
+        file_format=FORMAT_NAME,
+        file_version=FORMAT_VERSION,
+        codec=codec,
+        **encode_kwargs,
+    )

comdas/utils.py

@@ -0,0 +1,35 @@
+"""
+Utility functions for working with compressed patches.
+Includes:
+- :func:`compress_patch` Compress an in-memory DASCore patch's data using a specified codec.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from comdas.arrays.duck_array import DuckArray
+from comdas.codecs.base import Codec
+
+
+def compress_patch(patch, codec: Codec, **encode_kwargs):
+    """
+    Return a new ``dascore.Patch`` whose data is compressed with ``codec``.
+
+    Coords and attrs are unchanged. The returned Patch is an ordinary Patch
+    (via ``patch.new``) with the internal data replaced by a
+    :class:`~comdas.arrays.duck_array.DuckArray`.
+
+    :param patch: The uncompressed source patch.
+    :type patch: dascore.Patch
+    :param codec: The codec to compress ``patch.data`` with.
+    :type codec: Codec
+    :param encode_kwargs: Forwarded verbatim to ``codec.encode``.
+    :returns: A new Patch with the same coords/attrs, backed by a
+        :class:`~comdas.arrays.duck_array.DuckArray`.
+    :rtype: dascore.Patch
+    """
+    dense = np.asarray(patch.data)
+    payload = codec.encode(dense, **encode_kwargs)
+    compressed_array = DuckArray(payload, codec)
+    return patch.new(data=compressed_array)

comdas-0.1.0.dist-info/METADATA

@@ -0,0 +1,69 @@
+Metadata-Version: 2.4
+Name: ComDAS
+Version: 0.1.0
+Summary: DASCORE companion package that adds support for compressed DAS data
+Author-email: Byron-Selvage <bselvage@mines.edu>
+License: LGPL-3.0-or-later
+Keywords: dascore,compression,das,dasdae
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: dascore>=0.1.0
+Requires-Dist: h5py
+Requires-Dist: PyWavelets
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: furo; extra == "dev"
+Requires-Dist: sphinx-autodoc-typehints; extra == "dev"
+Requires-Dist: myst-nb; extra == "dev"
+Requires-Dist: matplotlib; extra == "dev"
+Requires-Dist: ipykernel; extra == "dev"
+Dynamic: license-file
+
+# ComDAS
+
+![Python versions tested](https://img.shields.io/badge/Python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue)
+
+ComDAS (Compressed DAS) lets you store and work with compressed distributed acoustic sensing (DAS) data using ordinary [DASCore](https://dascore.org/) tools.
+
+With ComDAS you can:
+
+- **Shrink DAS data in memory.** Compress a DASCore patch and keep using it like any other.
+- **Save compressed files.** Write patches, spools, or whole directories of raw data to a compressed HDF5 file or files.
+- **Read compressed files with plain DASCore.** Once ComDAS is installed, `dascore.spool()` opens compressed files directly without needing to know how the data was compressed.
+- **Choose your compression level.** Pick a codec and set how much detail to keep, from near-lossless to aggressive compression.
+- **Implement your own algorithm.** Add a new compression method by writing a codec class.
+
+> [!NOTE]
+> ComDAS is in active development. The file format and public API may change between releases.
+
+## Supported codecs
+
+| Codec | Main setting | Reference |
+| --- | --- | --- |
+| [SVD](https://byron-selvage.github.io/ComDAS/codecs/svd.html) | `rank` or retained `energy` | [NumPy SVD](https://numpy.org/doc/1.22/reference/generated/numpy.linalg.svd.html) |
+| [Wavelet](https://byron-selvage.github.io/ComDAS/codecs/wavelet.html) | `keep_fraction` or `threshold` | [PyWavelets](https://doi.org/10.21105/joss.01237); [DeVore et al. (1992)](https://doi.org/10.1109/18.119733) |
+
+## Installation
+
+ComDAS requires Python 3.10 or newer.
+
+```bash
+pip install comdas
+```
+
+## Documentation
+
+Full documentation is available at [byron-selvage.github.io/ComDAS](https://byron-selvage.github.io/ComDAS/), including:
+
+- [Quick start](https://byron-selvage.github.io/ComDAS/quickstart.html): compress a patch, save a compressed file, and read it back
+- [Codecs](https://byron-selvage.github.io/ComDAS/codecs/svd.html): configuration options and examples for each codec
+- [Examples](https://byron-selvage.github.io/ComDAS/examples/compression_comparison.html)
+- [API reference](https://byron-selvage.github.io/ComDAS/api/index.html)
+
+## License
+
+ComDAS is licensed under the [GNU Lesser General Public License](LICENSE).

comdas-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+comdas/__init__.py,sha256=R7M9IFbD5BdOqQGCU1onPjTQNSDMn5tNV8t9kJHlQno,1187
+comdas/utils.py,sha256=A-aeA8VlYB9AmyXnqG1SUOCdmV4PFXjfXgSQRpJvcR4,1203
+comdas/arrays/__init__.py,sha256=X1vsUKtcblCQVpi047nGtXsOW556b4nbacH1bNJ9DnQ,221
+comdas/arrays/duck_array.py,sha256=3BN-xESymPtnEtOSTC2UszUtMbDf_-swLMG9cFSIlnU,10775
+comdas/codecs/__init__.py,sha256=baaEoxKe09LuZ2El_wnqhFxRa7RUs-vhgTlz-DN2v7U,356
+comdas/codecs/base.py,sha256=hNSMc7AqItonSNWp7TVhEwBgCX3rBvd2A2aiub3F5cc,9712
+comdas/codecs/svd.py,sha256=RRWhBs29ELiDEdkR-KYMnBwvw_uB4LZHn6nGnGa5dA8,8656
+comdas/codecs/wavelet.py,sha256=-GjauCly_-dqqxSjBW9JTR_GQterzCc2l-vPgImDPAg,9719
+comdas/io/__init__.py,sha256=aoweeIH-QebldKGyZXNLPn4aQFhZ2CMg627kOSE_ywU,564
+comdas/io/h5_container.py,sha256=Xi5WnNKg9nJrFAhjRkq5GqdDXaba-6OyqrUAiqkXQig,15272
+comdas-0.1.0.dist-info/licenses/LICENSE,sha256=pWgb-bBdsU2Gd2kwAXxketnm5W_2u8_fIeWEgojfrxs,7651
+comdas-0.1.0.dist-info/METADATA,sha256=jIKvsK5Rl9GDFfX9TXa0AqZogoYOLUJQnoZV3eZdwow,3042
+comdas-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+comdas-0.1.0.dist-info/entry_points.txt,sha256=dHkx7dydw-xH3uJ4SYKbhF6EvxjHGFxrh99I5_DV8xY,64
+comdas-0.1.0.dist-info/top_level.txt,sha256=zkMksyvy9YtJW3mt1ZseiS55h0NbPT84Sbt4B2KrRtE,7
+comdas-0.1.0.dist-info/RECORD,,

comdas-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

comdas-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[dascore.fiber_io]
+COMDAS__V1 = comdas.io.h5_container:ComdasV1