defair-data 0.4.0rc2__py3-none-any.whl

defair_data/__init__.py

@@ -0,0 +1,113 @@
+#
+# Copyright 2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import atexit
+import os
+import sys
+
+# CRITICAL: Must be set before ANY module imports zarr
+# Zarr checks this at import time, and xarray imports zarr
+os.environ.setdefault("ZARR_V3_EXPERIMENTAL_API", "1")
+
+# Silence noisy S3/HTTP dependency loggers BEFORE any defair / defair_data
+# submodule is imported, so the first boto3/botocore record (e.g. credential
+# discovery on Dataset.from_source) never reaches pytest's caplog or any
+# user-attached root handler. Required for consumers that never invoke
+# ``defair.logging.setup_logging`` — e.g. the vendor pytest suite.
+from defair_data import _logging_defaults as _logging_defaults  # noqa: I001  side effect: silence noisy loggers
+from importlib.metadata import version
+
+from defair.plugin_manager import load_reader
+from defair_data.constants import CF_VERSION
+from defair_data.core import Dataset
+from defair_data.dask_manager import S3CredentialsPlugin, propagate_s3_credentials
+from defair_data.data_reader_plugin import DataReaderPlugin
+from defair_data.logging import get_logger
+from defair_data.readers import (
+    ReaderInfo,
+    ReaderProduct,
+    describe_readers,
+    find_reader_for_path,
+    get_reader,
+    get_reader_info,
+    list_readers,
+)
+from defair_data.readers.msg15.msg15_native_reader_plugin import MSG15NativeReaderPlugin
+
+__version__ = version("defair-data")
+
+logger = get_logger(__name__)
+
+# The netCDF4 wheel bundles its own libhdf5 build and h5py bundles a different one. In a
+# single process the library imported first binds the shared symbols, so when netCDF4 wins
+# every HDF5-backed reader here fails with "Unspecified error in H5DSget_num_scales" and the
+# interpreter can dump core at shutdown -- a failure whose message names neither library.
+# defair-data no longer depends on netCDF4, but earthkit-data still installs it, and
+# "import netCDF4" is a common first line in an Earth-observation notebook. Saying so at the
+# only moment the clash is detectable is the whole defence a library has here.
+if sys.modules.get("netCDF4") is not None:
+    logger.warning(
+        "netCDF4 was imported before defair_data. The two bundle different libhdf5 builds and the "
+        "first import wins the shared symbols, so HDF5-backed readers (MTG FCI, MTG LI, Sentinel-3 "
+        "and others) may fail with 'Unspecified error in H5DSget_num_scales' and the process may "
+        "crash at exit. Import defair_data before netCDF4, or keep netCDF4 out of this process."
+    )
+
+
+def _close_native_handles_before_teardown() -> None:
+    """Release HDF5 and projection handles while their libraries are still alive.
+
+    xarray deliberately stops closing cached files at exit, so an h5py handle left in
+    a CachingFileManager is finalised during interpreter teardown, after libhdf5 (and,
+    in environments where pyproj and fiona each bundle a libproj, after one of those)
+    has already been torn down. A plain FCI read then exits with SIGSEGV after printing
+    a correct result. Registered here, after xarray and the HDF5 stack were imported,
+    this runs before their own exit hooks and leaves nothing for teardown to finalise.
+    """
+    import gc  # noqa: PLC0415
+
+    from xarray.backends.file_manager import FILE_CACHE  # noqa: PLC0415
+
+    from defair_data.readers.mtg_fci.mtg_fci_l1c_nc_reader_plugin import release_retired_managers  # noqa: PLC0415
+
+    try:
+        release_retired_managers(force=True)
+        FILE_CACHE.clear()
+    except Exception:  # best effort: an exit hook must never raise
+        logger.debug("exit cleanup of native handles failed", exc_info=True)
+    gc.collect()
+
+
+atexit.register(_close_native_handles_before_teardown)
+
+__all__ = [
+    "CF_VERSION",
+    "DataReaderPlugin",
+    "Dataset",
+    "MSG15NativeReaderPlugin",
+    "ReaderInfo",
+    "ReaderProduct",
+    "S3CredentialsPlugin",
+    "__version__",
+    "describe_readers",
+    "find_reader_for_path",
+    "get_logger",
+    "get_reader",
+    "get_reader_info",
+    "list_readers",
+    "load_reader",
+    "propagate_s3_credentials",
+]

defair_data/_grid_split.py

@@ -0,0 +1,457 @@
+#
+# Copyright 2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Helpers for presenting one multi-grid dataset as one dataset per grid.
+
+A reader that puts two sampling grids in a single :class:`xarray.Dataset` has to
+disambiguate the second grid's spatial names, because ``x``/``y``/``lat``/``lon``
+are already taken by the first: MSG SEVIRI emits ``x_hrv``/``y_hrv``/``lat_hrv``/
+``lon_hrv`` for HRV, MTG FCI emits ``x_1km``/``x_2km``/``x_500m``. Every consumer
+then has to know those product-specific spellings.
+
+Once a grid stands alone there is nothing left to collide with, so the suffix has
+no reason to survive the split. These helpers strip it and repair the CF metadata
+that named the old spelling, which is what lets a split grid be an ordinary
+single-grid dataset that needs no product-specific handling downstream.
+
+The georeferencing needs the same treatment for the same reason: a multi-grid
+dataset stores one affine describing one of its grids, so isolating a grid has to
+re-aim it as well. That part is used by every single-grid selection, not only by
+the split.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from os.path import commonprefix
+from typing import Any
+
+import xarray as xr
+
+from defair_data.cdm import CommonDataModel, GriddedModel, SpatialModel
+from defair_data.logging import get_logger
+
+logger = get_logger(__name__)
+
+# CF attributes whose value is purely a whitespace-separated list of variable
+# names, so rewriting them is a token substitution. `grid_mapping` is deliberately
+# absent: CF-1.13 §5.6 also allows the expanded `crs: x y` form, whose colon-tagged
+# mapping variable is not a plain name, and a split never renames the mapping
+# variable anyway.
+_CF_NAME_LIST_ATTRS = ("coordinates", "ancillary_variables", "bounds")
+
+# CF attributes that tag each variable with the role it fills, so only every
+# second word names a variable: `cell_measures` (CF-1.13 §7.2, `area: cell_area`)
+# and `formula_terms` (§4.3.3, `a: a b: b ps: ps`).
+_CF_PAIR_ATTRS = ("cell_measures", "formula_terms")
+
+# The name a role takes once its grid stands alone. A suffix is only stripped when
+# every renamed name lands on its role's canonical spelling, so a semantically
+# meaningful suffix (``along_track``/``across_track`` share ``_track``) is never
+# mistaken for a grid discriminator.
+_CANONICAL_BY_ROLE = {
+    "x_dim": frozenset({"x"}),
+    "y_dim": frozenset({"y"}),
+    "x_coord": frozenset({"x"}),
+    "y_coord": frozenset({"y"}),
+    "longitude": frozenset({"lon", "longitude"}),
+    "latitude": frozenset({"lat", "latitude"}),
+}
+
+
+def grid_mapping_references(value: str) -> list[str]:
+    """Return every variable a CF ``grid_mapping`` attribute refers to.
+
+    CF-1.13 §5.6 allows either a bare mapping-variable name or expanded clauses in
+    which a word ending in a colon names the mapping variable and the words after
+    it name the coordinates it applies to. Both the mapping variables and those
+    coordinates are real references, so a reference walk has to see all of them --
+    reading the raw words would look for a variable called ``crs:``.
+    """
+    if ":" not in value:
+        return value.split()
+    names: list[str] = []
+    for token in value.split():
+        head, sep, tail = token.partition(":")
+        names.append(head if sep else token)
+        if sep and tail:
+            names.append(tail)
+    return names
+
+
+def pair_tagged_references(value: str) -> list[str]:
+    """Return the variable names in a CF ``term: variable`` attribute.
+
+    ``cell_measures`` and ``formula_terms`` pair a role tag with the variable
+    filling it, so reading every word would look for a variable called ``ps:``
+    and treat the tags as references.
+    """
+    return value.split()[1::2]
+
+
+def rename_grid_mapping(value: str, renames: dict[str, str]) -> str:
+    """Rewrite a CF ``grid_mapping`` value, renaming only the coordinates in it.
+
+    The mapping variable keeps its name and its colon; only the coordinate names a
+    clause applies to are renamed. A bare mapping name has no coordinates in it and
+    is returned unchanged.
+    """
+    if ":" not in value:
+        return value
+    out: list[str] = []
+    for token in value.split():
+        head, sep, tail = token.partition(":")
+        if not sep:
+            out.append(renames.get(token, token))
+        elif tail:
+            out.append(f"{head}:{renames.get(tail, tail)}")
+        else:
+            out.append(f"{head}:")
+    return " ".join(out)
+
+
+def _common_suffix(left: str, right: str) -> str:
+    """Return the longest common suffix of two names."""
+    return commonprefix([left[::-1], right[::-1]])[::-1]
+
+
+def _spatial_names(group: GriddedModel) -> dict[str, str]:
+    """Map a regular grid's spatial-role slots to the names it declares.
+
+    Only regular grids are handled. A curvilinear or point group's coordinates are
+    not interchangeable with a canonical ``x``/``y`` axis pair, so there is no
+    role-preserving rename to make for them.
+    """
+    names = {
+        "x_dim": group.x_dim,
+        "y_dim": group.y_dim,
+        "x_coord": group.x_coord,
+        "y_coord": group.y_coord,
+    }
+    geo = group.geostationary
+    spec = group.latlon.spec
+    # A grid read projection-only names its lat/lon pair in the spec rather than in
+    # the GEO parameters, and that latent name has to be canonicalised too, or a
+    # materialisation after the split would reintroduce the suffix.
+    names["latitude"] = (geo.latitude if geo is not None else None) or (spec.lat_name if spec else "")
+    names["longitude"] = (geo.longitude if geo is not None else None) or (spec.lon_name if spec else "")
+    return {role: name for role, name in names.items() if name}
+
+
+def canonical_renames(ds: xr.Dataset, group: SpatialModel) -> dict[str, str]:
+    """Return the {old: new} renames that drop *group*'s disambiguating suffix.
+
+    The suffix comes from the two spatial dimension names, since those are what
+    actually collide when several grids share one dataset. It is only accepted once
+    every name it would rename lands on the canonical spelling for that name's role
+    (``x_hrv`` -> ``x``, ``lat_1km`` -> ``lat``). That check is what keeps a
+    meaningful suffix from being stripped: ``along_track``/``across_track`` share
+    ``_track``, but ``along`` is not a canonical x-axis name, so nothing is renamed.
+
+    Returns an empty mapping when there is no suffix to strip, when a renamed name
+    would not be canonical, or when stripping would collide with a name already in
+    use — a partial rename must never silently shadow an existing variable.
+    """
+    if not isinstance(group, GriddedModel):
+        return {}
+    names = _spatial_names(group)
+    x_dim, y_dim = names.get("x_dim", ""), names.get("y_dim", "")
+    if not (x_dim and y_dim):
+        return {}
+
+    suffix = _common_suffix(x_dim, y_dim)
+    if not suffix.startswith("_"):
+        return {}
+
+    renames: dict[str, str] = {}
+    for role, name in names.items():
+        if not name.endswith(suffix):
+            continue  # a role spelled without the suffix keeps its own name
+        stripped = name[: -len(suffix)]
+        if stripped not in _CANONICAL_BY_ROLE.get(role, frozenset()):
+            logger.debug(
+                "Not stripping the '{}' suffix from group '{}': '{}' would become '{}', "
+                "which is not a canonical {} name, so the suffix is meaningful.",
+                suffix,
+                group.id,
+                name,
+                stripped,
+                role,
+            )
+            return {}
+        renames[name] = stripped
+    if not renames:
+        return {}
+
+    taken = (set(ds.variables) | set(ds.sizes)) - set(renames)
+    collisions = sorted(new for new in renames.values() if new in taken)
+    if collisions or len(set(renames.values())) != len(renames):
+        logger.warning(
+            "Not stripping the '{}' suffix from group '{}': target name(s) {} already in use.",
+            suffix,
+            group.id,
+            collisions or sorted(renames.values()),
+        )
+        return {}
+    return renames
+
+
+def _cf_stores(var: xr.Variable) -> tuple[dict[str, Any], dict[str, Any]]:
+    """Both places a CF reference lives on *var*: its attrs and its encoding.
+
+    ``xarray`` moves ``coordinates`` and ``grid_mapping`` out of ``attrs`` into
+    ``encoding`` when it decodes a file, and writes them back out of ``encoding``
+    on save. A repair that touches ``attrs`` alone is therefore inert for any
+    dataset that has been through disk, and the stale names it left behind reach
+    the next file: the Zarr writer copies the encoded ``grid_mapping`` back into
+    attrs, and a plain ``to_netcdf`` writes the encoded ``coordinates`` verbatim.
+    """
+    return (var.attrs, var.encoding)
+
+
+def cf_reference_value(var: xr.Variable, key: str) -> str:
+    """The live value of one CF reference attribute on *var*, reading both stores.
+
+    A decoded dataset carries its references only in ``encoding``; an in-memory
+    one only in ``attrs``. When both stores carry *key* they can disagree (an
+    ``encoding`` leftover from an earlier decode goes stale the moment attrs are
+    edited), so attrs wins: it is the live user-visible store, and a union of
+    the two would follow the stale name as if it were a real reference.
+    """
+    for store in _cf_stores(var):
+        if key in store:
+            return str(store[key])
+    return ""
+
+
+def rename_cf_references(ds: xr.Dataset, renames: dict[str, str]) -> None:
+    """Rewrite CF reference attributes in place to follow *renames*.
+
+    ``xarray.Dataset.rename`` moves variables but leaves attributes that quote the
+    old names, so a renamed ``lat_hrv`` would still be advertised as ``lat_hrv`` by
+    every ``coordinates`` attribute pointing at it.
+
+    ``grid_mapping`` is rewritten under its own grammar (CF-1.13 §5.6): a
+    ``<variable>:`` tag introduces the coordinates it maps and is not itself a
+    coordinate, so it is preserved while the names after it are renamed.
+    """
+    for var in ds.variables.values():
+        for store in _cf_stores(var):
+            for attr in _CF_NAME_LIST_ATTRS:
+                value = store.get(attr)
+                if isinstance(value, str) and value:
+                    store[attr] = " ".join(renames.get(tok, tok) for tok in value.split())
+            mapping = store.get("grid_mapping")
+            if isinstance(mapping, str) and mapping:
+                store["grid_mapping"] = rename_grid_mapping(mapping, renames)
+
+
+def _prune_dangling(store: dict[str, Any], known: set[str]) -> None:
+    """Drop the entries of one attrs/encoding mapping that name a missing variable.
+
+    ``grid_mapping`` is deliberately not pruned: the mapping container is kept by
+    the reference walk that precedes this, so a dangling one would mean the walk
+    is broken, and silently deleting the link would hide that.
+    """
+    for attr in (*_CF_NAME_LIST_ATTRS, *_CF_PAIR_ATTRS):
+        value = store.get(attr)
+        if not isinstance(value, str) or not value:
+            continue
+        tokens = value.split()
+        if attr in _CF_PAIR_ATTRS:
+            if len(tokens) % 2:
+                logger.warning("CF '{}' attribute has an unpaired trailing token: {!r}", attr, value)
+            pairs = list(zip(tokens[::2], tokens[1::2], strict=False))
+            kept_pairs = [(tag, name) for tag, name in pairs if name in known]
+            # A formula is computed from all of its terms at once, so a partial one
+            # is not a weaker formula but an uncomputable one: drop it whole. Cell
+            # measures are independent of each other, so surviving ones still stand.
+            if attr == "formula_terms" and len(kept_pairs) != len(pairs):
+                kept_pairs = []
+            kept = " ".join(f"{tag} {name}" for tag, name in kept_pairs)
+        else:
+            kept = " ".join(tok for tok in tokens if tok in known)
+        if kept == value:
+            continue
+        if kept:
+            store[attr] = kept
+        else:
+            del store[attr]
+
+
+def drop_dangling_cf_references(ds: xr.Dataset) -> None:
+    """Drop CF references that no longer resolve in *ds*, in place.
+
+    Isolating one grid leaves the shared grid-mapping container advertising the
+    other grid's ``coordinates``. Leaving that dangling reference in place breaks
+    metadata-reference closure for any consumer that resolves it: CF-1.13's §5
+    conformance requirement that every name in a ``coordinates`` attribute exist in
+    the file applies wherever the attribute sits, so this is conformance rather
+    than tidiness. The container is left without the attribute rather than
+    repointed at the surviving grid, because CF Appendix A admits ``coordinates``
+    on data, geometry-container and domain variables only; the association between
+    a mapping and its coordinates belongs on the data variable, which the split
+    repairs directly.
+
+    Name lists lose the entries that stopped resolving; a ``term: variable``
+    attribute loses whole pairs, since a tag without its variable is not a
+    reference either.
+    """
+    known = {str(name) for name in ds.variables}
+    for var in ds.variables.values():
+        for store in _cf_stores(var):
+            _prune_dangling(store, known)
+
+
+def drop_container_coordinates(ds: xr.Dataset) -> None:
+    """Remove ``coordinates`` from every grid-mapping container in *ds*, in place.
+
+    CF Appendix A admits ``coordinates`` on data, geometry-container and domain
+    variables; a grid-mapping variable is none of those, and CF-1.13 §5.6 makes it
+    a container for the mapping's parameters, associating it with coordinates
+    through the expanded ``grid_mapping`` syntax on the data variable instead. Our
+    readers write one anyway, and one container serves every grid in a multi-grid
+    dataset, so it can only ever name one of them.
+
+    Isolating a grid would otherwise leave that attribute naming the dropped grid,
+    or surviving on whichever grid it happened to name and not on its sibling.
+    Removing it is what makes the split's containers consistent, and loses nothing:
+    the data variables' own references carry the association and are repaired here.
+    ``grid_mapping_name`` is the one attribute CF requires on a container, so it is
+    how a container is recognised.
+    """
+    for var in ds.variables.values():
+        if "grid_mapping_name" not in var.attrs:
+            continue
+        for store in _cf_stores(var):
+            store.pop("coordinates", None)
+
+
+def _gdal_geotransform(ds: xr.Dataset, group: SpatialModel) -> str | None:
+    """Return *group*'s own affine as a GDAL ``GeoTransform`` string, if it has one.
+
+    GDAL orders the six terms ``(origin_x, x_res, 0, origin_y, 0, y_res)`` and
+    anchors the origin on the outer cell edge, half a pixel beyond the first
+    centre. Two centres are needed to measure a pixel, so a grid only one cell
+    wide or tall cannot state its own resolution and gets no affine.
+    """
+    if not isinstance(group, GriddedModel):
+        return None
+    x_name, y_name = group.x_coord, group.y_coord
+    if x_name not in ds.variables or y_name not in ds.variables:
+        return None
+    x, y = ds[x_name], ds[y_name]
+    if x.ndim != 1 or y.ndim != 1 or x.size < 2 or y.size < 2:
+        return None
+    x_res, y_res = float(x[1] - x[0]), float(y[1] - y[0])
+    return f"{float(x[0]) - x_res / 2} {x_res} 0.0 {float(y[0]) - y_res / 2} 0.0 {y_res}"
+
+
+def retarget_geotransform(ds: xr.Dataset, group: SpatialModel) -> None:
+    """Re-aim a cached ``GeoTransform`` at *group*'s own axes, in place.
+
+    A multi-grid dataset carries at most one affine, written for whichever grid the
+    reader georeferenced first. Consumers that prefer a stored affine to
+    recomputing one, because the stored value is byte-exact across a round trip
+    through disk, then read that grid's pixel size out of every other grid: our own
+    ``ReferenceGrid`` does exactly that, and so does any GDAL reader of a file we
+    write. Recomputing keeps such a consumer working, and lands on the identical
+    string for the grid the affine already described.
+
+    A grid too small to state its own resolution has the stale value removed rather
+    than replaced, since deriving the affine from its coordinates is then the
+    consumer's only honest option.
+    """
+    stale = [name for name, var in ds.variables.items() if "GeoTransform" in var.attrs]
+    if not stale:
+        return
+    geotransform = _gdal_geotransform(ds, group)
+    for name in stale:
+        if geotransform is None:
+            del ds[name].attrs["GeoTransform"]
+        else:
+            ds[name].attrs["GeoTransform"] = geotransform
+    if geotransform is None:
+        logger.warning(
+            "Group '{}' is too small to state its own affine, so the cached GeoTransform on {} was "
+            "dropped rather than re-aimed; consumers must derive the transform from the coordinates.",
+            group.id,
+            sorted(str(name) for name in stale),
+        )
+
+
+def rename_group(group: SpatialModel, renames: dict[str, str]) -> SpatialModel:
+    """Return *group* with every declared name and dim following *renames*.
+
+    Only the dims are rewritten for a non-regular group: ``canonical_renames``
+    produces renames for regular grids alone, so a sibling point or curvilinear
+    group carried alongside one keeps its own coordinate names.
+    """
+    dims = tuple(renames.get(d, d) for d in group.dims)
+
+    def _renamed(name: str | None) -> str | None:
+        return renames.get(name, name) if name else name
+
+    if isinstance(group, GriddedModel):
+        geostationary = group.geostationary
+        if geostationary is not None:
+            geostationary = replace(
+                geostationary,
+                latitude=_renamed(geostationary.latitude),
+                longitude=_renamed(geostationary.longitude),
+            )
+        # The lat/lon spec names the pair this grid would carry, whether it is
+        # attached yet or not, so it follows the rename for the same reason the
+        # attached names do: a materialisation after the split must land on the
+        # canonical spelling, not resurrect the suffix the split just removed.
+        latlon = group.latlon
+        if latlon.spec is not None:
+            latlon = replace(
+                latlon,
+                spec=replace(
+                    latlon.spec,
+                    lat_name=renames.get(latlon.spec.lat_name, latlon.spec.lat_name),
+                    lon_name=renames.get(latlon.spec.lon_name, latlon.spec.lon_name),
+                ),
+            )
+        return replace(
+            group,
+            dims=dims,
+            x_dim=renames.get(group.x_dim, group.x_dim),
+            y_dim=renames.get(group.y_dim, group.y_dim),
+            x_coord=renames.get(group.x_coord, group.x_coord),
+            y_coord=renames.get(group.y_coord, group.y_coord),
+            geostationary=geostationary,
+            latlon=latlon,
+        )
+    return replace(group, dims=dims)
+
+
+def rename_cdm(cdm: CommonDataModel, renames: dict[str, str]) -> CommonDataModel:
+    """Return *cdm* with every referenced name following *renames*."""
+    temporal_groups = tuple(
+        replace(
+            group,
+            variables=tuple(renames.get(variable, variable) for variable in group.variables),
+            time_coord=renames.get(group.time_coord, group.time_coord),
+            sample_dim=renames.get(group.sample_dim, group.sample_dim) if group.sample_dim is not None else None,
+        )
+        for group in cdm.temporal_groups
+    )
+    return replace(
+        cdm,
+        spatial_groups=tuple(rename_group(group, renames) for group in cdm.spatial_groups),
+        temporal_groups=temporal_groups,
+    )

defair_data/_logging_defaults.py

@@ -0,0 +1,63 @@
+#
+# Copyright 2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Import-time defaults for noisy third-party loggers.
+
+Called from ``defair_data/__init__.py`` so the suppression is in place for any
+consumer that imports the package — CLI, notebooks, ad-hoc scripts, and
+externally-rooted pytest suites that never reach our ``setup_logging()`` fixture.
+
+Politeness: a level is only installed when the logger is still at ``NOTSET`` —
+so a caller who explicitly opted in (e.g. ``logging.getLogger("botocore").setLevel(INFO)``)
+before importing DEFAIR is respected.  ``setup_logging()`` keeps stricter
+"always set" semantics for the explicit configuration path.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+# Third-party loggers whose INFO chatter pollutes DEFAIR output (and pytest
+# captured-log sections under ``log_level = "INFO"``).  Re-exported as the
+# single source of truth for ``defair.logging.setup_logging``.
+NOISY_LOGGERS: tuple[str, ...] = ("botocore", "boto3", "urllib3", "s3transfer", "aiobotocore")
+
+
+def _noisy_floor_from_env() -> int:
+    """Resolve the noisy-logger floor from ``DEFAIR_LOG_LEVEL`` (default WARNING)."""
+    if os.environ.get("DEFAIR_LOG_LEVEL", "INFO").upper() in ("DEBUG", "TRACE"):
+        return logging.DEBUG
+    return logging.WARNING
+
+
+def configure_noisy_dependency_loggers() -> None:
+    """Install a sensible default level on noisy third-party loggers.
+
+    Only adjusts a logger that is still at ``NOTSET``; a deliberately-set level
+    is left untouched.  Safe to call multiple times and from any import path.
+    """
+    floor = _noisy_floor_from_env()
+    for name in NOISY_LOGGERS:
+        logger_obj = logging.getLogger(name)
+        if logger_obj.level == logging.NOTSET:
+            logger_obj.setLevel(floor)
+
+
+# Apply at module import so ``import defair_data`` is sufficient — even for
+# consumers that never call ``defair.logging.setup_logging`` (vendor pytest
+# suite, notebooks, third-party scripts).
+configure_noisy_dependency_loggers()

defair_data/_path_utils.py

@@ -0,0 +1,54 @@
+#
+# Copyright 2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Path utilities for glob pattern detection and manipulation."""
+
+from os import PathLike
+
+_GLOB_CHARS = {"*", "?", "[", "]"}
+
+
+def is_glob_pattern(path: str | PathLike) -> bool:
+    """Check if path contains glob wildcards.
+
+    Args:
+        path: Path string to check
+
+    Returns:
+        True if path contains glob wildcard characters (*, ?, [, ])
+    """
+    return any(c in str(path) for c in _GLOB_CHARS)
+
+
+def join_uri(base: str | PathLike, *parts: str) -> str:
+    """Join a base URI or filesystem path with one or more child segments."""
+    out = str(base).rstrip("/")
+
+    for part in parts:
+        if part is None:
+            continue
+
+        s = str(part).strip("/")
+
+        # Skip empty or "." literals
+        if not s or s == ".":
+            continue
+
+        if ".." in s.split("/"):
+            raise ValueError(f"join_uri rejects '..' segments: {part!r}")
+
+        out = f"{out}/{s}"
+    return out