deepcontrail-core 0.1.0__py3-none-any.whl

deepcontrail/core/__init__.py

@@ -0,0 +1,49 @@
+"""Shared, torch-free foundations: dataset format, config, composites, splits, metrics."""
+
+from importlib.metadata import version
+
+from deepcontrail.core.config import DatasetConfig
+from deepcontrail.core.contract import (
+    BandChannel,
+    BandManifest,
+    DatasetContractError,
+    DatasetInfo,
+    read_band_manifest,
+    read_dataset_info,
+)
+from deepcontrail.core.false_color import (
+    FalseColorChannel,
+    FalseColorConfig,
+    check_false_color,
+    false_color_from_spectral,
+    load_spectral_false_color,
+)
+from deepcontrail.core.metrics import dice_coefficient, iou_score
+from deepcontrail.core.splits import context_frames, partition_stems, split_fingerprint
+from deepcontrail.core.validate import Issue, ValidationReport, validate_dataset
+
+__version__ = version("deepcontrail-core")
+
+__all__ = [
+    "BandChannel",
+    "BandManifest",
+    "DatasetConfig",
+    "DatasetContractError",
+    "DatasetInfo",
+    "FalseColorChannel",
+    "FalseColorConfig",
+    "Issue",
+    "ValidationReport",
+    "__version__",
+    "check_false_color",
+    "context_frames",
+    "dice_coefficient",
+    "false_color_from_spectral",
+    "iou_score",
+    "load_spectral_false_color",
+    "partition_stems",
+    "read_band_manifest",
+    "read_dataset_info",
+    "split_fingerprint",
+    "validate_dataset",
+]

deepcontrail/core/config.py

@@ -0,0 +1,89 @@
+"""Pydantic configuration of how scenes are read from a dataset root.
+
+Serializable via ``model_dump()`` → YAML/JSON for experiment logging.
+Uses only pydantic + pyyaml so that ``import deepcontrail.core`` stays torch-free.
+Task packages extend ``DatasetConfig`` with their own loader fields.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Any, Literal, Self
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from deepcontrail.core.false_color import FalseColorConfig  # noqa: TC001 (pydantic field)
+
+
+class DatasetConfig(BaseModel):
+    """Which scenes to read from a dataset root, and how to build their images.
+
+    Parameters
+    ----------
+    root
+        Path to a dataset root in the DeepContrail format (``spectral_bands/``
+        with its ``bands.json``, label directories, ``metadata/``).
+    split
+        ``"train"``, ``"val"``, or ``"test"``.  Both ``"train"`` and ``"val"``
+        read from the ``train/`` directory on disk; the dataset partitions stems
+        deterministically using ``val_fraction``.  ``"test"`` reads from ``test/``.
+    false_color
+        Bands and bounds of the composite built from ``spectral_bands/``, by the
+        band keys of the dataset's ``bands.json``. Required: there is no default.
+    pixel_shift
+        Sub-pixel translation (right and down) applied to images at load time to
+        align them with the masks. Use the dataset's ``recommended_pixel_shift``
+        (``metadata/dataset_info.json``); default 0.0 (no shift).
+    seed
+        RNG seed for the train/val partition and for reproducible augmentation.
+    val_fraction
+        Fraction of training stems held out for validation (seeded by ``seed``).
+        Must be > 0 when ``fold`` is not set.  Ignored when ``fold`` is set
+        (fold partitioning takes over).
+    fold
+        K-fold cross-validation fold index, or ``None`` to disable CV.
+    num_folds
+        Total number of CV folds. Only used when ``fold`` is set.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    root: str
+    split: Literal["train", "val", "test"] = "train"
+    false_color: FalseColorConfig
+    pixel_shift: Annotated[float, Field(ge=0.0)] = 0.0
+    seed: Annotated[int, Field(ge=0)] = 42
+    val_fraction: Annotated[float, Field(default=0.1, ge=0.0, lt=1.0)] = 0.1
+    fold: int | None = None
+    num_folds: Annotated[int, Field(default=4, ge=2)] = 4
+
+    @model_validator(mode="after")
+    def _validate_split_strategy(self) -> Self:
+        """Validate the split configuration.
+
+        - ``fold`` must be in ``[0, num_folds)`` when set.
+        - ``val_fraction`` must be > 0 when ``fold`` is not set.
+        """
+        if self.fold is not None and not (0 <= self.fold < self.num_folds):
+            msg = f"fold {self.fold} out of range [0, {self.num_folds})"
+            raise ValueError(msg)
+        if self.val_fraction <= 0 and self.fold is None:
+            msg = "val_fraction must be > 0 (use fold for CV instead)"
+            raise ValueError(msg)
+        return self
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> Self:
+        """Build a config from a plain dictionary.
+
+        Pydantic handles nested dict → sub-model parsing automatically.
+        """
+        return cls.model_validate(d)
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> Self:
+        """Load a config from the ``data:`` section of a YAML file."""
+        with Path(path).open(encoding="utf-8") as f:
+            raw = yaml.safe_load(f)
+        return cls.model_validate(raw["data"])

deepcontrail/core/contract.py

@@ -0,0 +1,196 @@
+"""DeepContrail dataset format, version 1: the metadata every dataset root carries.
+
+A dataset is any directory that follows the format; the OpenContrails converter is one
+producer. Two JSON files describe it:
+
+- ``spectral_bands/bands.json`` (``BandManifest``): the channel order of every
+  ``spectral_bands/{split}/<id>.npy`` file, with the key configs use for each channel.
+- ``metadata/dataset_info.json`` (``DatasetInfo``): where the data comes from, how many
+  labelled scenes each split holds, which labels exist and the recommended pixel shift.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Literal, Self, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
+
+FORMAT_VERSION = 1
+BAND_MANIFEST_PATH = "spectral_bands/bands.json"
+DATASET_INFO_PATH = "metadata/dataset_info.json"
+
+_Schema = TypeVar("_Schema", bound=BaseModel)
+
+Label = Literal["semantic", "probability", "annotator", "instance"]
+"""A label directory: ``semantic_masks/``, ``probability_masks/``, ``annotator_masks/``,
+``instance_masks/``."""
+
+
+class DatasetContractError(ValueError):
+    """A dataset root does not follow the DeepContrail dataset format."""
+
+
+class BandChannel(BaseModel):
+    """One channel (last axis) of the ``spectral_bands/`` arrays.
+
+    Parameters
+    ----------
+    index
+        0-based position on the last axis.
+    key
+        Name configs use to refer to the channel (``[a-z0-9_]+``).
+    units
+        Physical units, e.g. ``"K"`` for brightness temperatures.
+    sensor_band
+        The sensor's own band ID (``"13"`` for ABI, ``"IR10.5"`` for FCI). Never
+        interpreted by the library; it lets people check a recipe.
+    wavelength_um
+        Central wavelength in µm.
+    name
+        Human-readable description.
+    derived_from
+        ``(a, b)`` when the channel is stored as ``a - b`` of two other channels.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    index: Annotated[int, Field(ge=0)]
+    key: Annotated[str, Field(pattern=r"^[a-z0-9_]+$")]
+    units: str
+    sensor_band: str | None = None
+    wavelength_um: float | None = None
+    name: str | None = None
+    derived_from: tuple[str, str] | None = None
+
+
+class BandManifest(BaseModel):
+    """``spectral_bands/bands.json``: what each channel of the spectral arrays holds."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    format_version: Literal[1]
+    sensor: str
+    dtype: Literal["float32"]
+    channels: Annotated[tuple[BandChannel, ...], Field(min_length=1)]
+    description: str | None = None
+
+    @property
+    def keys(self) -> tuple[str, ...]:
+        """Channel keys in channel order."""
+        return tuple(c.key for c in self.channels)
+
+    def index_of(self, key: str) -> int:
+        """Return the channel index of ``key``.
+
+        Raises
+        ------
+        DatasetContractError
+            If no channel has that key; the message lists the available keys.
+        """
+        try:
+            return self.keys.index(key)
+        except ValueError:
+            msg = f"unknown band key {key!r}; available: {', '.join(self.keys)}"
+            raise DatasetContractError(msg) from None
+
+    @model_validator(mode="after")
+    def _validate_channels(self) -> Self:
+        keys = [c.key for c in self.channels]
+        duplicates = sorted({k for k in keys if keys.count(k) > 1})
+        if duplicates:
+            msg = f"duplicate channel keys: {duplicates}"
+            raise ValueError(msg)
+        for position, channel in enumerate(self.channels):
+            if channel.index != position:
+                msg = f"channel {channel.key!r} has index {channel.index}, expected {position}"
+                raise ValueError(msg)
+            unknown = [k for k in channel.derived_from or () if k not in keys]
+            if unknown:
+                msg = f"channel {channel.key!r}: derived_from refers to unknown keys {unknown}"
+                raise ValueError(msg)
+        return self
+
+
+class DatasetInfo(BaseModel):
+    """``metadata/dataset_info.json``: provenance, record counts, labels and pixel shift.
+
+    Parameters
+    ----------
+    source
+        What produced the data.
+    records
+        Number of labelled scenes per split.
+    labels
+        Label directories present in the dataset.
+    recommended_pixel_shift
+        Shift in pixels (right and down) that aligns the images with the masks;
+        ``0.0`` when they are aligned, ``None`` when unknown.
+    temporal
+        Whether the dataset stores context frames, ``<id>_b<k>`` / ``<id>_a<k>`` next to scene
+        ``<id>`` (see :func:`deepcontrail.core.splits.context_frames`); ``None`` means no.
+    export, naming, pixel_shift_note
+        Provenance notes for people; not read by the library.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    format_version: Literal[1]
+    source: str
+    records: dict[Literal["train", "test"], Annotated[int, Field(ge=0)]]
+    labels: tuple[Label, ...]
+    recommended_pixel_shift: Annotated[float, Field(ge=0.0)] | None
+    temporal: bool | None = None
+    export: str | None = None
+    naming: str | None = None
+    pixel_shift_note: str | None = None
+
+
+def _read(root: str | Path, relative: str, schema: type[_Schema]) -> _Schema:
+    path = Path(root) / relative
+    if not path.is_file():
+        msg = f"{path}: missing; every dataset root needs {relative} (DeepContrail format v1)"
+        raise DatasetContractError(msg)
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except UnicodeDecodeError as exc:
+        msg = f"{path}: must be UTF-8 encoded ({exc})"
+        raise DatasetContractError(msg) from exc
+    except json.JSONDecodeError as exc:
+        msg = f"{path}: not valid JSON ({exc})"
+        raise DatasetContractError(msg) from exc
+    if isinstance(payload, dict) and "format_version" not in payload:
+        msg = (
+            f"{path}: written before DeepContrail format v1 (no format_version). Regenerate the "
+            "metadata; for OpenContrails data: deepcontrail-convert-opencontrails "
+            "--metadata-only --output-dir <root>"
+        )
+        raise DatasetContractError(msg)
+    try:
+        return schema.model_validate(payload)
+    except ValidationError as exc:
+        msg = f"{path}: does not follow DeepContrail format v1:\n{exc}"
+        raise DatasetContractError(msg) from exc
+
+
+def read_band_manifest(root: str | Path) -> BandManifest:
+    """Read and validate ``<root>/spectral_bands/bands.json``.
+
+    Raises
+    ------
+    DatasetContractError
+        If the file is missing, is not JSON, or does not follow format v1.
+    """
+    return _read(root, BAND_MANIFEST_PATH, BandManifest)
+
+
+def read_dataset_info(root: str | Path) -> DatasetInfo:
+    """Read and validate ``<root>/metadata/dataset_info.json``.
+
+    Raises
+    ------
+    DatasetContractError
+        If the file is missing, is not JSON, or does not follow format v1.
+    """
+    return _read(root, DATASET_INFO_PATH, DatasetInfo)

deepcontrail/core/false_color.py

@@ -0,0 +1,118 @@
+"""False-colour composites built from spectral bands named in the dataset's manifest."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Annotated
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from deepcontrail.core.contract import DatasetContractError
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from deepcontrail.core.contract import BandManifest
+
+BandKey = Annotated[str, Field(min_length=1)]
+
+
+class FalseColorChannel(BaseModel):
+    """One composite channel: a band, or the difference of two bands, scaled by ``bounds``.
+
+    ``bands=(a,)`` gives ``a``; ``bands=(a, b)`` gives ``a - b``, where ``a`` and ``b``
+    are keys of the dataset's ``spectral_bands/bands.json``. The value is mapped
+    linearly from ``bounds = (lo, hi)`` to ``[0, 1]`` and clipped.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    bands: tuple[BandKey] | tuple[BandKey, BandKey]
+    bounds: tuple[float, float]
+
+    @model_validator(mode="after")
+    def _validate_bounds(self) -> FalseColorChannel:
+        lo, hi = self.bounds
+        if not lo < hi:
+            msg = f"bounds must be increasing (lo < hi), got {self.bounds}"
+            raise ValueError(msg)
+        return self
+
+
+class FalseColorConfig(BaseModel):
+    """Bands and bounds of the 3-channel composite built from ``spectral_bands/``.
+
+    There is no default recipe: band keys belong to the dataset, so the recipe is
+    always written in the config (e.g. the standard ash RGB in
+    ``configs/experiment.yaml``).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    channels: tuple[FalseColorChannel, FalseColorChannel, FalseColorChannel]
+
+
+def check_false_color(config: FalseColorConfig, manifest: BandManifest) -> None:
+    """Check that every band of the recipe is a key of the manifest.
+
+    Raises
+    ------
+    DatasetContractError
+        Naming every unknown key and listing the available ones.
+    """
+    unknown = [k for channel in config.channels for k in channel.bands if k not in manifest.keys]
+    if unknown:
+        msg = (
+            f"false_color uses band keys not in spectral_bands/bands.json: {unknown}; "
+            f"available: {', '.join(manifest.keys)}"
+        )
+        raise DatasetContractError(msg)
+
+
+def false_color_from_spectral(
+    bands: np.ndarray, config: FalseColorConfig, manifest: BandManifest
+) -> np.ndarray:
+    """Build the 3-channel composite from a spectral array.
+
+    Parameters
+    ----------
+    bands
+        Spectral array with shape ``(H, W, C)``, channels in manifest order (the
+        ``spectral_bands/*.npy`` layout).
+    config
+        Bands and bounds of each output channel.
+    manifest
+        The dataset's band manifest, mapping each key to its channel.
+
+    Returns
+    -------
+    np.ndarray
+        Float32 ``(H, W, 3)`` array with values in ``[0, 1]``.
+
+    Raises
+    ------
+    ValueError
+        If ``bands`` is not ``(H, W, C)`` with the manifest's ``C`` channels.
+    DatasetContractError
+        If the recipe uses a key the manifest does not declare.
+    """
+    n = len(manifest.channels)
+    if bands.ndim != 3 or bands.shape[-1] != n:
+        msg = f"expected spectral bands of shape (H, W, {n}) ({n} channels), got {bands.shape}"
+        raise ValueError(msg)
+
+    channels = []
+    for channel in config.channels:
+        value = bands[..., manifest.index_of(channel.bands[0])]
+        if len(channel.bands) == 2:
+            value = value - bands[..., manifest.index_of(channel.bands[1])]
+        lo, hi = channel.bounds
+        channels.append((value - lo) / (hi - lo))
+    return np.clip(np.stack(channels, axis=-1), 0, 1).astype(np.float32)
+
+
+def load_spectral_false_color(
+    path: str | Path, config: FalseColorConfig, manifest: BandManifest
+) -> np.ndarray:
+    """Load a ``spectral_bands/*.npy`` file and build its composite with ``config``."""
+    return false_color_from_spectral(np.load(path), config, manifest)

deepcontrail/core/metrics.py

@@ -0,0 +1,113 @@
+"""Pure evaluation metrics for binary segmentation (no gradient, numpy-based)."""
+
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from numpy.typing import ArrayLike
+
+
+def _to_numpy(x: object) -> np.ndarray:
+    """Convert input to a flat float32 numpy array, handling torch tensors safely.
+
+    float32 is enough for binary {0, 1} mask metrics; it halves the host-side
+    memory footprint when full val/test sets are concatenated.
+    """
+    # A tensor can only exist if torch is already imported: look it up instead of
+    # importing it, so numpy-only callers never load torch (deepcontrail-core has no torch).
+    torch = sys.modules.get("torch")
+    if torch is not None and isinstance(x, torch.Tensor):
+        tensor: Any = x
+        return tensor.detach().cpu().numpy().astype(np.float32).ravel()
+    return np.asarray(x, dtype=np.float32).ravel()
+
+
+def _check_binary_pair(pred: np.ndarray, target: np.ndarray) -> None:
+    """Raise ``ValueError`` unless both masks have the same size and hold only 0 and 1."""
+    if pred.size != target.size:
+        msg = f"inputs must have the same number of elements; got {pred.size} and {target.size}"
+        raise ValueError(msg)
+    for name, values in (("pred", pred), ("target", target)):
+        # Elementwise test: no sort, so cheap on whole validation sets.
+        if not ((values == 0) | (values == 1)).all():
+            bad = values[(values != 0) & (values != 1)]
+            msg = f"{name} must contain only 0 and 1, got e.g. {bad[:5].tolist()}"
+            raise ValueError(msg)
+
+
+def dice_coefficient(pred: ArrayLike, target: ArrayLike, smooth: float = 1e-6) -> float:
+    """Compute the Dice coefficient between binary masks.
+
+    Parameters
+    ----------
+    pred
+        Predicted binary mask (values in {0, 1}, boolean accepted). Any shape.
+    target
+        Ground-truth binary mask (values in {0, 1}, boolean accepted). Same number
+        of elements as ``pred``.
+    smooth
+        Smoothing constant to avoid division by zero.
+
+    Returns
+    -------
+    float
+        Dice coefficient in [0, 1], pooled over all elements (one global score,
+        not a per-image mean). Returns 1.0 when both masks are empty (all zeros).
+
+    Raises
+    ------
+    ValueError
+        If ``pred`` and ``target`` have different numbers of elements, or if either
+        contains values outside {0, 1}.
+    """
+    pred_flat = _to_numpy(pred)
+    target_flat = _to_numpy(target)
+
+    _check_binary_pair(pred_flat, target_flat)
+
+    if pred_flat.sum() == 0 and target_flat.sum() == 0:
+        return 1.0
+
+    intersection = (pred_flat * target_flat).sum()
+    return float((2.0 * intersection + smooth) / (pred_flat.sum() + target_flat.sum() + smooth))
+
+
+def iou_score(pred: ArrayLike, target: ArrayLike, smooth: float = 1e-6) -> float:
+    """Compute the Intersection over Union (Jaccard index) between binary masks.
+
+    Parameters
+    ----------
+    pred
+        Predicted binary mask (values in {0, 1}, boolean accepted).
+    target
+        Ground-truth binary mask (values in {0, 1}, boolean accepted).
+    smooth
+        Smoothing constant.
+
+    Returns
+    -------
+    float
+        IoU score in [0, 1], pooled over all elements (one global score, not a
+        per-image mean). Returns 1.0 when both masks are empty (all zeros).
+
+    Raises
+    ------
+    ValueError
+        If ``pred`` and ``target`` have different numbers of elements, or if either
+        contains values outside {0, 1}.
+    """
+    pred_flat = _to_numpy(pred)
+    target_flat = _to_numpy(target)
+
+    _check_binary_pair(pred_flat, target_flat)
+
+    if pred_flat.sum() == 0 and target_flat.sum() == 0:
+        return 1.0
+
+    intersection = (pred_flat * target_flat).sum()
+    union = pred_flat.sum() + target_flat.sum() - intersection
+    return float((intersection + smooth) / (union + smooth))