firecube 0.1.4__py3-none-any.whl

firecube/core/zarr/scrub.py

@@ -0,0 +1,200 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Zarr scrub orchestration (mutating) used by CLI and API.
+
+This module *acts* on validation results: it maps debris chunk keys back to
+manifest entries and deletes them via ChunkManager.
+
+Keep this separate from `firecube.core.zarr.validation`, which is read-only.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from firecube.core.controlplane import ChunkInfo, ChunkManager
+from firecube.core.filesystem import create_filesystem
+from firecube.core.storage.binding import StorageBinding
+from firecube.core.storage.session import StorageSession
+from firecube.core.zarr.validation import validate_group_with_fs
+
+
+@dataclass
+class ScrubResult:
+    """Result of a scrub operation for a single group."""
+
+    product: str
+    group: str
+    extra_chunks: list[str]
+    deleted_chunks: int
+    deleted_size_bytes: int
+    storage_errors: list[str]
+    manifest_errors: list[str]
+
+
+def run_scrub(
+    session: StorageSession,
+    group: str,
+    *,
+    max_chunks: int | None = None,
+    include_orphans: bool = False,
+) -> ScrubResult:
+    """Scrub debris chunks for a given array group using ChunkManager.
+
+    The ``session`` carries the product binding (URI, driver, credentials);
+    no string-concat URI building or raw credential dicts live here.
+
+    This helper:
+      1) Validates the group to find out-of-range chunks.
+      2) Maps those chunk paths back to manifest keys.
+      3) Uses ChunkManager to delete only matching chunks from storage
+         and manifests.
+    """
+    product_uri = session.product.product_uri
+    product_name = session.product.product_name
+    fs = session.fs()
+
+    try:
+        report = validate_group_with_fs(
+            fs,
+            product_uri,
+            group,
+        )
+    except ValueError as exc:
+        if "is a Zarr group (container)" not in str(exc):
+            raise
+        return ScrubResult(
+            product=product_name,
+            group=group,
+            extra_chunks=[],
+            deleted_chunks=0,
+            deleted_size_bytes=0,
+            storage_errors=[],
+            manifest_errors=[],
+        )
+    extra_paths = list(report.extra_chunks)
+
+    if max_chunks is not None and max_chunks >= 0:
+        extra_paths = extra_paths[:max_chunks]
+
+    prefix = f"{product_name}/"
+    manifest_keys: list[str] = []
+    for path in extra_paths:
+        key = path
+        if prefix in path:
+            key = path.split(prefix, 1)[1]
+        manifest_keys.append(key)
+
+    if not manifest_keys:
+        return ScrubResult(
+            product=product_name,
+            group=group,
+            extra_chunks=[],
+            deleted_chunks=0,
+            deleted_size_bytes=0,
+            storage_errors=[],
+            manifest_errors=[],
+        )
+
+    binding = StorageBinding(identity=session.product, driver=session.driver)
+    manager = ChunkManager(binding=binding, filesystem=create_filesystem(binding))
+    try:
+        all_chunks: list[ChunkInfo] = []
+        for key in manifest_keys:
+            matches = manager.list_chunks(pattern=key, product=product_name)
+            all_chunks.extend(matches)
+
+        seen = set()
+        unique_chunks: list[ChunkInfo] = []
+        for chunk in all_chunks:
+            k = (chunk.product, chunk.key, chunk.manifest_path)
+            if k not in seen:
+                seen.add(k)
+                unique_chunks.append(chunk)
+
+        total_deleted_chunks = 0
+        total_deleted_size = 0
+        storage_errors: list[str] = []
+        manifest_errors: list[str] = []
+
+        if unique_chunks:
+            plan = manager.create_deletion_plan()
+            plan.chunks = unique_chunks
+            plan.total_size = sum(c.size for c in unique_chunks)
+            plan.products_affected = {product_name}
+            plan.manifest_files = {c.manifest_path for c in unique_chunks}
+
+            result = manager.execute_deletion(
+                plan,
+                delete_storage=True,
+                delete_manifest=True,
+                storage_config=None,
+                dry_run=False,
+            )
+            total_deleted_chunks += result.get("deleted_chunks", 0)
+            total_deleted_size += result.get("deleted_size_bytes", 0)
+            storage_errors.extend(result.get("storage_errors", []))
+            manifest_errors.extend(result.get("manifest_errors", []))
+
+        if include_orphans:
+            manifest_key_set = {c.key for c in unique_chunks}
+            orphan_keys = [k for k in manifest_keys if k not in manifest_key_set]
+
+            if orphan_keys:
+                from time import time as _time
+
+                now_ts = _time()
+                orphan_chunks: list[ChunkInfo] = [
+                    ChunkInfo(
+                        key=key,
+                        product=product_name,
+                        chunk_type="chunk",
+                        size=0,
+                        timestamp=now_ts,
+                        manifest_path="",
+                    )
+                    for key in orphan_keys
+                ]
+
+                orphan_plan = manager.create_deletion_plan()
+                orphan_plan.chunks = orphan_chunks
+                orphan_plan.total_size = 0
+                orphan_plan.products_affected = {product_name}
+                orphan_plan.manifest_files = set()
+
+                orphan_result = manager.execute_deletion(
+                    orphan_plan,
+                    delete_storage=True,
+                    delete_manifest=False,
+                    storage_config=None,
+                    dry_run=False,
+                )
+                if not orphan_result.get("storage_errors"):
+                    total_deleted_chunks += len(orphan_keys)
+                total_deleted_size += orphan_result.get("deleted_size_bytes", 0)
+                storage_errors.extend(orphan_result.get("storage_errors", []))
+                manifest_errors.extend(orphan_result.get("manifest_errors", []))
+
+        return ScrubResult(
+            product=product_name,
+            group=group,
+            extra_chunks=manifest_keys,
+            deleted_chunks=total_deleted_chunks,
+            deleted_size_bytes=total_deleted_size,
+            storage_errors=storage_errors,
+            manifest_errors=manifest_errors,
+        )
+    finally:
+        manager.close()

firecube/core/zarr/state.py

@@ -0,0 +1,258 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared helpers for Firecube-managed Zarr state arrays.
+
+This module centralizes the semantics around the per-timestamp state array
+(`firecube_timestamp_state`) used by span-based deletion/resume logic.
+
+It is intentionally small and "dumb": callers decide *when* to create/update
+the state array; this module only provides safe primitives.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, cast
+
+from firecube.core.uris import storage_uri_from_target
+
+if TYPE_CHECKING:
+    from firecube.core.config import StorageConfig
+    from firecube.core.filesystem.store_factory import ZarrStoreHandle
+
+DEFAULT_STATE_MEANING: dict[str, str] = {
+    "0": "unknown",
+    "1": "present",
+    "2": "deleted_by_firecube",
+    "3": "failed_batch",
+}
+
+
+def _session_zarr_store(
+    *,
+    store_uri: str,
+    storage_config: StorageConfig,
+    mode: str,
+) -> ZarrStoreHandle:
+    from firecube.core.product.identity import ProductIdentity
+    from firecube.core.storage.binding import StorageBinding
+    from firecube.core.storage.driver_config import StorageDriverConfig
+    from firecube.core.storage.session import StorageSession
+
+    uri = storage_uri_from_target(store_uri)
+    session = StorageSession(
+        StorageBinding(
+            identity=ProductIdentity.from_uri(uri, format="zarr", product_name=store_uri),
+            driver=StorageDriverConfig.from_storage_config(storage_config),
+        )
+    )
+    return session.zarr.create_store(uri=uri, mode=mode)
+
+
+def attach_timestamp_state_dataset(
+    dataset: Any,
+    *,
+    dim: str = "timestamp",
+    var_name: str = "firecube_timestamp_state",
+    present_value: int = 1,
+    meaning: dict[str, str] | None = None,
+) -> Any:
+    """Attach a per-timestamp state variable to an xarray Dataset (best effort).
+
+    This is used by ingestors so span-based tooling (delete spans / scrub) can
+    update state without relying on plugin-specific semantics.
+    """
+    meaning = meaning or DEFAULT_STATE_MEANING
+    try:
+        import numpy as np
+    except Exception as exc:
+        raise RuntimeError("numpy required for timestamp state dataset helper") from exc
+
+    if dataset is None:
+        return dataset
+
+    length = 0
+    try:
+        length = int(getattr(dataset, "sizes", {}).get(dim, 0))
+    except Exception:
+        length = 0
+    if length <= 0:
+        return dataset
+
+    try:
+        state_vals = np.full((length,), np.uint8(int(present_value)), dtype=np.uint8)
+        dataset[var_name] = ((dim,), state_vals)
+        attrs = getattr(dataset[var_name], "attrs", None)
+        if isinstance(attrs, dict):
+            attrs.update({"firecube_meaning": dict(meaning)})
+        else:
+            dataset[var_name].attrs.update({"firecube_meaning": dict(meaning)})
+    except Exception:
+        return dataset
+
+    return dataset
+
+
+def _normalize_time_index_ranges(ranges: Sequence[Any]) -> list[tuple[int, int]]:
+    normalized: list[tuple[int, int]] = []
+    for pair in ranges:
+        if not (isinstance(pair, (list, tuple)) and len(pair) == 2):
+            continue
+        try:
+            start = int(pair[0])
+            end = int(pair[1])
+        except Exception:
+            continue
+        if end < start:
+            continue
+        normalized.append((start, end))
+    return normalized
+
+
+def expand_time_index_ranges_to_chunk_boundaries(
+    ranges: Sequence[Any],
+    *,
+    chunk_len: int,
+    length: int,
+) -> list[list[int]]:
+    """Expand time index ranges to full chunk coverage boundaries.
+
+    This is used when deleting whole chunks for spans that are not aligned:
+    the state array should be updated for the full removed chunk coverage.
+    """
+    chunk_len = int(chunk_len or 0)
+    length = int(length or 0)
+    if chunk_len <= 0 or length <= 0:
+        return []
+
+    normalized = _normalize_time_index_ranges(ranges)
+    if not normalized:
+        return []
+
+    chunk_indices = set()
+    for start, end in normalized:
+        start_idx = start // chunk_len
+        end_idx = end // chunk_len
+        for idx in range(start_idx, end_idx + 1):
+            chunk_indices.add(idx)
+
+    expanded: list[list[int]] = []
+    for idx in sorted(i for i in chunk_indices if i >= 0):
+        start = idx * chunk_len
+        end = min(length - 1, (idx + 1) * chunk_len - 1)
+        if end >= start:
+            expanded.append([start, end])
+    return expanded
+
+
+def ensure_timestamp_state_array(
+    *,
+    store_uri: str,
+    array_path: str,
+    length: int,
+    chunk_len: int,
+    dim: str = "timestamp",
+    storage_config: StorageConfig | None = None,
+    storage_options: dict[str, Any] | None = None,
+    present_value: int = 1,
+    fill_value: int = 0,
+    meaning: dict[str, str] | None = None,
+) -> None:
+    """Ensure a `uint8` timestamp state array exists (best effort)."""
+    length = int(length or 0)
+    if length <= 0:
+        return
+
+    try:
+        import numpy as np
+        import zarr
+    except Exception as exc:
+        raise RuntimeError("zarr+numpy required for timestamp state updates") from exc
+
+    meaning = meaning or DEFAULT_STATE_MEANING
+    chunk_len = int(chunk_len or length)
+    chunk_len = max(1, min(chunk_len, length))
+
+    if storage_config is not None:
+        handle = _session_zarr_store(store_uri=store_uri, storage_config=storage_config, mode="a")
+        root = zarr.open_group(**handle.zarr_kwargs(), mode="a", zarr_format=3)
+    else:
+        root = zarr.open_group(store=store_uri, mode="a", storage_options=storage_options)
+    parts = str(array_path).strip("/").split("/")
+    grp = root
+    for part in parts[:-1]:
+        grp = grp.require_group(part)
+
+    name = parts[-1]
+    if name in grp:
+        return
+
+    try:
+        arr = grp.create_array(
+            name,
+            shape=(length,),
+            dtype="u1",
+            chunks=(chunk_len,),
+            fill_value=int(fill_value),
+            dimension_names=(dim,),
+            overwrite=False,
+        )
+        arr[:] = np.uint8(int(present_value))
+        arr.attrs.update({"firecube_meaning": dict(meaning)})
+    except Exception as exc:
+        # Zarr v3 raises ContainsArrayError if overwrite=False.
+        # Fallback to existing if someone else created it in the meantime.
+        if "ContainsArrayError" in type(exc).__name__ or "exists" in str(exc).lower():
+            return
+        raise
+
+
+def update_timestamp_state(
+    *,
+    store_uri: str,
+    array_path: str,
+    time_index_ranges: Sequence[Any],
+    value: int,
+    storage_config: StorageConfig | None = None,
+    storage_options: dict[str, Any] | None = None,
+) -> None:
+    """Best-effort update of a per-timestamp state array (uint8)."""
+    normalized = _normalize_time_index_ranges(time_index_ranges)
+    if not normalized:
+        return
+
+    try:
+        import numpy as np
+        import zarr
+    except Exception as exc:
+        raise RuntimeError("zarr+numpy required for timestamp state updates") from exc
+
+    if storage_config is not None:
+        handle = _session_zarr_store(store_uri=store_uri, storage_config=storage_config, mode="a")
+        root = zarr.open_group(**handle.zarr_kwargs(), mode="a", zarr_format=3)
+    else:
+        root = zarr.open_group(store=store_uri, mode="a", storage_options=storage_options)
+    parts = str(array_path).strip("/").split("/")
+    grp = root
+    for part in parts[:-1]:
+        grp = grp.require_group(part)
+    name = parts[-1]
+    if name not in grp:
+        return
+
+    arr = grp[name]
+    state_val = np.uint8(int(value))
+    for start, end in normalized:
+        cast(Any, arr)[start : end + 1] = state_val

firecube/core/zarr/time_decode.py

@@ -0,0 +1,71 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Self-describing time-array decode helper.
+
+Dispatches on (dtype, attrs) — firecube's vocabulary for encoded time arrays.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import numpy as np
+
+__all__ = ["decode_time_array"]
+
+
+def decode_time_array(values: np.ndarray, attrs: Mapping[str, Any]) -> np.ndarray:
+    """Return a ``datetime64`` array decoded from *values* using *attrs*.
+
+    The decoded array preserves its native resolution rather than being forced
+    to a fixed granularity. Coverage bounds and dedup keys are derived from this
+    output, so coarsening to seconds here would silently collapse distinct
+    sub-second timestamps into one (corrupting dedup/coverage); coarsening is
+    therefore left to the storage layer, which owns the on-disk precision
+    contract. The resolution that ``decode_cf_datetime`` selects is range-aware,
+    so this also avoids forcing a finer unit that could overflow for
+    out-of-range epochs.
+    """
+
+    values = np.asarray(values)
+
+    if values.dtype.kind == "M":
+        return values
+
+    if values.dtype.kind in ("f", "i", "u"):
+        units = attrs.get("units") if attrs else None
+        if units is None:
+            raise ValueError(
+                f"Cannot decode numeric dtype {values.dtype!r}: no 'units' attr found. "
+                "Expected a units string like 'seconds since 1970-01-01'."
+            )
+        units_str = str(units)
+        if "since" not in units_str:
+            raise ValueError(
+                f"Cannot decode numeric dtype {values.dtype!r}: 'units' attr {units_str!r} "
+                "does not contain 'since'. Expected a reference-epoch string like "
+                "'seconds since 1970-01-01'."
+            )
+        from xarray.coding.times import decode_cf_datetime
+
+        calendar = str(attrs.get("calendar", "standard"))
+        decoded = decode_cf_datetime(values, units=units_str, calendar=calendar)
+        return np.asarray(decoded)
+
+    raise ValueError(
+        f"Cannot decode time array with dtype {values.dtype!r}: not a datetime64 or "
+        "a numeric type with 'units' containing 'since'."
+    )