firecube 0.1.4__py3-none-any.whl

firecube/core/slot_index.py

@@ -0,0 +1,205 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Slot-index model for firecube product time axes.
+
+The slot-index model describes how a product's time axis is partitioned into
+discrete slots: one or more named groups, each with its own cadence and
+rounding mode, anchored on a shared UTC epoch. The model is content-addressed
+via a canonical-JSON SHA-256 (``identity_hash``) so independent writers can
+verify they agree on the partitioning scheme without coordinating beforehand.
+
+Stability rules:
+
+* ``canonical_bytes()`` is deterministic: groups are emitted in alphabetical
+  order, JSON keys are sorted, and the separators are tight. ``time_unit=None``
+  is always serialised as ``"time_unit":null``.
+* ``identity_hash`` is never embedded inside the hashed payload.
+* Epoch normalisation is the explicit responsibility of
+  :func:`normalize_epoch_iso`; ``canonical_bytes()`` does NOT silently mutate
+  the stored epoch string. ``"Z"`` and ``"+00:00"`` therefore deliberately
+  produce different hashes -- callers that want them to converge must
+  normalise the epoch before constructing the model.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Literal
+
+SLOT_INDEX_MODEL_ATTR = "firecube_slot_index_model"
+SLOT_INDEX_MODEL_IDENTITY_HASH_ATTR = "firecube_slot_index_model_identity_hash"
+
+
+@dataclass(frozen=True, slots=True)
+class SlotAxis:
+    """A single time-axis partitioning rule for one group.
+
+    Attributes:
+        cadence_s: Slot width in seconds; must be strictly positive.
+        mode: ``"exact"`` means timestamps must align exactly on a slot
+            boundary; ``"floor"`` means timestamps are floored to the
+            preceding boundary.
+    """
+
+    cadence_s: int
+    mode: Literal["exact", "floor"]
+
+    def __post_init__(self) -> None:
+        if self.cadence_s <= 0:
+            raise ValueError(f"cadence_s must be > 0, got {self.cadence_s!r}")
+        if self.mode not in {"exact", "floor"}:
+            raise ValueError(f"mode must be 'exact' or 'floor', got {self.mode!r}")
+
+
+@dataclass(frozen=True, slots=True)
+class SlotIndexModel:
+    """Content-addressable description of a product's slot-index partitioning.
+
+    Attributes:
+        name: Human-readable identifier for the model (e.g. ``"opera_v1"``).
+        epoch: ISO-8601 UTC anchor (e.g. ``"2026-01-01T00:00:00Z"``); used as
+            the origin from which slot indices are counted.
+        groups: Per-group :class:`SlotAxis` mapping; at least one entry.
+        time_unit: Optional storage-precision hint
+            (e.g. ``"seconds"``, ``"milliseconds"``); ``None`` is permitted
+            and is serialised as JSON ``null``.
+    """
+
+    name: str
+    epoch: str
+    groups: dict[str, SlotAxis]
+    time_unit: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.name:
+            raise ValueError("name must be non-empty")
+        if not self.epoch:
+            raise ValueError("epoch must be non-empty")
+        if not self.groups:
+            raise ValueError("groups must be non-empty")
+        for k in self.groups:
+            if not k:
+                raise ValueError("group keys must be non-empty strings")
+
+    def canonical_bytes(self) -> bytes:
+        """Return the canonical UTF-8 JSON encoding of this model.
+
+        Determinism rules:
+
+        * ``groups`` are emitted in alphabetical key order.
+        * ``json.dumps(..., sort_keys=True, separators=(",", ":"))`` is used so
+          that whitespace and key order are stable across runs and platforms.
+        * ``time_unit=None`` is serialised as JSON ``null``.
+        * The output does NOT contain :attr:`identity_hash` -- the hash is
+          computed over the canonical bytes, not stored inside them.
+        """
+
+        payload = {
+            "schema_version": "v1",
+            "name": self.name,
+            "epoch": self.epoch,
+            "time_unit": self.time_unit,
+            "groups": {
+                group: {"cadence_s": axis.cadence_s, "mode": axis.mode}
+                for group, axis in sorted(self.groups.items())
+            },
+        }
+        return json.dumps(
+            payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+        ).encode("utf-8")
+
+    @property
+    def identity_hash(self) -> str:
+        """SHA-256 hex digest of :meth:`canonical_bytes`.
+
+        Two :class:`SlotIndexModel` instances are considered equivalent
+        partitioning schemes iff their ``identity_hash`` values match.
+        """
+
+        return hashlib.sha256(self.canonical_bytes()).hexdigest()
+
+
+def iso_to_epoch_s(iso: str) -> int:
+    """Convert an ISO-8601 UTC timestamp to whole seconds since Unix epoch.
+
+    Only UTC inputs are accepted. The trailing ``"Z"`` or the explicit
+    ``"+00:00"`` offset are both honoured; any other offset raises
+    :class:`ValueError`. Numpy is imported lazily so that ``slot_index`` stays
+    cheap to import for code paths that never touch the epoch helpers.
+    """
+
+    import numpy as np
+
+    if not iso:
+        raise ValueError("iso must be a non-empty string")
+
+    text = iso.strip()
+    if text.endswith("Z"):
+        bare = text[:-1]
+    elif text.endswith("+00:00"):
+        bare = text[: -len("+00:00")]
+    else:
+        # Any other explicit timezone offset (including non-UTC offsets) is rejected.
+        # np.datetime64 silently accepts naive ISO strings, so we cannot rely on it
+        # to validate the offset for us.
+        for sign in ("+", "-"):
+            sign_idx = text.rfind(sign)
+            # The leading sign of the year is at index 0 (or 1 after a space), so
+            # only treat a sign that appears at position >= 10 as a TZ offset.
+            if sign_idx >= 10:
+                raise ValueError(f"iso must be UTC ('Z' or '+00:00' offset), got {iso!r}")
+        bare = text
+
+    try:
+        when = np.datetime64(bare, "s")
+    except (ValueError, TypeError) as exc:
+        raise ValueError(f"could not parse ISO-8601 timestamp {iso!r}: {exc}") from exc
+
+    epoch = np.datetime64("1970-01-01T00:00:00", "s")
+    return int((when - epoch).astype("int64"))
+
+
+def epoch_s_to_iso(seconds: int) -> str:
+    """Convert whole seconds-since-epoch to a canonical ``"YYYY-MM-DDTHH:MM:SSZ"``."""
+
+    import numpy as np
+
+    when = np.datetime64("1970-01-01T00:00:00", "s") + np.timedelta64(int(seconds), "s")
+    # str(np.datetime64(..., 's')) renders as "YYYY-MM-DDTHH:MM:SS"; append the
+    # UTC marker so that the round-trip with iso_to_epoch_s is exact.
+    return f"{when!s}Z"
+
+
+def normalize_epoch_iso(iso: str) -> str:
+    """Round-trip an ISO-8601 UTC string through ``epoch_s`` and back to ``"...Z"``.
+
+    Equivalent to ``epoch_s_to_iso(iso_to_epoch_s(iso))``; raises
+    :class:`ValueError` for non-UTC inputs via :func:`iso_to_epoch_s`.
+    """
+
+    return epoch_s_to_iso(iso_to_epoch_s(iso))
+
+
+__all__ = [
+    "SLOT_INDEX_MODEL_ATTR",
+    "SLOT_INDEX_MODEL_IDENTITY_HASH_ATTR",
+    "SlotAxis",
+    "SlotIndexModel",
+    "epoch_s_to_iso",
+    "iso_to_epoch_s",
+    "normalize_epoch_iso",
+]

firecube/core/storage/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Storage primitives: URI value objects, driver configuration, sessions, transfer.
+
+Public imports remain stable at ``firecube.core.storage``.
+"""
+
+from firecube.core.storage.binding import StorageBinding
+from firecube.core.storage.cache_key import StorageCacheKey
+from firecube.core.storage.driver_config import StorageDriverConfig
+from firecube.core.storage.results import StorageWriteResult
+from firecube.core.storage.session import StorageSession
+from firecube.core.storage.transfer import copy_file, session_for_uri
+from firecube.core.storage.uri import (
+    LOCAL_PROTOCOLS,
+    REMOTE_PROTOCOLS,
+    SUPPORTED_PROTOCOLS,
+    StorageUri,
+)
+
+__all__ = [
+    "LOCAL_PROTOCOLS",
+    "REMOTE_PROTOCOLS",
+    "SUPPORTED_PROTOCOLS",
+    "StorageBinding",
+    "StorageCacheKey",
+    "StorageDriverConfig",
+    "StorageSession",
+    "StorageUri",
+    "StorageWriteResult",
+    "copy_file",
+    "session_for_uri",
+]

firecube/core/storage/binding.py

@@ -0,0 +1,39 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from firecube.core.storage.driver_config import StorageDriverConfig
+
+if TYPE_CHECKING:
+    from firecube.core.product.identity import ProductIdentity
+
+
+@dataclass(frozen=True, slots=True)
+class StorageBinding:
+    """Credentials are immutable and set once at the run boundary via `from_storage_config()`.
+    No mid-run credential rotation is supported."""
+
+    identity: ProductIdentity
+    driver: StorageDriverConfig
+
+    def cache_key(self):
+        from firecube.core.storage.cache_key import (
+            StorageCacheKey,
+        )
+
+        return StorageCacheKey.from_binding(self)

firecube/core/storage/cache_key.py

@@ -0,0 +1,40 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class StorageCacheKey:
+    driver: str
+    protocol: str
+    authority: str | None
+    endpoint_url: str | None
+    region: str | None
+    credential_fingerprint: str | None
+
+    @classmethod
+    def from_binding(cls, binding):
+
+        creds = binding.driver.credentials
+        return cls(
+            driver=binding.driver.driver,
+            protocol=binding.identity.product_uri.protocol,
+            authority=binding.identity.product_uri.authority,
+            endpoint_url=binding.driver.endpoint_url,
+            region=binding.driver.region,
+            credential_fingerprint=creds.fingerprint() if creds is not None else None,
+        )

firecube/core/storage/completion.py

@@ -0,0 +1,245 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import contextlib
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from firecube.core.config import StorageConfig
+from firecube.core.errors import StorageError
+from firecube.core.filesystem import path_stats
+from firecube.core.product import CompletionRoute, write_mode_policy
+from firecube.core.product.identity import ProductIdentity
+from firecube.core.product.target import ProductTarget
+from firecube.core.storage.binding import StorageBinding
+from firecube.core.storage.driver_config import StorageDriverConfig
+from firecube.core.storage.results import StorageWriteResult
+from firecube.core.storage.session import StorageSession, storage_config_from_binding
+from firecube.core.storage.uri import StorageUri
+from firecube.core.uris import is_remote_target, local_path_from_target
+
+if TYPE_CHECKING:
+    from firecube.core.storage.session import StorageSession as _StorageSession
+
+
+def _output_session(ctx: Any) -> _StorageSession | None:
+    storage = ctx.storage
+    return storage.output if storage is not None else None
+
+
+def _output_product_name(ctx: Any, default: str = "") -> str:
+    session = _output_session(ctx)
+    if session is not None:
+        return str(session.product.product_name)
+    return default
+
+
+def _storage_config_from_ctx(ctx: Any) -> StorageConfig | None:
+    session = _output_session(ctx)
+    if session is None:
+        return None
+    return storage_config_from_binding(session._binding)
+
+
+class StorageCompleter:
+    def complete_output(self, result: Any, ctx: Any) -> StorageWriteResult:
+        session = _output_session(ctx)
+        if session is None:
+            raise StorageError("Storage completion requires ctx.storage.output.")
+        storage_config = storage_config_from_binding(session._binding)
+
+        effective_write_mode = str(result.metrics.write_mode or ctx.option("write_mode") or "")
+        product_uri = session.product.product_uri
+        if not product_uri.is_remote():
+            return self.complete_local_storage(result, ctx)
+        return self.complete_s3_storage(result, ctx, storage_config, effective_write_mode)
+
+    def complete_local_storage(self, result: Any, ctx: Any) -> StorageWriteResult:
+        session = _output_session(ctx)
+        if session is None:
+            raise StorageError("Local completion requires ctx.storage.output.")
+
+        product_uri = session.product.product_uri
+        final_target = Path(product_uri.path)
+        output_path = str(result.outputs.primary or "")
+        if "://" in output_path:
+            source_uri = StorageUri.parse(output_path)
+            if source_uri.protocol != "file":
+                raise StorageError(f"Local staged output must be file://, got {output_path}")
+            source_path = Path(source_uri.path).resolve()
+        else:
+            source_path = Path(output_path).resolve()
+
+        is_already_in_place = False
+        try:
+            source_path.relative_to(final_target.resolve())
+            is_already_in_place = True
+        except ValueError:
+            pass
+
+        if is_already_in_place:
+            return StorageWriteResult(
+                path=str(source_path),
+                bytes_written=0,
+                files_written=0,
+                duration_s=0.0,
+                storage_type="local",
+            )
+
+        if not source_path.exists():
+            raise StorageError(f"Local staged output not found at {source_path}")
+
+        upload_workers = int(ctx.option("upload_workers", 4))
+        return session.upload_tree(
+            StorageUri.from_local_path(source_path),
+            StorageUri.from_local_path(final_target),
+            parallel_workers=upload_workers,
+        )
+
+    def complete_s3_storage(
+        self,
+        result: Any,
+        ctx: Any,
+        storage_config: StorageConfig,
+        effective_write_mode: str,
+    ) -> StorageWriteResult:
+        session = _output_session(ctx)
+        if session is None:
+            raise StorageError("Remote completion requires ctx.storage.output.")
+        final_target_uri = session.product.product_uri.to_str()
+
+        route = write_mode_policy(effective_write_mode).completion_route
+        if route is CompletionRoute.DIRECT:
+            return self.complete_s3_direct(result, storage_config, final_target_uri)
+
+        return self.complete_s3_staged(result, ctx, final_target_uri)
+
+    def complete_s3_direct(
+        self,
+        result: Any,
+        storage_config: StorageConfig,
+        final_target_uri: str,
+    ) -> StorageWriteResult:
+        storage_summary = result.metrics.storage
+        if storage_summary is not None:
+            return StorageWriteResult(
+                path=str(storage_summary.path or final_target_uri),
+                bytes_written=int(storage_summary.bytes or 0),
+                files_written=int(storage_summary.files or 0),
+                duration_s=float(storage_summary.duration_s or 0.0),
+                storage_type="s3",
+            )
+
+        stats = path_stats(final_target_uri, storage_config=storage_config)
+        return StorageWriteResult(
+            path=str(final_target_uri),
+            bytes_written=int(stats.get("bytes", 0) or 0),
+            files_written=int(stats.get("files", 0) or 0),
+            duration_s=0.0,
+            storage_type="s3",
+        )
+
+    def complete_s3_staged(
+        self,
+        result: Any,
+        ctx: Any,
+        final_target_uri: str,
+    ) -> StorageWriteResult:
+        source_path = Path(str(result.outputs.primary or ""))
+        if result.output_format == "zarr":
+            zarr_out = result.outputs.zarr
+            zarr_out_str = str(zarr_out) if zarr_out is not None else ""
+            if zarr_out_str and not is_remote_target(zarr_out_str):
+                candidate = Path(zarr_out_str)
+                if candidate.exists():
+                    source_path = candidate
+            elif zarr_out_str and zarr_out_str.startswith("file://"):
+                try:
+                    candidate = local_path_from_target(zarr_out_str)
+                    if candidate.exists():
+                        source_path = candidate
+                except Exception:
+                    pass
+
+        resolved_source = source_path.resolve()
+        if resolved_source == Path.home().resolve() or resolved_source == Path("/").resolve():
+            raise StorageError(f"Critical Safety: Attempted to upload {resolved_source}. Aborting.")
+
+        if not source_path.exists():
+            raise StorageError(
+                f"Ingestion failed: write_mode is 'staged' but no local output "
+                f"was found at {source_path}. "
+                "If the plugin writes directly to the target, use --write-mode direct."
+            )
+
+        storage_config = _storage_config_from_ctx(ctx)
+        if storage_config is None:
+            raise StorageError("Staged remote upload requires ctx.storage.output.")
+
+        driver_config = StorageDriverConfig.from_storage_config(storage_config)
+        storage_session = _output_session(ctx)
+        session = storage_session
+        if session is None:
+            resolved_product = ProductTarget.resolve(
+                final_target_uri,
+                driver_config,
+                product_name=_output_product_name(ctx, final_target_uri),
+                plugin_default_format=result.output_format or "zarr",
+            )
+            session = StorageSession(
+                StorageBinding(
+                    identity=ProductIdentity(
+                        product_name=resolved_product.product_name,
+                        product_uri=resolved_product.product_uri,
+                        control_root_uri=resolved_product.control_root_uri,
+                        format=resolved_product.format,
+                    ),
+                    driver=driver_config,
+                )
+            )
+
+        effective_target_uri = (
+            session.product.product_uri.to_str()
+            if storage_session is not None
+            else final_target_uri
+        )
+        telemetry = getattr(ctx, "telemetry", None)
+        upload_span = (
+            telemetry.span(
+                "firecube.upload_s3",
+                attributes={"firecube.target": str(final_target_uri)},
+            )
+            if telemetry is not None
+            else contextlib.nullcontext()
+        )
+        upload_workers = int(ctx.option("upload_workers", 4))
+        source_uri = StorageUri.from_local_path(source_path)
+        with upload_span:
+            stored = session.upload_tree(
+                source_uri,
+                StorageUri.parse(effective_target_uri),
+                parallel_workers=upload_workers,
+            )
+
+        result.metrics.storage_handled = True
+        pipeline_metrics = result.metrics.pipeline
+        if pipeline_metrics is not None:
+            pipeline_metrics.duration_upload_s = stored.duration_s
+            pipeline_metrics.duration_total_s = (
+                pipeline_metrics.duration_pipeline_s + stored.duration_s
+            )
+
+        return stored