firecube 0.1.4__py3-none-any.whl

firecube/ingestor/runtime/base_hooks.py

@@ -0,0 +1,129 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Reusable lifecycle and slice-metadata hooks for BaseIngestor."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from firecube.ingestor.types.context import (
+    PipelineBatch,
+    PipelineResult,
+    PipelineRunState,
+    PluginContext,
+)
+
+
+def _product_name(ctx: PluginContext, default: str) -> str:
+    storage = ctx.storage
+    if storage is not None and storage.output is not None:
+        return str(storage.output.product.product_name)
+    return default
+
+
+def _canonicalize_slice_value(value: Any) -> Any:
+    if isinstance(value, set):
+        value = list(value)
+    if isinstance(value, tuple):
+        value = list(value)
+    if isinstance(value, list):
+        try:
+            return sorted(value)
+        except TypeError:
+            return list(value)
+    return value
+
+
+class BaseIngestorHookMixin:
+    """Shared non-engine hooks used by BaseIngestor implementations."""
+
+    name: str
+    _log: logging.Logger
+    _span_recorder: Any
+
+    def slice_meta_keys(self) -> list[str]:
+        """Option keys that define a logical slice for this plugin."""
+        return []
+
+    def slice_meta(self, ctx: PluginContext) -> dict[str, Any]:
+        """Return canonical slice metadata for this run."""
+        meta: dict[str, Any] = {}
+        for key in self.slice_meta_keys():
+            if key in ctx.options:
+                meta[key] = _canonicalize_slice_value(ctx.options.get(key))
+        return meta
+
+    def validation_group(self, ctx: PluginContext) -> str | None:
+        """Optional hook to derive a Zarr group path for validate_zarr."""
+        _ = ctx
+        return None
+
+    def on_pipeline_start(self, ctx: PluginContext, state: PipelineRunState) -> None:
+        """Called before pipeline execution starts."""
+        _ = ctx
+        self._log.info(
+            "Starting pipeline execution workers=%d batch_size=%d",
+            state.pipeline_workers,
+            state.batch_size,
+        )
+
+    def on_batch_success(
+        self,
+        ctx: PluginContext,
+        state: PipelineRunState,
+        batch: PipelineBatch,
+        result: PipelineResult,
+    ) -> None:
+        """Called on successful batch completion (engine-owned bookkeeping)."""
+        _ = state
+        slice_meta = self.slice_meta(ctx)
+        slice_meta.setdefault("plugin", self.name)
+
+        run_id = ctx.run_id or str(ctx.option("run_id", "unknown"))
+        product = _product_name(ctx, self.name)
+
+        self._span_recorder.record_batch_success(
+            ctx=ctx,
+            batch=batch,
+            result=result,
+            slice_meta=slice_meta,
+            run_id=run_id,
+            product=product,
+        )
+
+    def on_batch_failure(
+        self,
+        ctx: PluginContext,
+        state: PipelineRunState,
+        batch: PipelineBatch,
+        result: PipelineResult,
+    ) -> None:
+        """Called on batch failure (engine-owned bookkeeping)."""
+        _ = state
+        slice_meta = self.slice_meta(ctx)
+        slice_meta.setdefault("plugin", self.name)
+
+        run_id = ctx.run_id or str(ctx.option("run_id", "unknown"))
+        product = _product_name(ctx, self.name)
+
+        self._span_recorder.record_batch_failure(
+            ctx=ctx,
+            batch=batch,
+            error=result.error,
+            slice_meta=slice_meta,
+            run_id=run_id,
+            product=product,
+        )

firecube/ingestor/runtime/batching.py

@@ -0,0 +1,161 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Batch creation logic for ingestion pipelines.
+
+This module provides the `BatchPlanner` service, which is responsible for
+discovering source files, filtering them, grouping them, and chunking them
+into batches for processing.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+from firecube.ingestor.contracts.interfaces import SourceFile
+from firecube.ingestor.types.context import PipelineBatch, PluginContext
+
+log = logging.getLogger("firecube.ingestor.batching")
+
+
+@runtime_checkable
+class BatchPlanHost(Protocol):
+    """Protocol for the host that drives the batch planning process.
+
+    The host provides the source files and the logic for filtering and grouping them.
+    Typically implemented by `BaseIngestor` or a subclass.
+    """
+
+    @property
+    def batch_id_prefix(self) -> str:
+        """prefix for batch IDs (e.g., 'plugin_name_')."""
+        ...
+
+    def discover_source_files(self, ctx: PluginContext) -> Iterable[Any]:
+        """Yield source files (paths, strings, or objects) to be processed."""
+        ...
+
+    def filter_item(self, item: Any, ctx: PluginContext) -> bool:
+        """Return True if the item should be processed, False otherwise."""
+        ...
+
+    def item_size_bytes(self, item: Any) -> int | None:
+        """Return the size of the item in bytes, or None if unknown."""
+        ...
+
+    def get_batch_groups(self, items: list[Any], ctx: PluginContext) -> list[str]:
+        """Return logical groups for a batch of items (e.g., horizon, product).
+
+        MUST be deterministic (return a sorted/stable list) to ensure consistent
+        spans and batch grouping across runs.
+        """
+        ...
+
+
+class BatchPlanner:
+    """Service that creates batches from discovered source files."""
+
+    def create_batches(
+        self, host: BatchPlanHost, ctx: PluginContext, batch_size: int
+    ) -> Iterator[PipelineBatch]:
+        """Yield batches via hooks (iter -> filter -> batch).
+
+        Args:
+            host: The host object providing hooks and source files.
+            ctx: The ingestion context.
+            batch_size: Maximum number of items per batch.
+
+        Yields:
+            PipelineBatch objects containing the items to be processed.
+        """
+        items_iter = host.discover_source_files(ctx)
+
+        current_batch_items: list[Any] = []
+        current_size = 0
+        batch_idx = 0
+
+        def _yield_batch(b_items: list[Any], b_size: int, b_idx: int) -> PipelineBatch:
+            # Deterministic grouping is enforced by the host contract,
+            # but we trust the host to return a stable list.
+            groups = host.get_batch_groups(b_items, ctx)
+
+            # ID Strategy: {prefix}batch_{index:04d} (Simple IDs)
+            batch_id = f"{host.batch_id_prefix}batch_{b_idx:04d}"
+
+            # Safety: Store item URIs in metadata for logging/manifests.
+            # We assume items are str/Path/SourceFile, so str(item) or .uri is reasonable.
+            item_uris = []
+            for item in b_items:
+                if isinstance(item, SourceFile):
+                    item_uris.append(item.uri)
+                else:
+                    item_uris.append(str(item))
+            item_uris_hash = hashlib.sha256(
+                "\n".join(sorted(item_uris)).encode("utf-8")
+            ).hexdigest()
+
+            # Limit metadata size for safety
+            preview_limit = 100
+            item_uris_total = len(item_uris)
+            item_uris_truncated = False
+            if item_uris_total > preview_limit:
+                item_uris_truncated = True
+                item_uris = [*item_uris[:preview_limit], "..."]
+                log.warning(
+                    "Batch metadata item_uris truncated (batch_id=%s total=%d preview_limit=%d).",
+                    batch_id,
+                    item_uris_total,
+                    preview_limit,
+                )
+
+            return PipelineBatch(
+                batch_id=batch_id,
+                data_path=Path(ctx.source),
+                items=list(b_items),  # Copy to avoid mutation issues
+                metadata={
+                    "item_uris": item_uris,
+                    "item_uris_total": item_uris_total,
+                    "item_uris_preview_limit": preview_limit,
+                    "item_uris_truncated": item_uris_truncated,
+                    "files_hash": item_uris_hash,
+                },
+                size_bytes=b_size,
+                files_count=len(b_items),
+                groups=groups,
+            )
+
+        for item in items_iter:
+            if not host.filter_item(item, ctx):
+                continue
+
+            # Calculate size (Best effort)
+            size = host.item_size_bytes(item)
+            if size is None:
+                size = 0
+
+            current_batch_items.append(item)
+            current_size += size
+
+            if len(current_batch_items) >= batch_size:
+                yield _yield_batch(current_batch_items, current_size, batch_idx)
+                batch_idx += 1
+                current_batch_items = []
+                current_size = 0
+
+        if current_batch_items:
+            yield _yield_batch(current_batch_items, current_size, batch_idx)

firecube/ingestor/runtime/configure.py

@@ -0,0 +1,195 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Options-tier configuration and execution-mode selection.
+
+This module is runtime-only and is responsible for splitting the flat
+`ctx.options` dict into strict options tiers (the user-facing CLI option
+hierarchy). These tiers are separate from the storage stack
+(StorageConfig / StorageDriverConfig / StorageBinding / StorageSession),
+which handles I/O wiring independently.
+
+Options tiers:
+  - EngineConfig (always present)
+  - TemplateConfig (optional, per template)
+  - PluginConfig (optional, per plugin)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import uuid
+from enum import Enum, auto
+from typing import Any
+
+import click
+
+from firecube.core.config import DATABASE_DUCKDB_KEYS
+from firecube.core.controlplane import ChunkManager
+from firecube.ingestor.config.engine import (
+    SYSTEM_KEYS,
+    EngineConfig,
+    config_keys,
+    is_experimental_option_key,
+)
+from firecube.ingestor.templates.config import TemplateConfig
+from firecube.ingestor.types.config import PluginConfig
+from firecube.ingestor.types.context import IngestContext
+
+
+class ExecutionMode(Enum):
+    SEQUENTIAL = auto()
+    PIPELINE = auto()
+
+
+class TierConfigurator:
+    """Splits a flat ``ctx.options`` dict into four strict options tiers.
+
+    These are the user-facing CLI option tiers, not the storage stack
+    (StorageConfig / StorageDriverConfig / StorageBinding). Storage I/O
+    wiring is handled separately by the storage layer.
+
+    Options tiers (in precedence order for key ownership):
+        EngineConfig   — pipeline execution settings (workers, batch size,
+                         write mode, workspace cleanup, …).  Always present.
+        TemplateConfig — output-format settings (Zarr chunk shape, Parquet
+                         partition config, …).  Present only if the ingestor
+                         declares ``template_config_class``.
+        DatabaseDuckDB — global ``[database.duckdb]`` defaults owned by core
+                         (DuckDB resource settings shared across plugins).
+        PluginConfig   — plugin-specific domain options (e.g. ``msg_region``,
+                         ``forecast_horizons``).  Present only if the ingestor
+                         declares ``plugin_config_class``.
+
+    Unknown keys (not owned by any tier) raise ``ValueError`` at configure
+    time, enforcing the strict schema contract.
+
+    ``configure()`` is pure: it does not mutate ``ctx``.
+    """
+
+    def __init__(
+        self,
+        template_config_class: type[TemplateConfig] | None,
+        plugin_config_class: type[PluginConfig] | None,
+        *,
+        plugin_name: str,
+    ) -> None:
+        self._template_config_class = template_config_class
+        self._plugin_config_class = plugin_config_class
+        self._plugin_name = plugin_name
+
+    def configure(
+        self, ctx: IngestContext
+    ) -> tuple[EngineConfig, TemplateConfig | None, PluginConfig | None]:
+        options = ctx.options or {}
+
+        if "output_name" in options:
+            raise click.UsageError(
+                "Config key 'output_name' is no longer accepted. "
+                "Use '--product-name' or 'default_product_name' instead.\n"
+                "Edit the CLI invocation or the config file passed via --config-file."
+            )
+
+        engine_keys = config_keys(EngineConfig)
+        template_keys = (
+            config_keys(self._template_config_class) if self._template_config_class else set()
+        )
+        plugin_keys = config_keys(self._plugin_config_class) if self._plugin_config_class else set()
+        duckdb_keys = set(DATABASE_DUCKDB_KEYS)
+
+        known = engine_keys | template_keys | duckdb_keys | plugin_keys | set(SYSTEM_KEYS)
+        unknown = {
+            key for key in options if key not in known and not is_experimental_option_key(key)
+        }
+        if unknown:
+            raise ValueError(
+                f"Unknown configuration options for {self._plugin_name}: "
+                f"{', '.join(sorted(unknown))}. Use --show-options to see valid keys. "
+                f"Experimental options must use the x_ prefix (e.g. x_foo=...)."
+            )
+
+        engine_opts = {k: options[k] for k in engine_keys if k in options}
+        template_opts = {k: options[k] for k in template_keys if k in options}
+        plugin_opts = {k: options[k] for k in plugin_keys if k in options}
+
+        engine_config = EngineConfig.from_options(engine_opts)
+        template_config = (
+            self._template_config_class.from_options(template_opts)
+            if self._template_config_class
+            else None
+        )
+        plugin_config = (
+            self._plugin_config_class.from_options(plugin_opts)
+            if self._plugin_config_class
+            else None
+        )
+
+        return engine_config, template_config, plugin_config
+
+
+def determine_execution_mode(engine_config: EngineConfig) -> ExecutionMode:
+    pipeline_requested = (
+        bool(engine_config.pipeline_parallel) or int(engine_config.pipeline_workers) > 1
+    )
+    if pipeline_requested:
+        return ExecutionMode.PIPELINE
+    return ExecutionMode.SEQUENTIAL
+
+
+def ensure_run_id(*, ctx: IngestContext, plugin_name: str) -> str:
+    """Resolve run_id from context/options or generate a new one (pure)."""
+    if ctx.run_id:
+        return ctx.run_id
+
+    existing = ctx.option("run_id") or ctx.option("manifest_run_id")
+    if existing:
+        return str(existing)
+
+    hostname = os.getenv("HOSTNAME", "host")
+    return f"{plugin_name}-{hostname}-{uuid.uuid4().hex}"
+
+
+def _validate_storage_driver(storage_driver: Any, *, storage_type: str) -> None:
+    driver = getattr(storage_driver, "driver", "fsspec")
+    if driver == "obstore":
+        from firecube.core.filesystem._obstore_compat import require_obstore
+
+        require_obstore()
+        if storage_type not in ("local", "s3"):
+            raise ValueError(
+                f"Storage driver 'obstore' supports 'local' and 's3' storage types, "
+                f"got '{storage_type}'. Use --storage-driver fsspec instead."
+            )
+        log = logging.getLogger(__name__)
+        log.info("Using obstore storage driver for all I/O (experimental)")
+
+
+def _storage_type_for_product_uri(product_uri: Any) -> str:
+    if product_uri.is_remote():
+        return "s3"
+    return "local"
+
+
+def ensure_chunk_manager_config(chunk_manager: ChunkManager, ctx: IngestContext) -> None:
+    """Configure ChunkManager from the ingestion context."""
+    if ctx.storage is not None and ctx.storage.output is not None:
+        product_uri = ctx.storage.output.product.product_uri
+        storage_type = _storage_type_for_product_uri(product_uri)
+        _validate_storage_driver(ctx.storage.output.driver, storage_type=storage_type)
+
+    if not getattr(chunk_manager, "base_uri", None):
+        raise RuntimeError(
+            "ChunkManager requires an output base URI. Provide a target or storage configuration."
+        )

firecube/ingestor/runtime/coverage.py

@@ -0,0 +1,177 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Per-group write coverage tracking for span recording.
+
+Extracted from the MTG FCI plugin as a generic runtime utility.
+Accumulates per-group writes, tracks time bounds, merges contiguous
+index ranges, and produces :class:`SpanCoverage` objects compatible
+with :meth:`SpanRecorder.record_batch_success`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+
+from firecube.core.controlplane.types import SpanCoverage
+
+
+class CoverageTracker:
+    """Collect per-group write coverage entries for span recording.
+
+    Coverage entries emitted by :meth:`build_coverage` follow the schema
+    expected by ingestion runtime span extraction.
+    """
+
+    def __init__(self, time_dim_name: str = "timestamp") -> None:
+        """Initialize empty per-group coverage state."""
+        self._time_dim_name = time_dim_name
+        self._groups: dict[str, dict[str, Any]] = {}
+
+    @staticmethod
+    def _time_to_iso8601(time_val: Any) -> str | None:
+        """Convert supported timestamp-like values to ISO-8601.
+
+        Args:
+            time_val: Timestamp-like value such as ``datetime``,
+                ``np.datetime64``, or an ISO string.
+
+        Returns:
+            ISO-8601 string when conversion is possible, otherwise ``None``.
+        """
+        if time_val is None:
+            return None
+
+        if isinstance(time_val, str):
+            return time_val
+
+        if hasattr(time_val, "isoformat"):
+            try:
+                return str(time_val.isoformat())
+            except Exception:
+                pass
+
+        if isinstance(time_val, np.datetime64):
+            text = np.datetime_as_string(time_val, timezone="UTC")
+            return text if text != "NaT" else None
+
+        return None
+
+    @staticmethod
+    def _merge_index_ranges(indices: set[int]) -> list[list[int]]:
+        """Merge timestamp indices into inclusive contiguous ranges.
+
+        Args:
+            indices: Set of written timestamp indices.
+
+        Returns:
+            Inclusive ``[start, end]`` ranges sorted by start.
+        """
+        if not indices:
+            return []
+
+        sorted_indices = sorted(indices)
+        ranges: list[list[int]] = []
+        start = sorted_indices[0]
+        end = sorted_indices[0]
+
+        for idx in sorted_indices[1:]:
+            if idx == end + 1:
+                end = idx
+                continue
+            ranges.append([start, end])
+            start = idx
+            end = idx
+
+        ranges.append([start, end])
+        return ranges
+
+    def record_write(
+        self,
+        group: str,
+        arrays: list[str],
+        ts_index: int,
+        time_val: Any,
+        aligned: bool,
+    ) -> None:
+        """Record one write operation for coverage accumulation.
+
+        Args:
+            group: Zarr group written by the operation.
+            arrays: Array paths touched by the write.
+            ts_index: Timestamp index written in this operation.
+            time_val: Timestamp value corresponding to ``ts_index``.
+            aligned: Whether this write was chunk-aligned.
+        """
+        entry = self._groups.setdefault(
+            group,
+            {
+                "arrays": set(),
+                "indices": set(),
+                "aligned": True,
+                "time_min": None,
+                "time_max": None,
+            },
+        )
+
+        entry["arrays"].update(arrays)
+        entry["indices"].add(int(ts_index))
+        entry["aligned"] = bool(entry["aligned"]) and bool(aligned)
+
+        iso_time = self._time_to_iso8601(time_val)
+        if iso_time is None:
+            return
+
+        current_min = entry["time_min"]
+        current_max = entry["time_max"]
+        if current_min is None or iso_time < current_min:
+            entry["time_min"] = iso_time
+        if current_max is None or iso_time > current_max:
+            entry["time_max"] = iso_time
+
+    def build_coverage(
+        self,
+        state_array_name: str = "firecube_timestamp_state",
+        time_dim_name: str | None = None,
+    ) -> list[SpanCoverage]:
+        """Build span-coverage entries for all recorded groups.
+
+        Args:
+            state_array_name: Timestamp-state variable name under each group.
+            time_dim_name: Time dimension name to record on each coverage
+                entry; falls back to the tracker's constructor value.
+
+        Returns:
+            List of :class:`SpanCoverage` objects, one per recorded group.
+        """
+        effective_dim = time_dim_name or self._time_dim_name
+        coverage: list[SpanCoverage] = []
+        for group in sorted(self._groups):
+            entry = self._groups[group]
+            coverage.append(
+                SpanCoverage(
+                    group=group,
+                    arrays=sorted(entry["arrays"]),
+                    time_index_ranges=self._merge_index_ranges(entry["indices"]),
+                    aligned=bool(entry["aligned"]),
+                    state_array=f"{group}/{state_array_name}",
+                    state_deleted_value=2,
+                    time_min=entry["time_min"],
+                    time_max=entry["time_max"],
+                    time_dim_name=effective_dim,
+                )
+            )
+        return coverage