firecube 0.1.4__py3-none-any.whl

firecube/ingestor/contracts/interfaces.py

@@ -0,0 +1,235 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Protocols and shared abstractions for the Firecube ingestion SDK.
+
+This module defines *interfaces* (Protocols) used by the ingestion engine and
+plugin implementations. It intentionally contains no runtime logic.
+
+Key concepts:
+- `SourceFile`: a local-or-remote, file-like input abstraction.
+- `Ingestor`: the minimal plugin contract (run an ingestion with an
+  `IngestContext`).
+- `PipelineHost`: the contract implemented by an ingestor that can be executed
+  by the pipeline runner (callbacks + batch processing).
+
+Plugins should prefer importing public types from `firecube.ingestor.api` rather
+than deep-importing from `firecube.ingestor.*`.
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Any, BinaryIO, ClassVar, Protocol, runtime_checkable
+
+from firecube.core.observability.telemetry import IngestionTelemetry
+from firecube.ingestor.types.context import (
+    IngestContext,
+    IngestResult,
+    PipelineBatch,
+    PipelineResult,
+    PipelineRunState,
+    PluginContext,
+    RuntimeIngestContext,
+)
+
+
+@runtime_checkable
+class DatasetProducer(Protocol):
+    """Protocol for batch templates that produce in-memory datasets.
+
+    Tensogram output is selected by this structural capability rather than by
+    requiring a specific Zarr base class.  Implementations share the
+    ``build_dataset`` and ``get_batch_groups`` hooks provided by
+    ``BaseIngestor``.
+    """
+
+    def build_dataset(self, group: str, items: list[Any], ctx: PluginContext) -> Any | None:
+        """Convert a sub-batch into a dataset for ``group``."""
+        ...
+
+    def get_batch_groups(self, items: Sequence[Any], ctx: PluginContext) -> list[str]:
+        """Return logical dataset groups/messages for ``items``."""
+        ...
+
+
+def is_dataset_producer(cls: type) -> bool:
+    """Return True iff *cls* genuinely satisfies the ``DatasetProducer`` contract.
+
+    ``@runtime_checkable`` Protocol's ``issubclass`` / ``isinstance`` checks
+    validate method-name presence only — Python deliberately does NOT check
+    signatures. That is structurally unsound for ``DatasetProducer`` because
+    ``GenericParquetIngestor`` declares
+    ``build_dataset(self, group, batch: PipelineBatch, ctx)`` which collides
+    with the protocol's ``build_dataset(self, group, items: list, ctx)`` by
+    name but is invoked incompatibly by ``TensogramWriteStrategy`` (which
+    passes a ``list`` of timestamps, not a ``PipelineBatch``).
+
+    This helper adds a signature-shape check: the second positional parameter
+    after ``group`` must NOT be named ``batch`` (the parquet convention).
+    Plugins using ``*args`` are accepted.
+    """
+    if not issubclass(cls, DatasetProducer):
+        return False
+    try:
+        sig = inspect.signature(cls.build_dataset)
+    except (TypeError, ValueError):
+        return False
+    positional = [
+        p
+        for p in sig.parameters.values()
+        if p.kind
+        in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
+        )
+    ]
+    if len(positional) < 3:
+        return any(p.kind is inspect.Parameter.VAR_POSITIONAL for p in sig.parameters.values())
+    return positional[2].name != "batch"
+
+
+@runtime_checkable
+class SourceFile(Protocol):
+    """Abstraction for a file-like object that may be local or remote.
+
+    This allows plugins to accept S3 objects as if they were local files,
+    supporting standard python file operations (seek, read).
+    """
+
+    @property
+    def uri(self) -> str:
+        """The canonical URI of the source file."""
+        ...
+
+    def open(self) -> BinaryIO:
+        """Open the file in binary read mode. Must be seekable."""
+        ...
+
+    def local_path(self) -> Path | None:
+        """Return a local Path if one exists efficiently.
+
+        Returns None for remote files (unless they are explicitly materialized).
+        This allows optimizations for libraries that prefer pathlib.Path only when cheap.
+        """
+        ...
+
+
+@runtime_checkable
+class Ingestor(Protocol):
+    """Protocol defining the core contract for an ingestor."""
+
+    def run(self, ctx: IngestContext) -> IngestResult: ...
+
+    def ingest(self, ctx: IngestContext) -> IngestResult: ...
+
+
+class PipelineHost(Protocol):
+    """Protocol for classes that can be executed by the pipeline runner.
+
+    The runtime pipeline executor expects these attributes and lifecycle hooks
+    to exist on the host object (typically an ingestor instance).
+    """
+
+    name: str
+    _log: logging.Logger
+
+    def _create_batches(
+        self, ctx: RuntimeIngestContext, batch_size: int
+    ) -> Iterable[PipelineBatch]: ...
+    def _aggregate_metrics(
+        self, ctx: RuntimeIngestContext, state: PipelineRunState
+    ) -> Mapping[str, Any]: ...
+
+    # Lifecycle hooks called by PipelineRunner
+    def on_pipeline_start(self, ctx: PluginContext, state: PipelineRunState) -> None: ...
+    def on_batch_success(
+        self,
+        ctx: PluginContext,
+        state: PipelineRunState,
+        batch: PipelineBatch,
+        result: PipelineResult,
+    ) -> None: ...
+    def on_batch_failure(
+        self,
+        ctx: PluginContext,
+        state: PipelineRunState,
+        batch: PipelineBatch,
+        result: PipelineResult,
+    ) -> None: ...
+    def finalize_pipeline(
+        self, ctx: RuntimeIngestContext, state: PipelineRunState
+    ) -> IngestResult: ...
+    def _process_batch(self, batch: PipelineBatch, ctx: PluginContext) -> PipelineResult: ...
+    def _resolve_time_dim_name(self) -> str:
+        """Resolve the firecube time-dim coordinate name for this ingestor.
+
+        Default ``"timestamp"``; plugins may override via
+        ``time_dim_name: ClassVar[str]`` on their BaseIngestor subclass.
+        Consumed by the runtime engine to seed coordinate-array chunks
+        in staged mode.
+        """
+        ...
+
+
+@runtime_checkable
+class SlotRangeCapable(Protocol):
+    """Protocol for hosts that support slot-range parallel ingestion.
+
+    The engine dispatches to the slot-range scheduling path when the host
+    structurally satisfies this protocol *and* ``SUPPORTS_SLOT_RANGE_PARALLELISM``
+    is True.  Plugins opt in by setting the class variable and implementing the
+    three methods below; they do not need to subclass any particular template.
+    """
+
+    SUPPORTS_SLOT_RANGE_PARALLELISM: ClassVar[bool]
+
+    def timestamp_to_ts_index(self, group: str, timestamp_val: Any) -> int:
+        """Globally deterministic mapping from the conceptual time axis to ts_index."""
+        ...
+
+    def global_expected_time_count(self, ctx: PluginContext) -> dict[str, int] | None:
+        """Return max ts_index + 1 per group across the planned run."""
+        ...
+
+    def filter_items_to_slot_range(
+        self,
+        items: Sequence[Any],
+        slot_start: int,
+        slot_end: int,
+        ctx: PluginContext,
+    ) -> Sequence[Any]:
+        """Filter source items to those whose ts_index falls in [slot_start, slot_end)."""
+        ...
+
+
+__all__ = [
+    "DatasetProducer",
+    "IngestContext",
+    "IngestResult",
+    "IngestionTelemetry",
+    "Ingestor",
+    "PipelineBatch",
+    "PipelineHost",
+    "PipelineResult",
+    "PipelineRunState",
+    "PluginContext",
+    "RuntimeIngestContext",
+    "SlotRangeCapable",
+    "SourceFile",
+    "is_dataset_producer",
+]

firecube/ingestor/devtools/_templates/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

firecube/ingestor/devtools/_templates/ingestor_base.py.tpl

@@ -0,0 +1,72 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Ingestor for {plugin_name}."""
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import ClassVar
+
+from firecube.core.api import discover_input_files
+from firecube.ingestor.api import (
+    BaseIngestor,
+    PipelineBatch,
+    PipelineResult,
+    PipelineRunState,
+    PluginConfig,
+    PluginContext,
+    RuntimeIngestContext,
+    merge_batch_metrics,
+    register_ingestor,
+)
+
+
+@dataclass
+class {class_name}Config(PluginConfig):
+    """Plugin configuration.
+
+    To accept ``--option key=value`` flags, add dataclass fields here, e.g.::
+
+        my_option: str = "default"
+        batch_threshold: int = 100
+
+    Fields appear under ``[PLUGIN]`` in ``firecube plugins describe {plugin_name}``.
+    See docs/concepts/plugins/create-a-plugin.md for details.
+    """
+
+    pass
+
+
+@register_ingestor("{plugin_name}")
+class {class_name}(BaseIngestor):
+    PRODUCT_NAME: ClassVar[str] = "{plugin_name}"
+    plugin_config_class = {class_name}Config
+
+    def discover_source_files(self, ctx: PluginContext) -> Iterable[str]:
+        # TODO: adjust include_suffixes / sniffing as needed for your input format.
+        files = discover_input_files(ctx.source, recursive=True)
+        return [str(p) for p in files]
+
+    def _process_batch(self, batch: PipelineBatch, ctx: PluginContext) -> PipelineResult:
+        raise NotImplementedError(
+            "{class_name}._process_batch(): implement this hook to process a batch of "
+            "source items and return a PipelineResult with outputs=OutputPaths(primary=...). "
+            "See docs/concepts/plugins/base-ingestor.md for the contract and examples."
+        )
+
+    def _aggregate_metrics(
+        self, ctx: RuntimeIngestContext, state: PipelineRunState
+    ) -> dict[str, object]:
+        return merge_batch_metrics(ctx, state)

firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl

@@ -0,0 +1,66 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Direct Zarr ingestor for {plugin_name}."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import ClassVar
+
+from firecube.ingestor.api import (
+    DirectZarrIngestor,
+    PipelineBatch,
+    PluginConfig,
+    PluginContext,
+    WriteIntent,
+    ZarrArraySpec,
+    ZarrGroupSpec,
+    register_ingestor,
+)
+
+
+@dataclass
+class {class_name}Config(PluginConfig):
+    """Plugin configuration.
+
+    To accept ``--option key=value`` flags, add dataclass fields here.
+    See docs/concepts/plugins/create-a-plugin.md.
+    """
+
+    pass
+
+
+@register_ingestor("{plugin_name}")
+class {class_name}(DirectZarrIngestor):
+    PRODUCT_NAME: ClassVar[str] = "{plugin_name}"
+    plugin_config_class = {class_name}Config
+
+    def zarr_schema(self, ctx: PluginContext) -> list[ZarrGroupSpec]:
+        raise NotImplementedError(
+            "{class_name}.zarr_schema(): implement this hook to declare the Zarr store "
+            "layout. Return a list[ZarrGroupSpec] describing groups, arrays, shapes, "
+            "dtypes, and chunks. The previous scaffold hard-coded fake shape "
+            "(1000, 100, 100) float32 here — you must replace that with your real schema. "
+            "See docs/concepts/plugins/direct-zarr.md for examples."
+        )
+
+    def build_write_intents(
+        self, batch: PipelineBatch, ctx: PluginContext
+    ) -> list[WriteIntent]:
+        raise NotImplementedError(
+            "{class_name}.build_write_intents(): implement this hook to convert a batch "
+            "into a list[WriteIntent] describing region writes, 1-D writes, or timestamp "
+            "writes. Return an empty list to intentionally skip a batch. "
+            "See docs/concepts/plugins/direct-zarr.md for the WriteIntent API."
+        )

firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl

@@ -0,0 +1,59 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic Parquet ingestor for {plugin_name}."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from firecube.ingestor.api import (
+    GenericParquetIngestor,
+    PipelineBatch,
+    PluginConfig,
+    PluginContext,
+    register_ingestor,
+)
+
+
+@dataclass
+class {class_name}Config(PluginConfig):
+    """Plugin configuration.
+
+    To accept ``--option key=value`` flags, add dataclass fields here.
+    See docs/concepts/plugins/create-a-plugin.md.
+    """
+
+    pass
+
+
+@register_ingestor("{plugin_name}")
+class {class_name}(GenericParquetIngestor):
+    PRODUCT_NAME: ClassVar[str] = "{plugin_name}"
+    plugin_config_class = {class_name}Config
+
+    def build_dataset(
+        self, group: str, batch: PipelineBatch, ctx: PluginContext
+    ) -> Any | None:
+        # Return a pyarrow.Table or pandas.DataFrame for this group/batch.
+        # Returning None skips writing for this group/batch (intentional skip).
+        #
+        # If you choose pandas.DataFrame, add `pandas` to your pyproject.toml
+        # dependencies.
+        raise NotImplementedError(
+            "{class_name}.build_dataset(): implement this hook to convert a batch of "
+            "source items into a pyarrow.Table or pandas.DataFrame for the given group. "
+            "Return None to intentionally skip this group/batch. "
+            "See docs/concepts/plugins/generic-parquet.md for examples."
+        )

firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl

@@ -0,0 +1,55 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic Zarr ingestor for {plugin_name}."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+import xarray as xr
+
+from firecube.ingestor.api import (
+    GenericZarrIngestor,
+    PluginConfig,
+    PluginContext,
+    register_ingestor,
+)
+
+
+@dataclass
+class {class_name}Config(PluginConfig):
+    """Plugin configuration.
+
+    To accept ``--option key=value`` flags, add dataclass fields here.
+    See docs/concepts/plugins/create-a-plugin.md.
+    """
+
+    pass
+
+
+@register_ingestor("{plugin_name}")
+class {class_name}(GenericZarrIngestor):
+    PRODUCT_NAME: ClassVar[str] = "{plugin_name}"
+    plugin_config_class = {class_name}Config
+
+    def build_dataset(
+        self, group: str, items: list[Any], ctx: PluginContext
+    ) -> xr.Dataset | None:
+        raise NotImplementedError(
+            "{class_name}.build_dataset(): implement this hook to convert a batch of "
+            "source items into an xarray.Dataset for the given group. Return None to "
+            "intentionally skip writing this group/batch. "
+            "See docs/concepts/plugins/generic-zarr.md for examples."
+        )

firecube/ingestor/devtools/_templates/readme.md.tpl

@@ -0,0 +1,40 @@
+# {start_case_name} Ingestor Plugin
+
+> ⚠️ **This scaffold is incomplete.** You must implement `{hook_summary}` before this plugin will produce output. Until then, ingestion runs will raise `NotImplementedError`.
+
+## What this plugin does
+
+<!-- TODO: write a one-line summary of what {start_case_name} ingests and emits. -->
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+```
+
+## Install into firecube
+
+```bash
+uv run firecube plugins install --editable .
+uv run firecube plugins describe {plugin_name}
+```
+
+`firecube plugins describe` should list `[ENGINE]` options. If you add plugin-specific options via a `PluginConfig` subclass, they will appear under `[PLUGIN]`.
+
+## After you implement the hook
+
+Once `{hook_summary}` is implemented, run:
+
+```bash
+uv run firecube ingest {plugin_name} \
+  --input-data /path/to/your/input \
+  --target file:///tmp/{plugin_name}_out.{output_extension} \
+  --product-name {plugin_name} \
+  --storage-type local \
+  --storage-driver fsspec \
+  --output-format {output_format} \
+  --write-mode {write_mode_default}
+```
+
+For S3 targets, swap `--storage-type local` for `--storage-type s3` and use an `s3://bucket/key.{output_extension}` URI.