firecube 0.1.4__py3-none-any.whl

firecube/ingestor/devtools/scaffolding.py

@@ -0,0 +1,229 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plugin scaffolding generator using uv and standards-compliant metadata."""
+
+import re
+from importlib.resources import files as _resource_files
+from pathlib import Path
+
+from firecube.core.filesystem import ensure_directory
+
+
+def _load_template(name: str) -> str:
+    """Load a scaffold template file by name from the _templates/ package resource."""
+    try:
+        return (
+            _resource_files("firecube.ingestor.devtools._templates")
+            .joinpath(name)
+            .read_text(encoding="utf-8")
+        )
+    except FileNotFoundError as exc:
+        raise RuntimeError(
+            f"Missing scaffold template resource: {name}. Reinstall firecube."
+        ) from exc
+
+
+# PEP 621 compliant pyproject.toml template
+PYPROJECT_TEMPLATE = """[project]
+name = "{package_name}"
+version = "0.1.0"
+description = "Firecube ingestor plugin for {start_case_name}"
+readme = "README.md"
+requires-python = ">=3.12"
+license = {{ text = "{license}" }}
+authors = [
+    {{ name = "{author_name}", email = "{author_email}" }}
+]
+dependencies = [
+{dependencies}
+]
+
+[project.entry-points."firecube.plugins"]
+{plugin_name} = "{import_name}"
+
+# [project.entry-points."firecube.plugin_cli"]
+# {plugin_name} = "{import_name}.plugin_cli:cli"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "ruff",
+    "mypy",
+]
+"""
+
+PLUGIN_INIT_TEMPLATE = """
+from .ingestor import {class_name}
+
+# Importing the package registers the ingestor via @register_ingestor.
+__all__ = ["{class_name}"]
+"""
+
+TEST_INGESTOR_TEMPLATE = '''"""Tests for {class_name}.
+
+This file is intentionally empty. The firecube scaffold cannot guess what your
+plugin does, so it does not generate placeholder tests that might give false
+confidence in coverage. ``pytest`` will collect zero tests from this file
+until you add them.
+
+Test ideas to consider once your hooks are implemented:
+
+  1. Registration check: import {import_name}, then assert {class_name} is
+     present in firecube's AVAILABLE_INGESTORS registry (proves the
+     @register_ingestor decorator fired correctly).
+  2. PRODUCT_NAME assertion: verify {class_name}.PRODUCT_NAME equals
+     "{plugin_name}".
+  3. Hook contract: instantiate {class_name}, invoke your hook(s) with a
+     known input fixture, assert the returned shape / dtype / row count
+     matches expectations.
+  4. Edge cases specific to your input format: empty input, partial input,
+     malformed input.
+
+See docs/concepts/plugins/create-a-plugin.md for full testing guidance.
+"""
+'''
+
+_DEPS_STRING_PER_TEMPLATE: dict[str, str] = {
+    "base": '    "firecube>=0.1.0"',
+    "zarr": '    "firecube>=0.1.0",\n    "xarray"',
+    "parquet": (
+        '    "firecube>=0.1.0",\n'
+        '    "pyarrow",\n'
+        '    # "pandas",  # uncomment if your build_dataset returns pandas.DataFrame'
+    ),
+    "direct_zarr": '    "firecube>=0.1.0",\n    "numpy"',
+}
+
+_README_SUBSTITUTIONS_PER_TEMPLATE: dict[str, dict[str, str]] = {
+    "base": {
+        "hook_summary": "_process_batch()",
+        "output_format": "zarr",
+        "output_extension": "zarr",
+        "write_mode_default": "staged",
+    },
+    "zarr": {
+        "hook_summary": "build_dataset()",
+        "output_format": "zarr",
+        "output_extension": "zarr",
+        "write_mode_default": "staged",
+    },
+    "parquet": {
+        "hook_summary": "build_dataset()",
+        "output_format": "parquet",
+        "output_extension": "parquet",
+        "write_mode_default": "staged",
+    },
+    "direct_zarr": {
+        "hook_summary": "zarr_schema() and build_write_intents()",
+        "output_format": "zarr",
+        "output_extension": "zarr",
+        "write_mode_default": "direct",
+    },
+}
+
+_INGESTOR_TPL_PER_TEMPLATE: dict[str, str] = {
+    "base": "ingestor_base.py.tpl",
+    "zarr": "ingestor_zarr.py.tpl",
+    "parquet": "ingestor_parquet.py.tpl",
+    "direct_zarr": "ingestor_direct_zarr.py.tpl",
+}
+
+
+def to_snake_case(name: str) -> str:
+    name = re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+    return name.replace("-", "_").replace(" ", "_")
+
+
+def to_pascal_case(name: str) -> str:
+    return "".join(word.title() for word in name.replace("-", "_").split("_"))
+
+
+def create_plugin_structure(
+    name: str,
+    target_dir: Path,
+    author_name: str = "Firecube Developer",
+    author_email: str = "dev@example.com",
+    license: str = "MIT",
+    template_type: str = "base",
+) -> Path:
+    """Generate a new plugin project structure."""
+
+    if template_type not in _INGESTOR_TPL_PER_TEMPLATE:
+        raise ValueError(
+            f"Unknown template type: {template_type}. "
+            f"Choices: {list(_INGESTOR_TPL_PER_TEMPLATE.keys())}"
+        )
+
+    plugin_name = to_snake_case(name)
+    class_name = f"{to_pascal_case(name)}Ingestor"
+    package_name = f"firecube-{plugin_name.replace('_', '-')}"
+    import_name = f"firecube_{plugin_name}"
+
+    root = target_dir / package_name
+    src_dir = root / "src" / import_name
+
+    if root.exists():
+        raise FileExistsError(f"Directory {root} already exists")
+
+    ensure_directory(src_dir)
+    ensure_directory(root / "tests")
+
+    # Write pyproject.toml with per-template dependencies
+    (root / "pyproject.toml").write_text(
+        PYPROJECT_TEMPLATE.format(
+            package_name=package_name,
+            start_case_name=to_pascal_case(name),
+            plugin_name=plugin_name,
+            import_name=import_name,
+            author_name=author_name,
+            author_email=author_email,
+            license=license,
+            dependencies=_DEPS_STRING_PER_TEMPLATE[template_type],
+        )
+    )
+
+    # Write template-specific README
+    readme_tpl = _load_template("readme.md.tpl")
+    (root / "README.md").write_text(
+        readme_tpl.format(
+            start_case_name=to_pascal_case(name),
+            plugin_name=plugin_name,
+            **_README_SUBSTITUTIONS_PER_TEMPLATE[template_type],
+        )
+    )
+
+    # Write source files
+    (src_dir / "__init__.py").write_text(PLUGIN_INIT_TEMPLATE.format(class_name=class_name))
+
+    ingestor_tpl = _load_template(_INGESTOR_TPL_PER_TEMPLATE[template_type])
+    (src_dir / "ingestor.py").write_text(
+        ingestor_tpl.format(class_name=class_name, plugin_name=plugin_name)
+    )
+
+    # Create test stub (placeholder comments only — no fake tests)
+    (root / "tests" / "__init__.py").touch()
+    (root / "tests" / "test_ingestor.py").write_text(
+        TEST_INGESTOR_TEMPLATE.format(
+            class_name=class_name,
+            import_name=import_name,
+            plugin_name=plugin_name,
+        )
+    )
+
+    return root

firecube/ingestor/errors.py

@@ -0,0 +1,79 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Exceptions for Firecube Ingestor."""
+
+from firecube.core.errors import (
+    FirecubeError,
+    ManifestError,
+    SchemaDriftError,
+    StorageError,
+)
+
+
+class IngestorError(FirecubeError):
+    """Base exception for all ingestor-layer errors."""
+
+
+class ConfigurationError(IngestorError):
+    """Configuration validation failed."""
+
+
+class SchemaSizeMismatchError(IngestorError):
+    """Raised when an existing Zarr array's shape is smaller than the global expected size.
+
+    Existing arrays mismatch the plan. Either delete them or update the plan to match.
+    """
+
+
+class ResumeConflictError(IngestorError):
+    """Resume/overwrite conflict detected."""
+
+
+class RangeOverlapError(ResumeConflictError):
+    """Raised when a new slot-range invocation overlaps with an active non-terminal run.
+
+    Overlapping ranges risk Zarr chunk-boundary corruption.
+    Abandon the conflicting run first: firecube chunks runs abandon ...
+    """
+
+
+class StagedMetadataError(IngestorError):
+    """Staged metadata seeding failed for an existing staged-write target.
+
+    Raised by ``seed_staged_store_metadata`` (strict mode) when zarr.json
+    metadata cannot be copied from the final target into the temp store.
+    """
+
+
+class WriteIntentRangeError(IngestorError):
+    """Raised when a WriteIntent's ts_index falls outside the assigned slot range.
+
+    This is a correctness violation — the plugin filter is advisory; this error
+    is the mandatory backstop. NEVER silently drop out-of-range intents.
+    """
+
+
+__all__ = [
+    "ConfigurationError",
+    "IngestorError",
+    "ManifestError",
+    "RangeOverlapError",
+    "ResumeConflictError",
+    "SchemaDriftError",
+    "SchemaSizeMismatchError",
+    "StagedMetadataError",
+    "StorageError",
+    "WriteIntentRangeError",
+]

firecube/ingestor/extensions/_binning.py

@@ -0,0 +1,152 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared binning primitives for the gridding extensions.
+
+A *binner* maps irregular ``(lat, lon)`` input samples onto a 1-D **target
+axis** of cells, recording for each usable input point the integer position of
+its target cell. Aggregation then scatters values onto that axis. This pattern
+is grid-agnostic: a regular lat/lon grid (:mod:`.grid`) and a HEALPix grid
+(:mod:`.healpix`) differ only in how the axis and the point->position mapping
+are computed, not in how values are aggregated.
+
+The target axis can be **derived from the data** or **supplied explicitly** by
+the caller. Supplying it explicitly is what lets a plugin pin one fixed axis and
+reuse it across many granules/timesteps (so results share a coordinate and can
+be appended), rather than re-deriving a different axis from each input.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+AGGREGATIONS = ("mean", "min", "max", "first", "last", "median", "any")
+
+
+def as_float_array(value: np.ndarray) -> np.ndarray:
+    arr = np.asarray(value)
+    if arr.dtype.kind == "f":
+        return arr
+    return arr.astype(np.float64, copy=False)
+
+
+def broadcast_latlon(
+    lat: np.ndarray, lon: np.ndarray, data: np.ndarray
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Normalize lat/lon/data shapes to 1D flattened arrays of equal length."""
+    lat_arr = np.asarray(lat)
+    lon_arr = np.asarray(lon)
+    data_arr = np.asarray(data)
+
+    if lat_arr.ndim == 1 and lon_arr.ndim == 1 and data_arr.ndim == 2:
+        # Rectilinear case: data[lat, lon] with 1D coords.
+        lon2, lat2 = np.meshgrid(lon_arr, lat_arr)
+        return lat2.ravel(), lon2.ravel(), data_arr.ravel()
+
+    if lat_arr.shape != lon_arr.shape or lat_arr.shape != data_arr.shape:
+        raise ValueError(
+            "lat, lon, and data must have matching shapes (or lat/lon 1D with data 2D). "
+            f"Got lat={lat_arr.shape}, lon={lon_arr.shape}, data={data_arr.shape}."
+        )
+    return lat_arr.ravel(), lon_arr.ravel(), data_arr.ravel()
+
+
+def aggregate_by_position(
+    *,
+    values_flat: np.ndarray,
+    valid_point_index: np.ndarray,
+    position: np.ndarray,
+    n_targets: int,
+    aggregation: str,
+) -> np.ndarray:
+    """Scatter flattened values onto a 1-D target axis of length ``n_targets``.
+
+    Parameters
+    ----------
+    values_flat:
+        Flattened input values (same flattening as the binner was built from).
+    valid_point_index:
+        Indices into ``values_flat`` for the points the binner kept.
+    position:
+        For each kept point, the target-axis index in ``[0, n_targets)``.
+    n_targets:
+        Length of the output axis.
+    aggregation:
+        One of :data:`AGGREGATIONS`. ``any`` returns a ``uint8`` presence mask;
+        all others return ``float32`` with ``NaN`` in empty cells.
+    """
+    aggregation = str(aggregation).lower()
+    if aggregation not in AGGREGATIONS:
+        raise ValueError(
+            f"Unknown aggregation method: {aggregation}. Options: {', '.join(AGGREGATIONS)}"
+        )
+
+    values_flat = as_float_array(values_flat).ravel()
+    values = values_flat[valid_point_index]
+
+    ok = ~np.isnan(values)
+    if ok.sum() == 0:
+        if aggregation == "any":
+            return np.zeros(n_targets, dtype=np.uint8)
+        return np.full(n_targets, np.nan, dtype=np.float32)
+
+    pos = position[ok]
+    vals = values[ok]
+
+    if aggregation == "any":
+        out = np.zeros(n_targets, dtype=np.uint8)
+        np.logical_or.at(out, pos, vals != 0)
+        return out
+
+    if aggregation == "mean":
+        sums = np.bincount(pos, weights=vals, minlength=n_targets).astype(np.float64, copy=False)
+        counts = np.bincount(pos, minlength=n_targets).astype(np.float64, copy=False)
+        out = np.full(n_targets, np.nan, dtype=np.float32)
+        nonzero = counts > 0
+        out[nonzero] = (sums[nonzero] / counts[nonzero]).astype(np.float32)
+        return out
+
+    if aggregation == "min":
+        out = np.full(n_targets, np.inf, dtype=np.float64)
+        np.minimum.at(out, pos, vals)
+        out[~np.isfinite(out)] = np.nan
+        return out.astype(np.float32)
+
+    if aggregation == "max":
+        out = np.full(n_targets, -np.inf, dtype=np.float64)
+        np.maximum.at(out, pos, vals)
+        out[~np.isfinite(out)] = np.nan
+        return out.astype(np.float32)
+
+    if aggregation in {"first", "last"}:
+        order = np.argsort(pos, kind="stable")
+        pos_sorted = pos[order]
+        vals_sorted = vals[order]
+        if aggregation == "last":
+            pos_sorted = pos_sorted[::-1]
+            vals_sorted = vals_sorted[::-1]
+        _, first_pos = np.unique(pos_sorted, return_index=True)
+        out = np.full(n_targets, np.nan, dtype=np.float32)
+        out[pos_sorted[first_pos]] = vals_sorted[first_pos].astype(np.float32, copy=False)
+        return out
+
+    # median
+    order = np.argsort(pos, kind="stable")
+    pos_sorted = pos[order]
+    vals_sorted = vals[order]
+    unique_pos, start_idx, counts = np.unique(pos_sorted, return_index=True, return_counts=True)
+    out = np.full(n_targets, np.nan, dtype=np.float32)
+    for cell, start, count in zip(unique_pos, start_idx, counts, strict=False):
+        out[int(cell)] = np.median(vals_sorted[start : start + count]).astype(np.float32)
+    return out

firecube/ingestor/extensions/duck.py

@@ -0,0 +1,153 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""DuckDB extension for Firecube Ingestors.
+
+Provides DuckDbMixin for managing DuckDB connection lifecycle.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import threading
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from firecube.ingestor.types.context import PluginContext
+from firecube.ingestor.utils import duckdb_utils
+from firecube.ingestor.utils.duckdb_utils import ensure_table
+
+__all__ = ["DuckDbMixin", "ensure_table"]
+
+if TYPE_CHECKING:
+    import duckdb
+
+
+class DuckDbMixin:
+    """Mixin that manages a thread-local DuckDB connection for batch processing.
+
+    Why thread-local?
+        In parallel pipeline mode, each worker thread calls ``batch_setup`` /
+        ``batch_teardown`` independently.  DuckDB connections are **not**
+        thread-safe for concurrent writes; giving each thread its own in-memory
+        connection avoids locking and eliminates the risk of file-level write
+        conflicts when using a persistent ``*.duckdb`` file.
+
+    Lifecycle per batch (driven by ``BaseIngestor``):
+        ``batch_setup(ctx)``  → opens connection, applies settings, calls
+                                ``prepare_duckdb_schema`` hook.
+        ``batch_teardown(ctx)`` → closes and deletes the thread-local connection.
+
+    MRO cooperation:
+        Both ``batch_setup`` and ``batch_teardown`` call ``super()``
+        cooperatively so that this mixin can be stacked with other mixins
+        (e.g. ``ProductDataMixin``) without silently dropping their setup logic.
+
+    Persistent mode (``duckdb_persist_batches=true``):
+        Workers use an in-memory DB by default.  Persistence can be enabled
+        for debugging or when accumulating rows across batches before a single
+        final export.  Persistent connections use a file under the run's temp
+        workspace; the main thread pre-initialises the schema in
+        ``GenericZarrIngestor.on_pipeline_start`` to avoid races on first write.
+    """
+
+    @property
+    def _db_local(self) -> threading.local:
+        # Lazy-init the threading.local storage so that subclasses with
+        # __slots__ or dataclass-style __init__ don't need to call super().__init__().
+        if not hasattr(self, "_db_local_storage"):
+            self._db_local_storage = threading.local()
+        return self._db_local_storage
+
+    @property
+    def con(self) -> duckdb.DuckDBPyConnection:
+        if not hasattr(self._db_local, "con"):
+            raise RuntimeError("DuckDB connection not initialized for this thread")
+        return self._db_local.con
+
+    @con.setter
+    def con(self, value: duckdb.DuckDBPyConnection) -> None:
+        self._db_local.con = value
+
+    def setup_duckdb(
+        self,
+        workspace: Path | None = None,
+        options: dict[str, Any] | None = None,
+        in_memory: bool = True,
+    ) -> None:
+        """Initialize DuckDB connection with settings (THREAD-LOCAL)."""
+        db_path = None
+        if not in_memory and workspace:
+            product_name = getattr(self, "PRODUCT_NAME", "product")
+            filename = (options or {}).get("duckdb_filename", f"{product_name}.duckdb")
+            db_path = workspace / filename
+
+        self.con = duckdb_utils.open_duckdb(in_memory=in_memory, db_path=db_path)
+
+        if options:
+            duckdb_utils.apply_duckdb_settings(
+                self.con,
+                max_temp_directory_size=options.get("duckdb_max_temp_directory_size"),
+                memory_limit=options.get("duckdb_memory_limit"),
+                threads=options.get("duckdb_threads"),
+                logger=getattr(self, "_log", None),
+            )
+
+    def teardown_duckdb(self) -> None:
+        """Close DuckDB connection (THREAD-LOCAL)."""
+        if hasattr(self._db_local, "con"):
+            with contextlib.suppress(Exception):
+                self.con.close()
+            del self._db_local.con
+
+    def batch_setup(self, ctx: PluginContext) -> None:
+        # Memory vs persist decision:
+        #   ctx.in_memory=True  → always use in-memory (test mode or explicit flag).
+        #   ctx.in_memory=False → default to in-memory unless duckdb_persist_batches=true,
+        #     which writes a per-worker file under the run's temp workspace.
+        persist = ctx.option("duckdb_persist_batches", False)
+        use_memory = True if ctx.in_memory else (not persist)
+        workspace = getattr(self, "_workspace", None)
+
+        self.setup_duckdb(
+            workspace=workspace.temp_root if workspace else None,
+            options=ctx.options,
+            in_memory=use_memory,
+        )
+
+        # Ensure schema exists (whether fresh in-memory or existing persistent)
+        try:
+            self.prepare_duckdb_schema(self.con, ctx)
+        except Exception as e:
+            logger = getattr(self, "_log", None)
+            if logger:
+                logger.warning("Failed to prepare batch DuckDB schema: %s", e)
+            raise
+
+        # Cooperative MI
+        # Optional by design: if no parent hook exists in the MRO, this is a no-op.
+        super_setup = getattr(super(), "batch_setup", None)
+        if callable(super_setup):
+            super_setup(ctx)
+
+    def batch_teardown(self, ctx: PluginContext) -> None:
+        self.teardown_duckdb()
+        # Optional by design: if no parent hook exists in the MRO, this is a no-op.
+        super_teardown = getattr(super(), "batch_teardown", None)
+        if callable(super_teardown):
+            super_teardown(ctx)
+
+    def prepare_duckdb_schema(self, con: duckdb.DuckDBPyConnection, ctx: PluginContext) -> None:
+        """Optional hook to initialize tables in persistent DB mode."""
+        pass