firecube 0.1.4__py3-none-any.whl

firecube/ingestor/types/result_metrics.py

@@ -0,0 +1,386 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Typed result metrics and outputs for ingestion results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from firecube.core.controlplane import SpanCoverage
+
+
+@dataclass(slots=True)
+class StorageMetrics:
+    """Typed storage summary from plugin results."""
+
+    path: str | None = None
+    bytes: int = 0
+    files: int = 0
+    duration_s: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Render storage metrics as a compatibility dictionary."""
+        return {
+            "path": self.path,
+            "bytes": self.bytes,
+            "files": self.files,
+            "duration_s": self.duration_s,
+        }
+
+
+@dataclass(slots=True)
+class PipelineMetrics:
+    """Typed pipeline metrics from plugin results."""
+
+    duration_pipeline_s: float = 0.0
+    rows_processed: int | None = None
+    rows_ingested: int | None = None
+    coverage: list[SpanCoverage] = field(default_factory=list)
+    duration_upload_s: float = 0.0
+    duration_total_s: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Render pipeline metrics as a compatibility dictionary."""
+        return {
+            "duration_pipeline_s": self.duration_pipeline_s,
+            "rows_processed": self.rows_processed,
+            "rows_ingested": self.rows_ingested,
+            "coverage": list(self.coverage),
+            "duration_upload_s": self.duration_upload_s,
+            "duration_total_s": self.duration_total_s,
+        }
+
+
+@dataclass(slots=True)
+class ResultMetrics:
+    """Shared metrics shape used by both batch- and run-level results."""
+
+    write_mode: str | None = None
+    storage: StorageMetrics | None = None
+    pipeline: PipelineMetrics | None = None
+    storage_handled: bool = False
+    _compat: dict[str, Any] = field(default_factory=dict, init=False, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        """Seed compatibility mapping state from typed fields when needed."""
+        if self._compat:
+            return
+        if self.write_mode is not None:
+            self._compat["write_mode"] = self.write_mode
+        if self.storage is not None:
+            self._compat["storage"] = self.storage.to_dict()
+        if self.pipeline is not None:
+            self._compat["pipeline"] = self.pipeline.to_dict()
+        if self.storage_handled:
+            self._compat["storage_handled"] = self.storage_handled
+
+    def to_dict(self) -> dict[str, Any]:
+        """Render typed fields as a public dict (no private ``_compat`` leak)."""
+        rendered: dict[str, Any] = {}
+        if self.write_mode is not None:
+            rendered["write_mode"] = self.write_mode
+        if self.storage is not None:
+            rendered["storage"] = self.storage.to_dict()
+        if self.pipeline is not None:
+            rendered["pipeline"] = self.pipeline.to_dict()
+        if self.storage_handled:
+            rendered["storage_handled"] = self.storage_handled
+        for key, value in self._compat.items():
+            if key not in rendered:
+                rendered[key] = value
+                continue
+            existing = rendered[key]
+            if isinstance(existing, dict) and isinstance(value, dict):
+                merged = dict(value)
+                merged.update(existing)
+                rendered[key] = merged
+        return rendered
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return a compatibility-mapped metric value by key."""
+        compat = self._compat
+        if key == "write_mode":
+            return self.write_mode if self.write_mode is not None else default
+        if key == "storage_handled":
+            return self.storage_handled
+        if key == "storage":
+            if self.storage is None:
+                return compat.get(key, default)
+            rendered = self.storage.to_dict()
+            compat_storage = compat.get("storage")
+            if isinstance(compat_storage, dict):
+                for k, v in compat_storage.items():
+                    if k not in rendered:
+                        rendered[k] = v
+            compat["storage"] = rendered
+            return rendered
+        if key == "pipeline":
+            if self.pipeline is None:
+                return compat.get(key, default)
+            rendered = self.pipeline.to_dict()
+            compat_pipeline = compat.get("pipeline")
+            if isinstance(compat_pipeline, dict):
+                for k, v in compat_pipeline.items():
+                    if k not in rendered:
+                        rendered[k] = v
+            compat["pipeline"] = rendered
+            return rendered
+        if key in compat:
+            return compat[key]
+        return default
+
+    def __getitem__(self, key: str) -> Any:
+        """Provide mapping-style access for compatibility code paths."""
+        value = self.get(key, None)
+        if value is None and key not in {"storage_handled"}:
+            raise KeyError(key)
+        return value
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        """Update a compatibility metric entry and keep typed fields in sync."""
+        if key == "write_mode":
+            self.write_mode = value
+            self._compat[key] = value
+            return
+        if key == "storage":
+            self.storage = _coerce_storage_metrics(value)
+            self._compat[key] = (
+                value
+                if isinstance(value, dict)
+                else self.storage.to_dict()
+                if self.storage
+                else None
+            )
+            return
+        if key == "pipeline":
+            self.pipeline = _coerce_pipeline_metrics(value)
+            self._compat[key] = (
+                value
+                if isinstance(value, dict)
+                else self.pipeline.to_dict()
+                if self.pipeline
+                else None
+            )
+            return
+        if key == "storage_handled":
+            self.storage_handled = bool(value)
+            self._compat[key] = self.storage_handled
+            return
+        raise KeyError(key)
+
+    def items(self):
+        """Return compatibility key/value pairs."""
+        return self._compat.items() if self._compat else ((key, self[key]) for key in self.keys())
+
+    def values(self):
+        """Return compatibility values."""
+        return self._compat.values() if self._compat else (self[key] for key in self.keys())
+
+    def update(self, other: Any = None, /, **kwargs: Any) -> None:
+        """Update compatibility metrics from another mapping or keyword args."""
+        if other is not None:
+            if hasattr(other, "items"):
+                for key, value in other.items():
+                    self[key] = value
+            else:
+                for key, value in other:
+                    self[key] = value
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def setdefault(self, key: str, default: Any = None) -> Any:
+        """Set a compatibility metric only when missing."""
+        if key in self._compat:
+            return self._compat[key]
+        self[key] = default
+        return self._compat.get(key, default)
+
+    def keys(self) -> tuple[str, ...]:
+        """Return the current compatibility keys in insertion order."""
+        if self._compat:
+            return tuple(self._compat.keys())
+        keys: list[str] = []
+        if self.write_mode is not None:
+            keys.append("write_mode")
+        if self.storage is not None:
+            keys.append("storage")
+        if self.pipeline is not None:
+            keys.append("pipeline")
+        if self.storage_handled:
+            keys.append("storage_handled")
+        return tuple(keys)
+
+    def __iter__(self):
+        """Iterate over compatibility metric keys."""
+        return iter(self.keys())
+
+    def __len__(self) -> int:
+        """Return the number of populated compatibility keys."""
+        return len(self.keys())
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._compat
+
+
+@dataclass(slots=True)
+class OutputPaths:
+    """Shared outputs shape used by both batch- and run-level results."""
+
+    primary: Path | str | None = None
+    zarr: Path | str | None = None
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Return an output path by compatibility key."""
+        if key in {"primary", "output_path", "path"}:
+            return self.primary if self.primary is not None else default
+        if key == "zarr":
+            if self.zarr is not None:
+                return self.zarr
+            if self.primary is not None:
+                return self.primary
+            return default
+        return default
+
+    def __getitem__(self, key: str) -> Any:
+        value = self.get(key, None)
+        if value is None:
+            raise KeyError(key)
+        return value
+
+    def keys(self) -> tuple[str, ...]:
+        """Return the populated output keys in insertion order."""
+        keys: list[str] = []
+        if self.primary is not None:
+            keys.append("primary")
+        if self.zarr is not None:
+            keys.append("zarr")
+        return tuple(keys)
+
+    def __iter__(self):
+        return iter(self.keys())
+
+    def __len__(self) -> int:
+        return len(self.keys())
+
+    def items(self):
+        """Return compatibility key/value pairs."""
+        return ((key, self[key]) for key in self.keys())
+
+
+def _coerce_storage_metrics(value: Any) -> StorageMetrics | None:
+    if value is None or isinstance(value, StorageMetrics):
+        return value
+    if isinstance(value, dict):
+        return StorageMetrics(
+            path=value.get("path"),
+            bytes=int(value.get("bytes", 0) or 0),
+            files=int(value.get("files", 0) or 0),
+            duration_s=float(value.get("duration_s", 0.0) or 0.0),
+        )
+    return None
+
+
+def _coerce_pipeline_metrics(value: Any) -> PipelineMetrics | None:
+    if value is None or isinstance(value, PipelineMetrics):
+        return value
+    if isinstance(value, dict):
+        return PipelineMetrics(
+            duration_pipeline_s=float(value.get("duration_pipeline_s", 0.0) or 0.0),
+            rows_processed=value.get("rows_processed"),
+            rows_ingested=value.get("rows_ingested"),
+            coverage=_coerce_span_coverage_list(value.get("coverage")),
+            duration_upload_s=float(value.get("duration_upload_s", 0.0) or 0.0),
+            duration_total_s=float(value.get("duration_total_s", 0.0) or 0.0),
+        )
+    return None
+
+
+def _coerce_span_coverage_list(value: Any) -> list[SpanCoverage]:
+    if not value:
+        return []
+    spans: list[SpanCoverage] = []
+    for item in value:
+        if isinstance(item, SpanCoverage):
+            spans.append(item)
+        elif isinstance(item, dict) and "group" in item:
+            spans.append(
+                SpanCoverage(
+                    group=item["group"],
+                    arrays=list(item.get("arrays", [])),
+                    time_index_ranges=(
+                        list(item["time_index_ranges"]) if "time_index_ranges" in item else None
+                    ),
+                    aligned=bool(item.get("aligned", True)),
+                    state_array=item.get("state_array"),
+                    state_deleted_value=int(item.get("state_deleted_value", 2)),
+                    time_min=item.get("time_min"),
+                    time_max=item.get("time_max"),
+                    region_spec=item.get("region_spec"),
+                    write_strategy=item.get("write_strategy"),
+                    time_dim_name=item.get("time_dim_name"),
+                )
+            )
+    return spans
+
+
+def _coerce_result_metrics(value: Any) -> ResultMetrics:
+    if value is None:
+        return ResultMetrics()
+    if isinstance(value, ResultMetrics):
+        return value
+    if isinstance(value, dict):
+        pipeline = _coerce_pipeline_metrics(value.get("pipeline"))
+        if pipeline is None and value.get("coverage"):
+            pipeline = PipelineMetrics(coverage=_coerce_span_coverage_list(value.get("coverage")))
+        metrics = ResultMetrics(
+            write_mode=value.get("write_mode"),
+            storage=_coerce_storage_metrics(value.get("storage")),
+            pipeline=pipeline,
+            storage_handled=bool(value.get("storage_handled", False)),
+        )
+        metrics._compat = dict(value)
+        return metrics
+    return ResultMetrics()
+
+
+def _coerce_output_paths(value: Any, *, output_format: str) -> OutputPaths:
+    if isinstance(value, OutputPaths):
+        outputs = value
+    elif isinstance(value, dict):
+        outputs = OutputPaths(
+            primary=value.get("primary")
+            or value.get("output_path")
+            or value.get("path")
+            or value.get(output_format),
+            zarr=value.get("zarr"),
+        )
+    else:
+        outputs = OutputPaths()
+
+    if outputs.zarr is None and output_format == "zarr" and outputs.primary is not None:
+        outputs.zarr = outputs.primary
+    return outputs
+
+
+__all__ = [
+    "OutputPaths",
+    "PipelineMetrics",
+    "ResultMetrics",
+    "StorageMetrics",
+    "_coerce_output_paths",
+    "_coerce_result_metrics",
+]

firecube/ingestor/utils/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

firecube/ingestor/utils/duckdb_utils.py

@@ -0,0 +1,235 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Reusable DuckDB helpers for Firecube ingestion plugins."""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from collections.abc import Iterable
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import duckdb
+
+log = logging.getLogger("firecube.ingestor.utils.duckdb")
+
+_IDENT_PART_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def _quote_ident(identifier: str) -> str:
+    """Safely quote a DuckDB identifier (table/column).
+
+    DuckDB does not support parameterizing identifiers. To prevent SQL injection,
+    we restrict identifiers to a conservative subset and always quote them.
+    """
+    identifier = str(identifier)
+    parts = [p for p in identifier.split(".") if p]
+    if not parts:
+        raise ValueError("Identifier cannot be empty")
+    for part in parts:
+        if not _IDENT_PART_RE.match(part):
+            raise ValueError(f"Invalid identifier: {identifier!r}")
+    return ".".join(f'"{p}"' for p in parts)
+
+
+def _quote_literal(value: str) -> str:
+    """Safely quote a SQL string literal."""
+    return "'" + str(value).replace("'", "''") + "'"
+
+
+def open_duckdb(in_memory: bool = True, db_path: Path | None = None) -> duckdb.DuckDBPyConnection:
+    """Create a DuckDB connection."""
+    import duckdb
+
+    if in_memory:
+        log.debug("Opening in-memory DuckDB instance")
+        return duckdb.connect(":memory:")
+    if not db_path:
+        raise ValueError("db_path must be provided when in_memory=False")
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    log.debug("Opening file-backed DuckDB at %s", db_path)
+    return duckdb.connect(str(db_path))
+
+
+def ensure_table(con: duckdb.DuckDBPyConnection, table: str, schema_sql: str) -> None:
+    """Ensure a table exists using the provided CREATE TABLE statement."""
+    con.execute(schema_sql)
+    log.debug("Ensured table %s exists", table)
+
+
+def truncate_table(con: duckdb.DuckDBPyConnection, table: str) -> None:
+    """Delete existing rows without dropping the table."""
+    con.execute(f"DELETE FROM {_quote_ident(table)}")
+    log.debug("Truncated table %s", table)
+
+
+def deduplicate(
+    con: duckdb.DuckDBPyConnection,
+    table: str,
+    group_by: str,
+    value_columns: str | None = None,
+) -> dict[str, Any]:
+    """
+    Deduplicate rows using a DuckDB GROUP BY statement.
+
+    Args:
+        con: DuckDB connection
+        table: table name to deduplicate
+        group_by: columns to group by (comma-separated)
+        value_columns: optional SQL projection for value columns (defaults to ANY_VALUE for all)
+    """
+    table_ident = _quote_ident(table)
+    total_rows_row = con.sql(f"SELECT COUNT(*) FROM {table_ident}").fetchone()
+    total_rows = total_rows_row[0] if total_rows_row else 0
+    if total_rows == 0:
+        return {"total_rows": 0, "unique_rows": 0, "duplicate_rows": 0, "duplicate_ratio": 0.0}
+
+    group_cols = [c.strip() for c in str(group_by).split(",") if c.strip()]
+    if not group_cols:
+        raise ValueError("group_by must include at least one column")
+    group_expr = ", ".join(_quote_ident(c) for c in group_cols)
+
+    if value_columns:
+        vc = str(value_columns)
+        if ";" in vc or "--" in vc or "/*" in vc or "*/" in vc:
+            raise ValueError("value_columns contains disallowed SQL tokens")
+        projection = f"{group_expr}, {vc}"
+    else:
+        projection = f"{group_expr}, ANY_VALUE(*)"
+
+    con.execute(
+        f"""
+        CREATE TEMPORARY TABLE __dedup AS
+        SELECT {projection}
+        FROM {table_ident}
+        GROUP BY {group_expr}
+        """
+    )
+    unique_rows_row = con.sql("SELECT COUNT(*) FROM __dedup").fetchone()
+    unique_rows = unique_rows_row[0] if unique_rows_row else 0
+    con.execute(f"DELETE FROM {table_ident}")
+    table_cols = [
+        row[1] for row in con.execute(f"PRAGMA table_info({_quote_literal(table)})").fetchall()
+    ]
+    if not table_cols:
+        raise RuntimeError(f"Unable to determine columns for table {table}")
+    col_list = ", ".join(_quote_ident(c) for c in table_cols)
+    con.execute(f"INSERT INTO {table_ident} ({col_list}) SELECT {col_list} FROM __dedup")
+    con.execute("DROP TABLE __dedup")
+
+    duplicate_rows = total_rows - unique_rows
+    ratio = duplicate_rows / total_rows
+    log.debug("Deduplicated table %s: %s duplicates", table, duplicate_rows)
+    return {
+        "total_rows": int(total_rows),
+        "unique_rows": int(unique_rows),
+        "duplicate_rows": int(duplicate_rows),
+        "duplicate_ratio": float(ratio),
+    }
+
+
+def export_parquet(con: duckdb.DuckDBPyConnection, table: str, output_path: Path) -> dict[str, Any]:
+    """Export a table to Parquet."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    start = time.time()
+    con.sql(f"COPY {_quote_ident(table)} TO {_quote_literal(str(output_path))} (FORMAT PARQUET)")
+    duration = time.time() - start
+    size_bytes = output_path.stat().st_size if output_path.exists() else 0
+    log.debug("Exported %s to Parquet (%s bytes)", table, size_bytes)
+    return {"format": "parquet", "path": output_path, "size_b": size_bytes, "duration_s": duration}
+
+
+def apply_duckdb_settings(
+    con: duckdb.DuckDBPyConnection,
+    *,
+    max_temp_directory_size: str | None = None,
+    memory_limit: str | None = None,
+    threads: int | None = None,
+    logger: logging.Logger | None = None,
+) -> dict[str, Any]:
+    """Best-effort apply common DuckDB PRAGMA/SET settings.
+
+    This helper is intentionally optional: plugins that do not use DuckDB can
+    ignore it entirely. Failures are logged and do not raise by default.
+    """
+    logger = logger or log
+    applied: dict[str, Any] = {}
+
+    if max_temp_directory_size:
+        try:
+            con.execute(
+                f"PRAGMA max_temp_directory_size={_quote_literal(str(max_temp_directory_size))}"
+            )
+            applied["max_temp_directory_size"] = str(max_temp_directory_size)
+        except Exception as exc:
+            logger.warning(
+                "Failed to apply DuckDB max_temp_directory_size=%s: %s",
+                max_temp_directory_size,
+                exc,
+            )
+
+    if memory_limit:
+        try:
+            con.execute(f"SET memory_limit={_quote_literal(str(memory_limit))}")
+            applied["memory_limit"] = str(memory_limit)
+        except Exception as exc:
+            logger.warning("Failed to apply DuckDB memory_limit=%s: %s", memory_limit, exc)
+
+    if threads is not None:
+        try:
+            threads_int = int(threads)
+            if threads_int > 0:
+                con.execute(f"SET threads={threads_int}")
+                applied["threads"] = threads_int
+        except Exception as exc:
+            logger.warning("Failed to apply DuckDB threads=%s: %s", threads, exc)
+
+    return applied
+
+
+def ensure_table_schema(
+    con: duckdb.DuckDBPyConnection,
+    *,
+    table: str,
+    create_sql: str,
+    expected_columns: Iterable[str] | None = None,
+    logger: logging.Logger | None = None,
+) -> bool:
+    """Ensure the table columns match the expected schema.
+
+    If `expected_columns` is provided and the existing column order differs,
+    the table is dropped and re-created from `create_sql`.
+
+    Returns True when a recreate happened.
+    """
+    logger = logger or log
+    if expected_columns is None:
+        return False
+
+    expected_list = [str(c) for c in expected_columns]
+    try:
+        info = con.execute(f"PRAGMA table_info({_quote_literal(table)})").fetchall()
+    except Exception:
+        info = []
+    existing = [row[1] for row in info] if info else []
+    if existing and existing != expected_list:
+        con.execute(f"DROP TABLE {_quote_ident(table)}")
+        con.execute(create_sql)
+        logger.warning("Recreated DuckDB table %s due to schema mismatch", table)
+        return True
+    return False

firecube/ingestor/validation.py

@@ -0,0 +1,55 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Validation logic for ingestor configuration contracts."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def validate_config_collisions(
+    base: dict[str, Any], new: dict[str, Any], scope: str = "Configuration"
+) -> None:
+    """Check for key collisions between two configuration dictionaries.
+
+    Args:
+        base: The existing configuration dictionary.
+        new: The new configuration dictionary to merge or check.
+        scope: A descriptive name for the scope being validated (e.g., "Engine options").
+
+    Raises:
+        TypeError: If a key in `new` already exists in `base`.
+    """
+    intersection = set(base.keys()) & set(new.keys())
+    if intersection:
+        keys = ", ".join(sorted(str(k) for k in intersection))
+        raise TypeError(f"Configuration key collision ({scope}): {keys}")
+
+
+def declares_new_abstract_member(cls: type[Any]) -> bool:
+    """Return True when a subclass explicitly introduces an abstract hook."""
+    return any(getattr(member, "__isabstractmethod__", False) for member in cls.__dict__.values())
+
+
+def validate_product_name_contract(cls: type[Any]) -> None:
+    """Require PRODUCT_NAME on concrete plugin subclasses at class definition time."""
+    product_name = getattr(cls, "PRODUCT_NAME", None)
+    if declares_new_abstract_member(cls) or (isinstance(product_name, str) and product_name):
+        return
+    raise TypeError(
+        f"{cls.__module__}.{cls.__name__} must declare PRODUCT_NAME: ClassVar[str] "
+        f"(non-empty string). This is the canonical logical name used in .firecube/ "
+        f"control-plane records, manifests, and telemetry. Add: PRODUCT_NAME = 'my_product_name'"
+    )