firecube 0.1.4__py3-none-any.whl

firecube/ingestor/registry/loader.py

@@ -0,0 +1,144 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plugin registry and discovery for Firecube ingestors.
+
+Core responsibilities:
+  - Provide the @register_ingestor decorator for PipelinedIngestor subclasses.
+  - Keep an in-memory mapping of available ingestors by name.
+  - Discover built-in plugins under firecube.ingestor.*
+  - Discover external plugins via entry points (group: 'firecube.plugins').
+
+Non-goals:
+
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from importlib.metadata import entry_points
+from types import ModuleType
+from typing import TypeVar
+
+from firecube.ingestor.contracts.interfaces import Ingestor
+
+AVAILABLE_INGESTORS: dict[str, type[Ingestor]] = {}
+_LOADED: bool = False
+_IngestorT = TypeVar("_IngestorT", bound=Ingestor)
+
+
+def register_ingestor(name: str):
+    """Decorator to register a plugin under a given name."""
+
+    def decorator(cls: type[_IngestorT]) -> type[_IngestorT]:
+        # Using runtime protocol check
+        if not issubclass(cls, Ingestor):
+            raise TypeError("Ingestor must satisfy the Ingestor protocol")
+
+        AVAILABLE_INGESTORS[name] = cls
+        cls.name = name  # pyright: ignore[reportAttributeAccessIssue]
+        return cls
+
+    return decorator
+
+
+def get_ingestor(name: str) -> type[Ingestor]:
+    """Retrieve a registered ingestor class by name.
+
+    Args:
+        name: The name used in @register_ingestor.
+
+    Returns:
+        The registered Ingestor class.
+
+    Raises:
+        KeyError: If the ingestor is not found.
+    """
+
+    try:
+        return AVAILABLE_INGESTORS[name]
+    except KeyError as exc:
+        raise KeyError(f"Ingestor '{name}' not registered") from exc
+
+
+def discover_ingestors() -> dict[str, type[Ingestor]]:
+    """Import all plugin modules so that decorators can register them.
+
+    Discovery strategy:
+      1) If already loaded, return cached copy.
+      2) Load entry points from group 'firecube.plugins'.
+         This triggers module imports, ensuring @register_ingestor runs.
+      3) If FIRECUBE_LOAD_BUILTIN_PLUGINS=1 is set, fallback to walking
+         the `firecube.plugins` namespace.
+    """
+    global _LOADED
+    if _LOADED:
+        return AVAILABLE_INGESTORS.copy()
+
+    import os
+
+    # 1. Entry Points (Standard)
+    group = "firecube.plugins"
+    eps = entry_points(group=group)
+
+    from firecube.ingestor.registry.version_compat import warn_if_incompatible
+
+    for ep in eps:
+        try:
+            ep.load()
+        except Exception as e:
+            import logging
+
+            logging.getLogger("firecube.registry").warning(
+                f"Failed to load plugin entry point '{ep.name}': {e}"
+            )
+            continue
+
+        registered = AVAILABLE_INGESTORS.get(ep.name)
+        if registered is not None:
+            warn_if_incompatible(ep.name, registered)
+
+    # 2. Legacy Fallback (Dev-only)
+    if os.environ.get("FIRECUBE_LOAD_BUILTIN_PLUGINS") == "1":
+        try:
+            import firecube.plugins as plugins_pkg  # pyright: ignore[reportMissingImports]
+        except ImportError:
+            plugins_pkg = None
+
+        def _import_submodules(package: ModuleType, prefix: str) -> None:
+            for mod_info in pkgutil.iter_modules(package.__path__):
+                full_name = f"{prefix}.{mod_info.name}"
+                try:
+                    module = importlib.import_module(full_name)
+                    if mod_info.ispkg:
+                        _import_submodules(module, full_name)
+                except Exception:
+                    pass
+
+        if plugins_pkg is not None:
+            _import_submodules(plugins_pkg, plugins_pkg.__name__)
+
+    _LOADED = True
+    return AVAILABLE_INGESTORS.copy()
+
+
+def reset_plugin_discovery_cache() -> None:
+    """Invalidate the plugin discovery cache.
+
+    Used by tests and plugin management CLI commands to force re-discovery.
+    """
+    global _LOADED
+    AVAILABLE_INGESTORS.clear()
+    _LOADED = False

firecube/ingestor/registry/metadata.py

@@ -0,0 +1,97 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Shared helpers for surfacing plugin metadata to CLI and API layers.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from firecube.ingestor.registry.loader import AVAILABLE_INGESTORS, discover_ingestors, get_ingestor
+
+
+@dataclass
+class PluginDescriptor:
+    name: str
+    module: str
+    description: str | None
+    products: dict[str, object]
+    defaults: dict[str, object] | None
+    required_options: dict[str, object] | None
+    product_name: str | None = None
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "name": self.name,
+            "module": self.module,
+            "description": self.description,
+            "products": self.products,
+            "defaults": self.defaults,
+            "required_options": self.required_options,
+            "product_name": self.product_name,
+        }
+
+
+def _ensure_loaded() -> None:
+    discover_ingestors()
+
+
+def list_plugins() -> list[str]:
+    _ensure_loaded()
+    return sorted(AVAILABLE_INGESTORS.keys())
+
+
+def get_plugin_descriptor(plugin: str) -> PluginDescriptor:
+    _ensure_loaded()
+    cls = get_ingestor(plugin)
+    description = (cls.__doc__ or "").strip() or None
+
+    products: dict[str, object] = {}
+    spec = getattr(cls, "spec", None)
+    if spec:
+        for attr in ("product", "variable", "file_glob"):
+            value = getattr(spec, attr, None)
+            if value:
+                products[attr] = value
+
+    defaults = None
+    if hasattr(cls, "plugin_defaults"):
+        defaults = cls.plugin_defaults()  # type: ignore[attr-defined]
+
+    required = None
+    if hasattr(cls, "required_options"):
+        required = cls.required_options()  # type: ignore[attr-defined]
+
+    resolved_product_name = getattr(cls, "PRODUCT_NAME", None)
+    product_name = (
+        resolved_product_name
+        if isinstance(resolved_product_name, str) and resolved_product_name
+        else None
+    )
+
+    return PluginDescriptor(
+        name=plugin,
+        module=cls.__module__,
+        description=description,
+        products=products,
+        defaults=defaults,
+        required_options=required,
+        product_name=product_name,
+    )
+
+
+def list_plugin_descriptors() -> list[PluginDescriptor]:
+    return [get_plugin_descriptor(name) for name in list_plugins()]

firecube/ingestor/registry/version_compat.py

@@ -0,0 +1,95 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Plugin version compatibility check.
+
+A plugin may declare ``firecube_core_min_version`` (string, PEP 440) on its
+registered ingestor class. When ``discover_ingestors()`` discovers a plugin, the
+declared minimum is compared against the running ``firecube.ingestor`` version.
+
+The check is intentionally permissive: a missing declaration is fine, an
+unparseable value is fine, and even an incompatible value only emits a warning.
+The goal is to surface *informational* drift between the core plugin contract
+and out-of-tree plugin builds, not to block ingestion.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from firecube.ingestor import __version__ as CORE_VERSION
+
+_LOG = logging.getLogger("firecube.registry.version_compat")
+
+VERSION_ATTR = "firecube_core_min_version"
+
+
+def _parse(value: str):
+    from packaging.version import InvalidVersion, Version
+
+    try:
+        return Version(value)
+    except InvalidVersion:
+        return None
+
+
+def check_plugin_compatibility(
+    plugin_name: str,
+    plugin_cls: type[Any],
+    *,
+    core_version: str = CORE_VERSION,
+) -> str | None:
+    """Compare a plugin's declared minimum core version against ``core_version``.
+
+    Returns a human-readable warning message when the plugin requires a newer
+    core than the one currently running, otherwise ``None``. The message is
+    purely informational — callers decide whether to log/raise.
+    """
+    declared = getattr(plugin_cls, VERSION_ATTR, None)
+    if declared is None:
+        return None
+    if not isinstance(declared, str):
+        return None
+
+    plugin_min = _parse(declared)
+    core_now = _parse(core_version)
+    if plugin_min is None or core_now is None:
+        return None
+
+    if core_now >= plugin_min:
+        return None
+
+    return (
+        f"Plugin '{plugin_name}' requires firecube.ingestor>={declared}, "
+        f"but {core_version} is installed. The plugin may not work correctly."
+    )
+
+
+def warn_if_incompatible(
+    plugin_name: str,
+    plugin_cls: type[Any],
+    *,
+    core_version: str = CORE_VERSION,
+    logger: logging.Logger | None = None,
+) -> bool:
+    """Emit a warning via ``logger`` when the plugin declares an incompatible minimum.
+
+    Returns True when a warning was emitted, False otherwise.
+    """
+    message = check_plugin_compatibility(plugin_name, plugin_cls, core_version=core_version)
+    if message is None:
+        return False
+    (logger or _LOG).warning(message)
+    return True

firecube/ingestor/runtime/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

firecube/ingestor/runtime/aggregation.py

@@ -0,0 +1,112 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run-level metric aggregation utilities for plugin and engine integration."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from firecube.ingestor.types.context import PipelineRunState, RuntimeIngestContext
+
+_RESERVED_AGGREGATE_METRICS_KEYS = frozenset({"pipeline"})
+
+
+def merge_batch_metrics(ctx: RuntimeIngestContext, state: PipelineRunState) -> dict[str, Any]:
+    """Merge successful batch metrics into one run-level mapping.
+
+    Default policy:
+    - Numeric values are summed.
+    - List values are concatenated.
+    - Other values keep the first successful value.
+    - Zarr coverage is always merged under ``metrics["zarr"]["coverage"]``.
+    """
+    merged_metrics: dict[str, Any] = {}
+    if ctx.output_format == "zarr":
+        merged_metrics["zarr"] = {"coverage": []}
+
+    for res in state.results:
+        if not res.success or not res.metrics:
+            continue
+
+        if "zarr" in res.metrics:
+            cov = res.metrics["zarr"].get("coverage")
+            if cov:
+                if "zarr" not in merged_metrics:
+                    merged_metrics["zarr"] = {"coverage": []}
+                merged_metrics["zarr"]["coverage"].extend(cov)
+
+        for key, value in res.metrics.items():
+            if key == "zarr":
+                continue
+            if isinstance(value, (int, float)):
+                current = merged_metrics.get(key, 0)
+                if isinstance(current, (int, float)):
+                    merged_metrics[key] = current + value
+                else:
+                    merged_metrics[key] = value
+            elif isinstance(value, list):
+                if key not in merged_metrics:
+                    merged_metrics[key] = []
+                if isinstance(merged_metrics[key], list):
+                    merged_metrics[key].extend(value)
+            else:
+                merged_metrics.setdefault(key, value)
+
+    return merged_metrics
+
+
+def normalize_plugin_aggregate_metrics(
+    raw_metrics: Any,
+    *,
+    logger: logging.Logger,
+    plugin_name: str,
+) -> dict[str, Any]:
+    """Normalize plugin aggregate metrics and enforce engine-owned key boundaries."""
+    if raw_metrics is None:
+        return {}
+    if not isinstance(raw_metrics, Mapping):
+        logger.warning(
+            "Plugin '%s' returned non-mapping aggregate metrics (%s); using empty metrics.",
+            plugin_name,
+            type(raw_metrics).__name__,
+        )
+        return {}
+
+    normalized: dict[str, Any] = {}
+    dropped_keys: list[str] = []
+    for key, value in raw_metrics.items():
+        if not isinstance(key, str):
+            dropped_keys.append(repr(key))
+            continue
+        normalized[key] = value
+
+    if dropped_keys:
+        logger.warning(
+            "Plugin '%s' returned non-string aggregate metric keys; ignoring: %s",
+            plugin_name,
+            ", ".join(sorted(dropped_keys)),
+        )
+
+    for reserved_key in _RESERVED_AGGREGATE_METRICS_KEYS:
+        if reserved_key in normalized:
+            logger.warning(
+                "Plugin '%s' returned reserved aggregate metrics key '%s'; ignoring plugin value.",
+                plugin_name,
+                reserved_key,
+            )
+            normalized.pop(reserved_key, None)
+    return normalized