firecube 0.1.4__py3-none-any.whl

firecube/cli/_slot_env.py

@@ -0,0 +1,102 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""K8s/env slot-range discovery with explicit precedence rules.
+
+Precedence (highest to lowest):
+1. CLI --slot-start/--slot-end (already parsed; passed as cli_slot_start/cli_slot_end)
+2. FIRECUBE_SLOT_START + FIRECUBE_SLOT_END env vars (both required)
+3. JOB_COMPLETION_INDEX + (--slot-size OR FIRECUBE_SLOT_SIZE) -> slot_start = index * size
+4. None (single-pod mode)
+
+Slot group precedence is independent:
+1. CLI --slot-group
+2. FIRECUBE_SLOT_GROUP
+3. None
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+import click
+
+log = logging.getLogger(__name__)
+
+
+def _parse_int(value: str, *, name: str) -> int:
+    try:
+        return int(value)
+    except ValueError as exc:
+        raise click.UsageError(f"{name} must be an integer, got {value!r}") from exc
+
+
+def resolve_slot_range_from_env(
+    cli_slot_start: int | None,
+    cli_slot_end: int | None,
+    cli_slot_size: int | None,
+    cli_slot_group: str | None = None,
+) -> tuple[int | None, int | None, str | None]:
+    """Resolve final (slot_start, slot_end, slot_group) from CLI args + environment."""
+    slot_group = cli_slot_group
+    env_slot_group = os.getenv("FIRECUBE_SLOT_GROUP")
+    if slot_group is None and env_slot_group is not None:
+        slot_group = env_slot_group
+
+    if cli_slot_start is not None and cli_slot_end is not None:
+        return cli_slot_start, cli_slot_end, slot_group
+
+    env_slot_start = os.getenv("FIRECUBE_SLOT_START")
+    env_slot_end = os.getenv("FIRECUBE_SLOT_END")
+    if env_slot_start is not None and env_slot_end is not None:
+        start = _parse_int(env_slot_start, name="FIRECUBE_SLOT_START")
+        end = _parse_int(env_slot_end, name="FIRECUBE_SLOT_END")
+        log.info(
+            "slot_range resolved from env vars FIRECUBE_SLOT_START/END: [%s, %s)",
+            start,
+            end,
+        )
+        return start, end, slot_group
+
+    job_completion_index = os.getenv("JOB_COMPLETION_INDEX")
+    if job_completion_index is not None:
+        slot_size = cli_slot_size
+        if slot_size is None:
+            env_slot_size = os.getenv("FIRECUBE_SLOT_SIZE")
+            slot_size = (
+                _parse_int(env_slot_size, name="FIRECUBE_SLOT_SIZE")
+                if env_slot_size is not None
+                else None
+            )
+
+        if slot_size is None:
+            log.warning(
+                "JOB_COMPLETION_INDEX detected but no slot size; parallel mode NOT activated"
+            )
+            return None, None, slot_group
+
+        index = _parse_int(job_completion_index, name="JOB_COMPLETION_INDEX")
+        start = index * slot_size
+        end = (index + 1) * slot_size
+        log.info(
+            "slot_range resolved from JOB_COMPLETION_INDEX=%s * slot_size=%s: [%s, %s)",
+            index,
+            slot_size,
+            start,
+            end,
+        )
+        return start, end, slot_group
+
+    return None, None, slot_group

firecube/cli/_slot_planning.py

@@ -0,0 +1,95 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Reusable slot-planning helpers for firecube zarr slots and parallel ingestion.
+
+Extracted from plan.py to be importable by firecube zarr slots without
+coupling to the legacy plan command.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+
+import click
+
+from firecube.ingestor.api import ZarrGroupSpec
+
+#: Plan JSON schema version. Independent of control-plane SCHEMA_VERSION.
+PLAN_SCHEMA_VERSION = "v1"
+
+
+def _resolve_per_group_slot_sizes(
+    schema: Sequence[ZarrGroupSpec],
+    explicit: int | None,
+) -> dict[str, int]:
+    if explicit is not None:
+        if explicit <= 0:
+            raise click.ClickException(f"--slot-size must be > 0, got {explicit}")
+        explicit = int(explicit)
+
+    per_group: dict[str, int] = {}
+    for group_spec in schema:
+        time_chunks = [
+            int(arr_spec.chunks[0])
+            for arr_spec in group_spec.arrays
+            if arr_spec.chunks is not None and arr_spec.chunks
+        ]
+        if not time_chunks:
+            continue
+        per_group[group_spec.group] = math.lcm(*time_chunks)
+
+    if explicit is not None:
+        invalid = {g: req for g, req in per_group.items() if explicit % req != 0}
+        if invalid:
+            details = "; ".join(
+                f"group '{g}' needs multiple of {req}" for g, req in sorted(invalid.items())
+            )
+            raise click.ClickException(f"--slot-size={explicit} not valid: {details}")
+        return dict.fromkeys(per_group, explicit)
+    return per_group
+
+
+def _chunk_aligned_remaining(
+    remaining: list[tuple[int, int]], slot_size: int, total: int
+) -> tuple[list[tuple[int, int]], list[tuple[int, int]]]:
+    """Split each ``remaining`` interval into chunk-aligned + blocked sub-intervals.
+
+    Returns ``(aligned, blocked)`` where:
+    - ``aligned`` is the executable list of chunk-aligned intervals (Phase 3.4 behavior preserved)
+    - ``blocked`` is the list of sub-intervals that cannot produce any aligned executable range:
+      * pre-alignment prefix when ``start`` is misaligned (e.g. ``[(73, 1000)]`` slot=100 →
+        aligned=[(100, 1000)], blocked=[(73, 100)])
+      * fully-dropped interval when ``aligned_start >= clamped_end`` (e.g. ``[(950, 1000)]``
+        slot=100 total=1000 → aligned=[], blocked=[(950, 1000)])
+
+    Callers MUST treat non-empty ``blocked`` as a fail-closed condition. Phase 3.5 introduced
+    this signature change to close the silent-drop bug surfaced by external review.
+    """
+    aligned: list[tuple[int, int]] = []
+    blocked: list[tuple[int, int]] = []
+    for start, end in remaining:
+        if start % slot_size != 0:
+            aligned_start = ((start // slot_size) + 1) * slot_size
+        else:
+            aligned_start = start
+        clamped_end = min(end, total)
+        if aligned_start < clamped_end:
+            aligned.append((aligned_start, clamped_end))
+            if aligned_start > start:
+                blocked.append((start, aligned_start))
+        else:
+            blocked.append((start, clamped_end))
+    return aligned, blocked

firecube/cli/_typed_options.py

@@ -0,0 +1,181 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Any, get_type_hints
+
+import click
+
+from firecube.cli._errors import UnknownOptionError
+from firecube.cli._shared_options import format_option as display_format_option
+from firecube.cli.plugins.registry import PluginConfigSchemas, resolve_plugin_configs
+from firecube.ingestor.config.coercion import coerce_cli_value
+from firecube.ingestor.config.engine import is_experimental_option_key
+
+__all__ = [
+    "TypedOptionsParam",
+    "coerce_options_for_plugin",
+    "display_format_option",
+    "output_path_option",
+]
+
+
+def output_path_option(
+    required: bool = True,
+    help_text: str = "destination file path",
+) -> Any:
+    """Shared ``-o/--output`` decorator for file destination paths.
+
+    The decorated callback receives the value as ``output_path``. The type is
+    ``click.Path()`` (string path), distinct from ``workspace_option`` which uses
+    ``click.Path(path_type=Path)``.
+
+    Args:
+        required: Whether ``--output`` must be provided.
+        help_text: Help text displayed in ``--help`` output.
+    """
+
+    def decorator(f: Any) -> Any:
+        return click.option(
+            "-o",
+            "--output",
+            "output_path",
+            required=required,
+            type=click.Path(),
+            help=help_text,
+        )(f)
+
+    return decorator
+
+
+# Keys owned by dedicated `firecube ingest` flags. They are valid EngineConfig
+# fields, so they would pass unknown-key validation — but the free-form merge
+# happens after typed-flag resolution and would silently override the explicit
+# flag. Hard-reject at parse time.
+_TYPED_FLAG_OWNED_KEYS: dict[str, str] = {
+    "write_mode": "--write-mode",
+    "slot_start": "--slot-start",
+    "slot_end": "--slot-end",
+    "slot_size": "--slot-size",
+    "slot_group": "--slot-group",
+}
+
+
+class TypedOptionsParam(click.ParamType):
+    """Parse-time validation of ``--option key=value`` pairs against plugin schema."""
+
+    name = "TYPED_OPTION"
+
+    def __init__(self, plugin_name: str | None = None) -> None:
+        """Create a typed option parser.
+
+        Args:
+            plugin_name: Optional plugin name for parse-time schema validation. If omitted,
+                the parser attempts to read ``plugin`` from the Click context.
+        """
+        self._plugin_name = plugin_name
+
+    def convert(
+        self,
+        value: str,
+        param: click.Parameter | None,
+        ctx: click.Context | None,
+    ) -> tuple[str, Any]:
+        if not isinstance(value, str):
+            self.fail("Option must be a key=value string.", param, ctx)
+
+        if "=" not in value:
+            self.fail("Use key=value syntax (e.g., --option pipeline_workers=4).", param, ctx)
+
+        key, raw_value = value.split("=", 1)
+        key = key.strip()
+
+        if not key:
+            self.fail("Option key cannot be empty.", param, ctx)
+        if raw_value == "":
+            self.fail(f"Option value for '{key}' cannot be empty.", param, ctx)
+
+        owning_flag = _TYPED_FLAG_OWNED_KEYS.get(key)
+        if owning_flag is not None:
+            self.fail(
+                f"'{key}' is owned by the typed flag {owning_flag}; "
+                f"pass {owning_flag} instead of --option {key}=...",
+                param,
+                ctx,
+            )
+
+        if is_experimental_option_key(key):
+            return key, raw_value
+
+        plugin_name = self._plugin_name
+        if plugin_name is None and ctx is not None:
+            plugin_name = ctx.params.get("plugin")
+
+        if plugin_name:
+            try:
+                schemas = resolve_plugin_configs(str(plugin_name))
+                valid_keys = _enumerate_all_valid_keys(schemas)
+                if valid_keys and key not in valid_keys:
+                    raise UnknownOptionError(key, str(plugin_name), sorted(valid_keys))
+
+                target_type = _get_field_type(schemas, key)
+                if target_type is not None:
+                    try:
+                        return key, coerce_cli_value(raw_value, target_type, key)
+                    except (TypeError, ValueError) as exc:
+                        self.fail(str(exc), param, ctx)
+            except (ImportError, AttributeError, KeyError):
+                pass
+
+        return key, raw_value
+
+
+def coerce_options_for_plugin(
+    plugin: str,
+    extra_options: tuple[tuple[str, object], ...],
+) -> tuple[tuple[str, object], ...]:
+    """Validate/coerce parsed ``--option`` pairs against a resolved plugin name."""
+    typed_param = TypedOptionsParam(plugin)
+    coerced: list[tuple[str, object]] = []
+    for key, value in extra_options:
+        if isinstance(value, str):
+            coerced.append(typed_param.convert(f"{key}={value}", None, None))
+        else:
+            coerced.append((key, value))
+    return tuple(coerced)
+
+
+def _enumerate_all_valid_keys(schemas: PluginConfigSchemas) -> set[str]:
+    """Gather all valid ``--option`` keys across engine, template, and plugin tiers."""
+    from firecube.ingestor.config.engine import config_keys
+
+    keys: set[str] = set()
+    keys.update(config_keys(schemas.engine))
+    if schemas.template is not None:
+        keys.update(config_keys(schemas.template))
+    if schemas.plugin is not None:
+        keys.update(config_keys(schemas.plugin))
+    return keys
+
+
+def _get_field_type(schemas: PluginConfigSchemas, key: str) -> Any | None:
+    """Return the expected Python annotation for an option key from any tier."""
+    for cls in (schemas.engine, schemas.template, schemas.plugin):
+        if cls is None:
+            continue
+        hints = get_type_hints(cls)
+        if key in hints:
+            return hints[key]
+    return None

firecube/cli/_uri_policy.py

@@ -0,0 +1,110 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import click
+
+from firecube.core.storage.uri import StorageUri
+
+SCHEME_TO_STORAGE_TYPE: Final[Mapping[str, str]] = {"file": "local", "s3": "s3"}
+
+
+@dataclass(frozen=True)
+class ParsedUri:
+    raw: str
+    scheme: str
+    normalized: str
+    local: bool
+
+
+def parse_product_uri(raw: str) -> ParsedUri:
+    """Parse product URI strings accepted at the CLI boundary."""
+    candidate = str(raw or "")
+    if not candidate:
+        raise click.UsageError("Product URI must not be empty.")
+
+    if "://" not in candidate:
+        suggestion = Path(candidate).expanduser().resolve().as_uri()
+        raise click.UsageError(
+            f"URI scheme required (file:// or s3://). Did you mean {suggestion}?"
+        )
+
+    parsed = urlparse(candidate)
+    scheme = parsed.scheme.lower()
+    if scheme == "file" and parsed.path in ("", "/"):
+        raise click.UsageError("file:// URI requires a non-empty path")
+    if scheme == "file":
+        return _parse_file_uri(candidate, parsed.netloc)
+    if scheme == "s3":
+        return _parse_s3_uri(candidate)
+    raise click.UsageError(f"URI scheme '{scheme}' not supported.")
+
+
+def resolve_storage_type(uri: ParsedUri) -> str | None:
+    """Return the storage type matching a parsed product URI, if known."""
+    return SCHEME_TO_STORAGE_TYPE.get(uri.scheme)
+
+
+def validate_uri_storage_coherence(uri: ParsedUri, storage_type: str) -> None:
+    """Raise UsageError when an explicit storage type contradicts the URI scheme."""
+    expected = resolve_storage_type(uri)
+    if expected is None:
+        raise click.UsageError(f"URI scheme '{uri.scheme}' not supported.")
+    if expected == storage_type:
+        return
+
+    alternate = "an s3:// URI" if storage_type == "s3" else "a file:// URI"
+    raise click.UsageError(
+        f"--storage-type '{storage_type}' is incompatible with URI scheme '{uri.scheme}'. "
+        f"Use --storage-type {expected} for {uri.scheme}:// targets, "
+        f"or change --target to {alternate}."
+    )
+
+
+def apply_smart_default(uri: ParsedUri, storage_type: str | None) -> str:
+    """Resolve storage_type: explicit CLI value wins, else inferred from scheme."""
+    if storage_type is not None:
+        validate_uri_storage_coherence(uri, storage_type)
+        return storage_type
+    return SCHEME_TO_STORAGE_TYPE[uri.scheme]
+
+
+def _parse_file_uri(raw: str, host: str) -> ParsedUri:
+    if host:
+        raise click.UsageError(
+            f"URI scheme 'file' with non-local host '{host}' not supported. "
+            "Use file:///path (three slashes, no host)."
+        )
+    try:
+        uri = StorageUri.parse(raw)
+    except ValueError as exc:
+        raise click.UsageError(str(exc)) from exc
+    return ParsedUri(raw=raw, scheme="file", normalized=Path(uri.path).as_uri(), local=True)
+
+
+def _parse_s3_uri(raw: str) -> ParsedUri:
+    try:
+        uri = StorageUri.parse(raw)
+    except ValueError as exc:
+        if "authority (bucket) is required" in str(exc):
+            raise click.UsageError("authority bucket is required for s3 URIs") from exc
+        raise click.UsageError(str(exc)) from exc
+    return ParsedUri(raw=raw, scheme="s3", normalized=uri.to_str(), local=False)