firecube 0.1.4__py3-none-any.whl

firecube/cli/chunks/_manager.py

@@ -0,0 +1,161 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import overload
+
+import click
+
+from firecube.cli._ctx import get_config, get_storage_config
+from firecube.cli._product import resolve_product_identity
+from firecube.core.config import StorageConfig
+from firecube.core.controlplane import ChunkManager
+from firecube.core.product.identity import ProductIdentity
+from firecube.core.runtime import identity_from_storage_config
+from firecube.core.storage.binding import StorageBinding
+from firecube.core.storage.driver_config import StorageDriverConfig
+from firecube.core.storage.uri import StorageUri
+
+log = logging.getLogger("firecube.cli")
+
+
+def resolve_manager(
+    ctx: click.Context,
+    workspace: Path | None,
+    product_uri: StorageUri | None = None,
+    *,
+    product_name: str | None = None,
+) -> ChunkManager:
+    """Resolve a ChunkManager for chunk operations using shared config + overrides."""
+    ctx.ensure_object(dict)
+    base_workspace = workspace if workspace is not None else ctx.obj.get("workspace")
+    cfg = get_config(ctx)
+    storage_config: StorageConfig | None = None
+    env_storage_type_present = bool(os.environ.get("FIRECUBE_STORAGE_TYPE"))
+    if cfg.get("storage") or env_storage_type_present:
+        storage_config = get_storage_config(ctx, set_global=False)
+
+    if product_uri is not None and product_name is not None:
+        driver = StorageDriverConfig.from_storage_config_or_default(storage_config)
+        binding = StorageBinding(
+            identity=ProductIdentity.from_uri(
+                uri=product_uri,
+                format="zarr",
+                product_name=product_name,
+            ),
+            driver=driver,
+        )
+        manager = ChunkManager(binding=binding, workspace=base_workspace)
+        ctx.call_on_close(manager.close)
+        log.info("Chunks using product URI target (product_uri=%s)", manager.base_uri)
+        return manager
+
+    base_output = None
+    if product_uri is not None:
+        base_output = product_uri.to_str()
+
+    if base_output is None and storage_config is not None:
+        base_uri_from_config = _base_uri_from_storage_config(storage_config)
+        base_output = base_uri_from_config.to_str() if base_uri_from_config is not None else None
+
+    if base_output is None:
+        msg = (
+            "Chunk operations require a full product URI or a [storage] configuration in config.toml "
+            "(no product URI or storage configuration found)."
+        )
+        log.error(msg)
+        raise click.ClickException(msg)
+
+    base_uri = _storage_uri_from_base(base_output)
+    driver = StorageDriverConfig.from_storage_config_or_default(storage_config)
+    binding = StorageBinding(
+        identity=ProductIdentity.from_uri(
+            base_uri.join("__firecube_controlplane__"),
+            "zarr",
+            product_name="__firecube_controlplane__",
+        ),
+        driver=driver,
+    )
+    manager = ChunkManager(binding=binding, workspace=base_workspace)
+    ctx.call_on_close(manager.close)
+
+    if storage_config is not None:
+        config_identity = identity_from_storage_config(storage_config)
+        bucket = config_identity.product_uri.authority if config_identity is not None else None
+        log.info(
+            "Chunks using storage configuration (storage_type=%s, bucket=%s, base_uri=%s)",
+            storage_config.storage_type,
+            bucket,
+            manager.base_uri,
+        )
+        if not ctx.obj.get("chunks_quiet"):
+            source = "env" if env_storage_type_present else "config"
+            click.echo(
+                f"[chunks] storage: type={storage_config.storage_type} "
+                f"base_uri={manager.base_uri} (from {source})",
+                err=True,
+            )
+    else:
+        log.info("Chunks using product URI base (base_uri=%s)", manager.base_uri)
+
+    return manager
+
+
+def _storage_uri_from_base(raw: str) -> StorageUri:
+    if "://" not in raw:
+        raise click.UsageError(
+            "Product target must resolve to a full URI like 's3://bucket/path' or "
+            f"'file:///abs/path'. Bare paths are no longer supported. Got: {raw!r}"
+        )
+    return StorageUri.parse(raw)
+
+
+def _base_uri_from_storage_config(storage_config: StorageConfig) -> StorageUri | None:
+    identity = identity_from_storage_config(storage_config)
+    return identity.product_uri if identity is not None else None
+
+
+@overload
+def resolve_cli_product(product: str, *, format: str = "zarr") -> tuple[str, StorageUri]: ...
+
+
+@overload
+def resolve_cli_product(product: None, *, format: str = "zarr") -> tuple[None, None]: ...
+
+
+def resolve_cli_product(
+    product: str | None, *, format: str = "zarr"
+) -> tuple[str | None, StorageUri | None]:
+    if product is None:
+        return None, None
+    if "://" not in product:
+        return product, None
+    product_label = product.rstrip("/").rsplit("/", 1)[-1]
+    identity = resolve_product_identity(
+        product, format=format, product_name=product_label, option_name="--product-name"
+    )
+    return identity.product_name, identity.product_uri
+
+
+def storage_config_from_ctx(ctx: click.Context) -> StorageConfig:
+    storage_config_obj: StorageConfig | None = ctx.obj.get("storage_config")
+    if storage_config_obj is None:
+        raise click.ClickException(
+            "Storage configuration not available. Define [storage] in config.toml or use --manifest-only."
+        )
+    return storage_config_obj

firecube/cli/chunks/_runs.py

@@ -0,0 +1,294 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from firecube.cli._product import require_full_uri
+from firecube.cli._shared_options import dry_run_flag, yes_flag
+from firecube.core.controlplane.types import AbandonSweepResult
+from firecube.core.errors import ManifestError
+from firecube.core.storage.uri import StorageUri
+
+from ._manager import resolve_cli_product, resolve_manager
+
+
+@click.group("runs")
+def runs_group() -> None:
+    """manage ingestion run records
+
+    inspect and manage tracked chunk runs."""
+
+
+def _echo_abandon_run_result(*, product_name: str, run_id: str, result: dict[str, object]) -> None:
+    if not result.get("abandoned"):
+        click.echo(f"Run {run_id} already terminal with status={result.get('status')}")
+        return
+    click.echo(f"Abandoned run {run_id} for {product_name}")
+
+
+def _echo_abandon_sweep_result(
+    *,
+    product_name: str,
+    result: AbandonSweepResult,
+    dry_run: bool,
+) -> None:
+    if not result.previewed:
+        click.echo("No stale runs found.")
+        return
+
+    if dry_run:
+        click.echo(f"[dry-run] Would abandon stale runs for {product_name}:")
+        for run_id in result.previewed:
+            click.echo(f"  - {run_id}")
+        return
+
+    click.echo(f"Abandoned stale runs for {product_name}:")
+    if result.abandoned:
+        click.echo("  Abandoned:")
+        for run_id in result.abandoned:
+            click.echo(f"    - {run_id}")
+    if result.skipped_fresh:
+        click.echo("  Skipped fresh:")
+        for run_id in result.skipped_fresh:
+            click.echo(f"    - {run_id}")
+    if result.skipped_already_terminal:
+        click.echo("  Skipped already terminal:")
+        for run_id in result.skipped_already_terminal:
+            click.echo(f"    - {run_id}")
+
+
+@runs_group.command(
+    "list",
+    epilog="""\b
+Examples:
+  # list all runs for a product
+  firecube chunks runs list --product-name <product>
+
+\b
+  # filter by status
+  firecube chunks runs list --product-name <product> --status started
+
+\b
+  # output as JSON
+  firecube chunks runs list --product-name <product> -f json
+
+See also: firecube chunks runs abandon, firecube chunks list
+""",
+)
+@click.option("-n", "--product-name", "product_name", required=True, help="product to inspect")
+@click.option(
+    "--target",
+    "target",
+    default=None,
+    metavar="URI",
+    help=(
+        "Explicit product target URI (file://... or s3://...). "
+        "Overrides config/env backend resolution. Use when --product-name "
+        "alone resolves to the wrong backend (e.g. after moving a product "
+        "or when config points elsewhere)."
+    ),
+)
+@click.option(
+    "--status", default=None, help="filter runs by status (started, complete, failed, abandoned)"
+)
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    help="output format",
+)
+@click.pass_context
+def list_cmd(
+    ctx: click.Context,
+    product_name: str,
+    target: str | None,
+    status: str | None,
+    workspace: Path | None,
+    output_format: str,
+) -> None:
+    """list tracked ingestion runs
+
+    list tracked ingestion runs for a product from the .firecube/ control
+    plane. shows run status (started, complete, failed, abandoned), timestamps,
+    and span counts. use to identify stuck or failed runs before abandoning"""
+    if target is not None:
+        require_full_uri(target, option_name="--target")
+        try:
+            product_uri = StorageUri.parse(target)
+        except ValueError as exc:
+            raise click.UsageError(str(exc)) from exc
+    else:
+        product_name, product_uri = resolve_cli_product(product_name)
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+    runs = manager.list_runs(product=product_name, status=status)
+
+    if output_format == "json":
+        click.echo(
+            json.dumps(
+                [
+                    {
+                        "product": run.product,
+                        "run_id": run.run_id,
+                        "status": run.status,
+                        "started_at": run.started_at,
+                        "updated_at": run.updated_at,
+                        "completed_at": run.completed_at,
+                        "slot_range": (
+                            list(run.slot_range) if run.slot_range is not None else None
+                        ),
+                        "slot_group": run.slot_group,
+                        "events": run.events,
+                        "parts": run.parts,
+                        "stale": run.stale,
+                        "error": run.error,
+                    }
+                    for run in runs
+                ],
+                indent=2,
+            )
+        )
+        return
+
+    if not runs:
+        click.echo(f"No runs found for {product_name}.")
+        return
+
+    click.echo(f"{'Run ID':<36} {'Status':<10} {'State':<8} {'Parts':<5} {'Events':<6}")
+    click.echo("-" * 80)
+    for run in runs:
+        state = "stale" if run.stale else "active"
+        click.echo(f"{run.run_id:<36} {run.status:<10} {state:<8} {run.parts:<5} {run.events:<6}")
+
+
+@runs_group.command(
+    "abandon",
+    epilog="""\b
+Examples:
+  # preview what would be abandoned
+  firecube chunks runs abandon --product-name <product> --run-id <run-id> --reason "crashed" --dry-run
+
+\b
+  # abandon a stuck run in non-interactive context
+  firecube chunks runs abandon --product-name <product> --run-id <run-id> --reason "process crashed" \\
+      --yes-i-really-mean-it
+
+See also: firecube chunks runs list, firecube chunks list
+""",
+)
+@click.option("-n", "--product-name", "product_name", required=True, help="product owning the run")
+@click.option(
+    "--target",
+    "target",
+    default=None,
+    metavar="URI",
+    help=(
+        "Explicit product target URI (file://... or s3://...). "
+        "Overrides config/env backend resolution. Use when --product-name "
+        "alone resolves to the wrong backend (e.g. after moving a product "
+        "or when config points elsewhere)."
+    ),
+)
+@click.option(
+    "--run-id",
+    default=None,
+    help="run identifier (mutually exclusive with --all-stale)",
+)
+@click.option(
+    "--all-stale",
+    is_flag=True,
+    default=False,
+    help=("abandon all stale non-terminal runs for the product (mutually exclusive with --run-id)"),
+)
+@click.option("--reason", required=True, help="operator reason for abandonment")
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@dry_run_flag
+@yes_flag
+@click.pass_context
+def abandon_cmd(
+    ctx: click.Context,
+    product_name: str,
+    target: str | None,
+    run_id: str | None,
+    all_stale: bool,
+    reason: str,
+    workspace: Path | None,
+    dry_run: bool,
+    yes_i_really_mean_it: bool,
+) -> None:
+    """mark a stuck run as abandoned
+
+    mark a non-terminal ingestion run as abandoned to unblock resume. a run
+    stuck in 'started' state blocks future ingestion for that product. use this
+    after confirming the original process is no longer active"""
+    if target is not None:
+        require_full_uri(target, option_name="--target")
+        try:
+            product_uri = StorageUri.parse(target)
+        except ValueError as exc:
+            raise click.UsageError(str(exc)) from exc
+    else:
+        product_name, product_uri = resolve_cli_product(product_name)
+
+    if (run_id is not None) == all_stale:
+        raise click.UsageError("Provide exactly one of --run-id or --all-stale.")
+
+    if all_stale:
+        manager = resolve_manager(
+            ctx, workspace, product_uri=product_uri, product_name=product_name
+        )
+        effective_dry_run = dry_run or not yes_i_really_mean_it
+        result = manager.abandon_stale_runs(
+            product=product_name,
+            reason=reason,
+            dry_run=effective_dry_run,
+        )
+        _echo_abandon_sweep_result(
+            product_name=product_name,
+            result=result,
+            dry_run=effective_dry_run,
+        )
+        return
+
+    assert run_id is not None
+
+    if not dry_run and not yes_i_really_mean_it:
+        if not sys.stdin.isatty():
+            raise click.UsageError(
+                "Confirmation required: re-run with --yes-i-really-mean-it or run interactively."
+            )
+        click.confirm(f"Abandon run '{run_id}'?", abort=True)
+
+    if dry_run:
+        click.echo(
+            f"[dry-run] Would abandon run '{run_id}' for product "
+            f"'{product_name}' (reason: {reason})"
+        )
+        return
+
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+    try:
+        result = manager.abandon_run(product=product_name, run_id=run_id, reason=reason)
+    except ManifestError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    _echo_abandon_run_result(product_name=product_name, run_id=run_id, result=result)

firecube/cli/chunks/_snapshots.py

@@ -0,0 +1,165 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+
+from firecube.cli._shared_options import dry_run_flag
+
+from ._manager import resolve_cli_product, resolve_manager
+
+
+@click.group("snapshots")
+def snapshots_group() -> None:
+    """manage control-plane snapshots
+
+    rebuild derived chunk snapshots."""
+
+
+@snapshots_group.command(
+    "rebuild",
+    epilog="""\b
+Examples:
+  # preview a snapshot rebuild
+  firecube chunks snapshots rebuild --product-name <product> --dry-run
+
+\b
+  # rebuild snapshot for a product
+  firecube chunks snapshots rebuild --product-name <product>
+
+\b
+  # rebuild and output result as JSON
+  firecube chunks snapshots rebuild --product-name <product> -f json
+
+See also: firecube chunks snapshots status, firecube chunks list
+""",
+)
+@click.option(
+    "-n",
+    "--product-name",
+    "product_name",
+    required=True,
+    help="product to rebuild into a fresh chunk snapshot",
+)
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    help="output format",
+)
+@dry_run_flag
+@click.pass_context
+def rebuild_cmd(
+    ctx: click.Context,
+    product_name: str,
+    workspace: Path | None,
+    output_format: str,
+    dry_run: bool,
+) -> None:
+    """rebuild the chunk snapshot
+
+    rebuild the derived chunk snapshot for a product from the chunk event log.
+    snapshots are cached read models; use after the snapshot is missing,
+    corrupt, or very stale to restore fast query performance for chunks list"""
+    product_name, product_uri = resolve_cli_product(product_name)
+
+    if dry_run:
+        click.echo(f"[dry-run] Would rebuild snapshot for product '{product_name}'")
+        return
+
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+    result = manager.rebuild_snapshot(product_name)
+
+    if output_format == "json":
+        click.echo(json.dumps(result, indent=2))
+        return
+
+    if result.get("locked"):
+        click.echo(
+            f"Snapshot rebuild skipped for {product_name}: another rebuild is already in progress."
+        )
+        return
+
+    click.echo(
+        f"Rebuilt snapshot for {product_name}: generation={result.get('generation')} "
+        f"records={result.get('records', 0)}"
+    )
+
+
+@snapshots_group.command(
+    "status",
+    epilog="""\b
+Examples:
+  # check snapshot status for a product
+  firecube chunks snapshots status --product-name <product>
+
+\b
+  # output as JSON
+  firecube chunks snapshots status --product-name <product> -f json
+
+See also: firecube chunks snapshots rebuild, firecube chunks list
+""",
+)
+@click.option(
+    "-n",
+    "--product-name",
+    "product_name",
+    required=True,
+    help="product to check snapshot status for",
+)
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    help="output format",
+)
+@click.pass_context
+def status_cmd(
+    ctx: click.Context,
+    product_name: str,
+    workspace: Path | None,
+    output_format: str,
+) -> None:
+    """show snapshot age and status
+
+    show snapshot age and status for a product's .firecube/ control plane.
+    reports whether the snapshot exists, how many events it covers, and when
+    it was last rebuilt. use to decide whether a rebuild is needed"""
+    product_name, product_uri = resolve_cli_product(product_name)
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+    result = manager.snapshot_status(product_name)
+
+    if output_format == "json":
+        click.echo(json.dumps(result, indent=2))
+        return
+
+    if not result.get("exists"):
+        click.echo(f"No snapshot found for {product_name}")
+        return
+
+    click.echo(f"Product:    {product_name}")
+    click.echo(f"Age:        {result['age_human']}")
+    click.echo(f"Cutoff:     {result.get('completed_before', 'unknown')}")
+    click.echo(f"Generation: {result.get('generation', 'unknown')}")
+    click.echo(f"Records:    {result.get('records', 'unknown')}")

firecube/cli/completion.py

@@ -0,0 +1,73 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from click.shell_completion import get_completion_class
+
+
+@click.command(
+    "completion",
+    context_settings={"help_option_names": ["-h", "--help"]},
+    epilog="""\b
+Examples:
+  # enable completion for bash (add to ~/.bashrc)
+  eval "$(firecube completion bash)"
+
+
+  # enable completion for zsh (add to ~/.zshrc)
+  eval "$(firecube completion zsh)"
+
+
+  # enable completion for fish
+  firecube completion fish | source
+""",
+)
+@click.argument("shell", type=click.Choice(["bash", "zsh", "fish"], case_sensitive=False))
+@click.option(
+    "--output",
+    "output_path",
+    type=click.Path(path_type=Path),
+    help="write the completion script to a file instead of stdout",
+)
+@click.pass_context
+def completion_cmd(ctx: click.Context, shell: str, output_path: Path | None) -> None:
+    """generate shell completion scripts
+
+    Generates a shell completion script for firecube for bash, zsh, or fish.
+    Run the output through eval in your shell profile to enable tab
+    completion for all firecube commands and options.
+    """
+    root = ctx.find_root()
+    # Always generate scripts for the installed console command name.
+    prog_name = "firecube"
+    complete_var = "_FIRECUBE_COMPLETE"
+
+    complete_cls = get_completion_class(shell)
+    if complete_cls is None:
+        raise click.ClickException(f"Unsupported shell: {shell}")
+
+    script = complete_cls(root.command, {}, prog_name, complete_var).source()
+    if output_path is not None:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(script)
+        click.echo(f"Wrote {shell} completion script to {output_path}")
+        return
+    click.echo(script)
+
+
+__all__ = ["completion_cmd"]