firecube 0.1.4__py3-none-any.whl

firecube/cli/catalog.py

@@ -0,0 +1,196 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CLI helpers for catalog-related operations (e.g. Intake)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import click
+
+from firecube.cli._ctx import get_storage_config
+from firecube.cli._product import resolve_product_identity
+from firecube.cli._shared_options import (
+    product_uri_option,
+    storage_driver_option,
+    storage_type_option,
+)
+from firecube.cli._uri_policy import apply_smart_default, parse_product_uri
+from firecube.core import observability
+from firecube.core.intake import (
+    build_catalog_source_specs,
+    build_intake_catalog,
+    discover_catalog_groups,
+)
+from firecube.core.storage.binding import StorageBinding
+from firecube.core.storage.driver_config import StorageDriverConfig
+from firecube.core.storage.session import StorageSession
+from firecube.ingestor.registry.loader import discover_ingestors
+
+log = logging.getLogger("firecube.cli")
+
+
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+def catalog() -> None:
+    """generate Intake YAML catalogs
+
+    Generate Intake YAML catalogs for Firecube products. Use catalog intake to
+    produce a machine-readable catalog that describes a product's storage
+    layout, groups, and storage options for use with the Intake data discovery
+    library.
+    """
+
+
+@catalog.command(
+    epilog="""\b
+Examples:
+  # generate an Intake catalog for a product
+  firecube catalog intake <plugin> -p <product> -o <path>/catalog.yaml --collection-id <id>
+
+\b
+  # generate without storage_options (for local or public stores)
+  firecube catalog intake <plugin> -p <product> -o <path>/catalog.yaml --collection-id <id> --no-storage-options
+
+See also: firecube plugins list, firecube zarr validate
+""",
+)
+@click.argument("plugin")
+@product_uri_option(tier="inspect")
+@click.option(
+    "-o",
+    "--output",
+    type=str,
+    required=True,
+    help="Output artifact URI (file:///abs/path); s3:// accepted but runtime support coming later",
+)
+@click.option(
+    "--collection-id",
+    required=True,
+    help="explicit collection id for the generated catalog",
+)
+@click.option(
+    "--include-storage-options/--no-storage-options",
+    default=True,
+    show_default=True,
+    help="include generic storage_options using FIRECUBE_* placeholders",
+)
+@storage_driver_option(required=False)
+@storage_type_option(required=False)
+@click.pass_context
+def intake(
+    ctx: click.Context,
+    plugin: str,
+    product: str,
+    output: str,
+    collection_id: str,
+    include_storage_options: bool,
+    storage_driver: str | None,
+    storage_type: str | None,
+) -> None:
+    """generate an Intake product catalog
+
+    Generates an Intake YAML catalog for a supported product store. Discovers
+    dataset groups using the specified plugin and writes a catalog file with
+    source specs and optional storage_options placeholders. Requires
+    --product, --output, and --collection-id.
+    """
+    observability.init_observability("firecube-catalog-intake")
+
+    # Ensure plugins are discovered before we try to instantiate the ingestor.
+    plugins = discover_ingestors()
+    if plugin not in plugins:
+        raise click.ClickException(f"Unknown plugin: {plugin}")
+
+    ingestor_cls = plugins[plugin]
+
+    parsed_uri = parse_product_uri(product)
+    storage_type = apply_smart_default(parsed_uri, storage_type)
+    parsed_output = parse_product_uri(output)
+    if parsed_output.scheme == "s3":
+        raise click.ClickException("Remote artifact output not yet supported")
+    local_output = Path(parsed_output.normalized.removeprefix("file://"))
+    storage_config = get_storage_config(
+        ctx,
+        overrides={"storage_type": storage_type, "storage_driver": storage_driver},
+        cache=False,
+    )
+
+    driver_config = StorageDriverConfig.from_storage_config(storage_config)
+    identity = resolve_product_identity(
+        parsed_uri.normalized,
+        format="zarr",
+        product_name=parsed_uri.normalized,
+    )
+    session = StorageSession(
+        StorageBinding(
+            identity=identity,
+            driver=driver_config,
+        )
+    )
+
+    store_uri = identity.product_uri.to_str()
+
+    # Instantiate ingestor and let Firecube discover/annotate catalog groups.
+    ingestor = ingestor_cls()
+    try:
+        groups = discover_catalog_groups(store_uri, storage_session=session)
+    except Exception as exc:
+        raise click.ClickException(str(exc)) from exc
+    source_specs = build_catalog_source_specs(
+        plugin_name=plugin,
+        product=parsed_uri.normalized,
+        store_uri=store_uri,
+        groups=groups,
+        group_info_resolver=getattr(ingestor, "catalog_group_info", None),
+        storage_session=session,
+    )
+    if not source_specs:
+        raise click.ClickException(
+            f"No catalogable dataset groups found for plugin '{plugin}' under product '{parsed_uri.normalized}'."
+        )
+
+    catalog_name = plugin
+    catalog_description = f"Firecube product '{parsed_uri.normalized}' for plugin '{plugin}'."
+    catalog_dict = build_intake_catalog(
+        catalog_name=catalog_name,
+        catalog_description=catalog_description,
+        collection_id=collection_id,
+        store_uri=store_uri,
+        sources=source_specs,
+        include_storage_options=include_storage_options,
+    )
+
+    # Dump YAML safely; fall back to JSON if PyYAML is unavailable.
+    try:
+        import yaml  # type: ignore[import-not-found]
+
+        local_output.parent.mkdir(parents=True, exist_ok=True)
+        with local_output.open("w", encoding="utf-8") as fh:
+            yaml.safe_dump(catalog_dict, fh, sort_keys=False)
+    except Exception as exc:
+        # As a fallback, write JSON so the user still gets a usable artifact.
+        log.warning("Failed to write YAML catalog (%s); falling back to JSON.", exc)
+        output_json = local_output.with_suffix(".json")
+        output_json.parent.mkdir(parents=True, exist_ok=True)
+        with output_json.open("w", encoding="utf-8") as fh:
+            json.dump(catalog_dict, fh, indent=2)
+        click.echo(
+            f"Intake catalog written as JSON fallback to: {output_json}",
+        )
+        return
+
+    click.echo(f"Intake catalog written to: {local_output}")

firecube/cli/chunks/__init__.py

@@ -0,0 +1,74 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Chunk management commands (firecube chunks)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from firecube.core import observability
+
+from ._claims import claims_group
+from ._delete import delete_cmd, delete_span_cmd
+from ._list import list_cmd
+from ._runs import runs_group
+from ._snapshots import snapshots_group
+
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.option(
+    "--workspace", type=click.Path(path_type=Path), help="local workspace directory to use"
+)
+@click.option(
+    "--quiet",
+    "quiet",
+    is_flag=True,
+    default=False,
+    help="Suppress storage configuration banner.",
+)
+@click.pass_context
+def chunks(
+    ctx: click.Context,
+    workspace: Path | None,
+    quiet: bool,
+) -> None:
+    """inspect and manage chunk records
+
+    Inspect and manage tracked chunk records across all products.
+
+    \b
+    firecube tracks chunk state in a product-local control plane
+    (event log, snapshots, claims). use chunks list to inspect, chunks delete
+    to remove storage and records, and chunks runs to review ingestion run
+    history"""
+    observability.init_observability("firecube-chunks")
+    ctx.ensure_object(dict)
+    if workspace is not None:
+        ctx.obj["workspace"] = workspace
+    ctx.obj["chunks_quiet"] = quiet
+
+
+chunks.add_command(list_cmd)
+chunks.add_command(delete_cmd)
+chunks.add_command(delete_span_cmd)
+chunks.add_command(claims_group)
+chunks.add_command(runs_group)
+chunks.add_command(snapshots_group)
+
+__all__ = ["chunks"]

firecube/cli/chunks/_claims.py

@@ -0,0 +1,229 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from firecube.cli._shared_options import dry_run_flag, yes_flag
+from firecube.core.controlplane.types import ClearSweepResult
+from firecube.core.errors import ClaimConflictError
+
+from ._manager import resolve_cli_product, resolve_manager
+
+
+@click.group("claims")
+def claims_group() -> None:
+    """manage write-coordination claims
+
+    inspect and manage chunk write claims."""
+
+
+@claims_group.command(
+    "list",
+    epilog="""\b
+Examples:
+  # list all claims across all products
+  firecube chunks claims list
+
+\b
+  # list claims for a specific product
+  firecube chunks claims list --product-name <product>
+
+See also: firecube chunks claims clear, firecube chunks runs list
+""",
+)
+@click.option("-n", "--product-name", "product_name", help="filter by product name")
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    help="output format",
+)
+@click.pass_context
+def list_cmd(
+    ctx: click.Context,
+    product_name: str | None,
+    workspace: Path | None,
+    output_format: str,
+) -> None:
+    """list active write claims
+
+    list active write-coordination claims for one or all products. claims are
+    short-lived locks written to .firecube/claims/ to coordinate concurrent
+    writes. stale claims from crashed processes can block ingestion -- use
+    claims clear to release them"""
+    product_name, product_uri = resolve_cli_product(product_name)
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+    claims = manager.list_claims(product=product_name)
+
+    if output_format == "json":
+        click.echo(
+            json.dumps(
+                [
+                    {
+                        "product": claim.product,
+                        "domain": claim.domain,
+                        "owner_id": claim.owner_id,
+                        "claim_path": claim.claim_path,
+                        "acquired_at": claim.acquired_at,
+                        "last_heartbeat_at": claim.last_heartbeat_at,
+                        "heartbeat_interval_s": claim.heartbeat_interval_s,
+                        "stale_threshold_s": claim.stale_threshold_s,
+                        "stale": claim.stale,
+                    }
+                    for claim in claims
+                ],
+                indent=2,
+            )
+        )
+        return
+
+    if not claims:
+        click.echo("No claims found.")
+        return
+
+    click.echo(f"{'Product':<24} {'State':<8} {'Owner':<24} Domain")
+    click.echo("-" * 92)
+    for claim in claims:
+        state = "stale" if claim.stale else "active"
+        click.echo(f"{claim.product:<24} {state:<8} {claim.owner_id:<24} {claim.domain}")
+
+
+@claims_group.command(
+    "clear",
+    epilog="""\b
+Examples:
+  # clear a stale claim (interactive confirmation)
+  firecube chunks claims clear --product-name <product> --domain <domain>
+
+\b
+  # clear a stale claim in non-interactive context
+  firecube chunks claims clear --product-name <product> --domain <domain> --yes-i-really-mean-it
+
+\b
+  # bypass the is_stale check (--force) and confirm (--yes-i-really-mean-it)
+  firecube chunks claims clear --product-name <product> --domain <domain> --force --yes-i-really-mean-it
+
+See also: firecube chunks claims list, firecube chunks runs abandon
+""",
+)
+@click.option(
+    "-n", "--product-name", "product_name", required=True, help="product owning the claim"
+)
+@click.option("--domain", "domain_id", help="full write-domain identifier")
+@click.option(
+    "--all-stale",
+    "all_stale",
+    is_flag=True,
+    default=False,
+    help="clear all stale claims for the product (mutually exclusive with --domain)",
+)
+@click.option("--workspace", type=click.Path(path_type=Path), help="workspace directory override")
+@click.option(
+    "--force",
+    is_flag=True,
+    help="operational bypass: clear even if the claim is still active (skips is_stale check)",
+)
+@dry_run_flag
+@yes_flag
+@click.pass_context
+def clear_cmd(
+    ctx: click.Context,
+    product_name: str,
+    domain_id: str | None,
+    all_stale: bool,
+    workspace: Path | None,
+    force: bool,
+    dry_run: bool,
+    yes_i_really_mean_it: bool,
+) -> None:
+    """clear a blocking write claim
+
+    clear a blocking write-coordination claim for a product and domain.
+    claims are normally released when a writer exits cleanly; use this when a
+    crashed process left a stale claim that blocks ingestion. verify the
+    original writer is no longer running first.
+
+    --force bypasses the is_stale check (operational bypass for active claims).
+    --yes-i-really-mean-it is the confirmation flag, required in non-TTY contexts."""
+    if bool(domain_id) == all_stale:
+        raise click.UsageError("Provide exactly one of --domain or --all-stale.")
+
+    if dry_run and not all_stale:
+        raise click.UsageError("--dry-run is only supported with --all-stale.")
+
+    product_name, product_uri = resolve_cli_product(product_name)
+    manager = resolve_manager(ctx, workspace, product_uri=product_uri, product_name=product_name)
+
+    if all_stale:
+        dry_run = dry_run or not yes_i_really_mean_it
+        result = manager.clear_stale_claims(product=product_name, dry_run=dry_run)
+        _echo_clear_sweep_result(result, dry_run=dry_run)
+        return
+
+    if not yes_i_really_mean_it:
+        if not sys.stdin.isatty():
+            raise click.UsageError(
+                "Confirmation required: re-run with --yes-i-really-mean-it or run interactively."
+            )
+        click.confirm("Clear claims?", abort=True)
+
+    if domain_id is None:
+        raise click.UsageError("Provide --domain when not using --all-stale.")
+    try:
+        cleared = manager.clear_claim(product=product_name, domain_id=domain_id, force=force)
+    except ClaimConflictError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    if not cleared:
+        raise click.ClickException(f"No claim found for product={product_name} domain={domain_id}")
+
+    click.echo(f"Cleared claim for {domain_id}")
+
+
+def _echo_clear_sweep_result(result: ClearSweepResult, *, dry_run: bool) -> None:
+    if not result.previewed:
+        click.echo("No stale claims found")
+        return
+
+    click.echo("Previewed stale claims:")
+    for domain_id in result.previewed:
+        click.echo(f"  - {domain_id}")
+
+    if dry_run:
+        click.echo("Dry run only; no claims cleared.")
+    else:
+        click.echo("Cleared stale claims:")
+        if result.cleared:
+            for domain_id in result.cleared:
+                click.echo(f"  - {domain_id}")
+        else:
+            click.echo("  - none")
+
+    click.echo(f"Skipped fresh claims: {len(result.skipped_fresh)}")
+    if result.skipped_fresh:
+        for domain_id in result.skipped_fresh:
+            click.echo(f"  - {domain_id}")
+    click.echo(f"Skipped missing claims: {len(result.skipped_missing)}")
+    if result.skipped_missing:
+        for domain_id in result.skipped_missing:
+            click.echo(f"  - {domain_id}")

firecube/cli/chunks/_common.py

@@ -0,0 +1,84 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime
+
+import click
+
+_TIME_RANGE_SEPARATOR_RE = re.compile(r":(?=\d{4}-\d{2}-\d{2}(?:$|[T\s]))")
+
+
+def confirm_deletion(plan, *, manifest_only: bool = False, storage_only: bool = False) -> bool:
+    """Confirm a deletion operation (CLI-only)."""
+    click.echo("")
+    click.echo("DANGEROUS OPERATION")
+    click.echo(f"About to delete {plan.count:,} tracked chunk records")
+    if plan.total_size > 0:
+        if plan.size_gb >= 1:
+            click.echo(f"Total size: {plan.size_gb:.1f} GB")
+        else:
+            click.echo(f"Total size: {plan.size_mb:.1f} MB")
+    if plan.products_affected:
+        click.echo(f"Products affected: {', '.join(sorted(plan.products_affected))}")
+
+    if manifest_only:
+        click.echo("Mode: tracked-records-only (storage objects will be kept)")
+    elif storage_only:
+        click.echo("Mode: storage-only (tracked chunk records will be kept)")
+    else:
+        click.echo("Mode: delete from storage + tracked chunk records")
+
+    return bool(click.confirm("Continue?", default=False))
+
+
+def parse_meta_filters(values: tuple[str, ...]) -> dict:
+    """Parse repeated --meta key=value filters (value may be JSON)."""
+    meta: dict = {}
+    for item in values:
+        if "=" not in item:
+            raise click.BadParameter(f"Invalid --meta '{item}', expected key=value")
+        key, raw = item.split("=", 1)
+        key = key.strip()
+        raw = raw.strip()
+        if not key:
+            raise click.BadParameter(f"Invalid --meta '{item}', empty key")
+        try:
+            meta[key] = json.loads(raw)
+        except Exception:
+            meta[key] = raw
+    return meta
+
+
+def parse_datetime(value: str | None) -> datetime | None:
+    """Parse a YYYY-MM-DD string into a datetime."""
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value, "%Y-%m-%d")
+    except ValueError as exc:
+        raise click.BadParameter(f"Invalid date format: {value}. Use YYYY-MM-DD") from exc
+
+
+def parse_time_range(value: str | None) -> tuple[str, str] | None:
+    """Parse 'START:END' into (start_iso, end_iso). Returns None if value is None/empty."""
+    if not value:
+        return None
+    parts = _TIME_RANGE_SEPARATOR_RE.split(value, maxsplit=1)
+    if len(parts) != 2 or not parts[0].strip() or not parts[1].strip():
+        raise click.BadParameter(f"Expected START:END format, got: {value!r}")
+    return parts[0].strip(), parts[1].strip()