firecube 0.1.4__py3-none-any.whl

firecube/core/product/identity.py

@@ -0,0 +1,76 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from firecube.core.storage.uri import StorageUri
+
+VALID_FORMATS = frozenset({"zarr", "parquet", "tensogram"})
+
+
+@dataclass(frozen=True, slots=True)
+class ProductIdentity:
+    product_name: str
+    product_uri: StorageUri
+    control_root_uri: StorageUri
+    format: str
+
+    def __post_init__(self) -> None:
+        if self.format not in VALID_FORMATS:
+            raise ValueError(
+                f"Invalid format {self.format!r}; expected one of {sorted(VALID_FORMATS)}"
+            )
+
+    @classmethod
+    def from_uri(cls, uri: StorageUri, format: str, product_name: str) -> ProductIdentity:
+        """Create a ProductIdentity from a URI with an explicit product name.
+
+        The product_name must be provided explicitly; it is not derived from the
+        URI basename. Use CLI --product-name, config default_product_name, or
+        plugin PRODUCT_NAME as the source.
+        """
+        if not product_name:
+            raise ValueError(f"product_name is required; cannot be empty. URI: {uri}")
+        control_root = uri.join(".firecube")
+        return cls(
+            product_name=product_name,
+            product_uri=uri,
+            control_root_uri=control_root,
+            format=format,
+        )
+
+
+def ensure_product_uri(base_uri: str, product: str) -> str:
+    """Append `product` to `base_uri` if the last path segment doesn't exactly match.
+
+    Uses URI path parsing to avoid substring false positives (e.g. 'daily' matching
+    'daily_archive').
+    """
+    from firecube.core.uris import parse_uri
+
+    base_uri = str(base_uri or "").rstrip("/")
+    product = str(product or "")
+
+    if not product:
+        return base_uri
+
+    parsed = parse_uri(base_uri)
+    path = parsed.get("path", "").rstrip("/")
+    last_segment = path.rsplit("/", 1)[-1] if "/" in path else path
+    if last_segment == product:
+        return base_uri
+
+    return base_uri + "/" + product

firecube/core/product/resolver.py

@@ -0,0 +1,121 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from pathlib import Path
+
+from firecube.core.product.identity import ProductIdentity, ensure_product_uri
+from firecube.core.storage.uri import StorageUri
+from firecube.core.uris import (
+    is_remote_target,
+    is_windows_absolute_path,
+)
+
+
+class CompletionRoute(StrEnum):
+    DIRECT = "direct"
+    STAGED = "staged"
+
+
+@dataclass(frozen=True, slots=True)
+class WriteModePolicy:
+    name: str
+    resolves_to_workspace: bool
+    seeds_staged_metadata: bool
+    storage_handled_by_engine: bool
+    completion_route: CompletionRoute
+
+
+STAGED_WRITE_MODE = WriteModePolicy(
+    name="staged",
+    resolves_to_workspace=True,
+    seeds_staged_metadata=True,
+    storage_handled_by_engine=False,
+    completion_route=CompletionRoute.STAGED,
+)
+DIRECT_WRITE_MODE = WriteModePolicy(
+    name="direct",
+    resolves_to_workspace=False,
+    seeds_staged_metadata=False,
+    storage_handled_by_engine=True,
+    completion_route=CompletionRoute.DIRECT,
+)
+
+_WRITE_MODE_POLICIES = {
+    STAGED_WRITE_MODE.name: STAGED_WRITE_MODE,
+    DIRECT_WRITE_MODE.name: DIRECT_WRITE_MODE,
+}
+
+
+def write_mode_policy(write_mode: str | WriteModePolicy) -> WriteModePolicy:
+    if isinstance(write_mode, WriteModePolicy):
+        return write_mode
+    try:
+        return _WRITE_MODE_POLICIES[str(write_mode)]
+    except KeyError as exc:
+        valid = ", ".join(sorted(_WRITE_MODE_POLICIES))
+        raise ValueError(f"Unknown write mode {write_mode!r}; expected one of: {valid}") from exc
+
+
+class ProductResolver:
+    @staticmethod
+    def resolve(target: str, format: str, product_name: str) -> ProductIdentity:
+        """Boundary-only: parse target string into typed ProductIdentity."""
+        try:
+            uri = StorageUri.parse(target)
+        except ValueError as e:
+            msg = str(e).lower()
+            if "scheme" in msg or "file://" in msg or "bare paths" in msg:
+                raise ValueError(
+                    "--target must be a full URI like 's3://bucket/path/x.zarr' or "
+                    "'file:///abs/path/x.zarr'. Bare names and relative paths are rejected. "
+                    f"Original error: {e}"
+                ) from e
+            raise
+        return ProductIdentity.from_uri(uri, format=format, product_name=product_name)
+
+
+def resolve_dataset_target(
+    target: str,
+    *,
+    write_mode: str | WriteModePolicy = "staged",
+    temp_root: Path | None = None,
+    direct_base_uri: str | None = None,
+) -> str:
+    """Resolve the URI for a dataset target (Dataset Directory).
+
+    This determines the "Package Root" for Zarr/Parquet output, considering:
+    1. Absolute/Full URIs (s3://, /abs/path) -> Used as-is.
+    2. Relative paths + Staged mode -> Anchored to temporary workspace.
+    3. Relative paths + Direct mode -> Anchored to the typed product URI's parent.
+    """
+    if (
+        is_remote_target(target)
+        or Path(target).is_absolute()  # firecube: STORAGE-URI
+        or target.startswith("file://")
+        or is_windows_absolute_path(target)
+    ):
+        return target
+
+    if write_mode_policy(write_mode).resolves_to_workspace:
+        workspace = temp_root if temp_root else Path.cwd()
+        return str(workspace / target)
+
+    if direct_base_uri:
+        return ensure_product_uri(direct_base_uri, target)
+
+    return target

firecube/core/product/target.py

@@ -0,0 +1,120 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from firecube.core.storage.driver_config import (
+    StorageDriverConfig,
+)
+from firecube.core.storage.uri import StorageUri
+from firecube.core.uris import parse_uri
+
+_OBSTORE_SUPPORTED_PROTOCOLS = {"s3", "file", "gs", "az"}
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedProduct:
+    product_name: str
+    product_uri: StorageUri
+    output_base_uri: StorageUri
+    control_root_uri: StorageUri
+    format: str
+
+
+class ProductTarget:
+    @classmethod
+    def resolve(
+        cls,
+        target_arg: str,
+        driver_config: StorageDriverConfig,
+        *,
+        product_name: str,
+        plugin_default_format: str | None = None,
+        default_base_uri: StorageUri | None = None,
+    ) -> ResolvedProduct:
+        target_value = str(target_arg or "")
+        if not product_name:
+            raise ValueError("product_name is required; cannot be empty.")
+        output_format = str(plugin_default_format or "zarr")
+        parsed = parse_uri(target_value)
+        protocol = parsed["protocol"]
+
+        if cls._is_bare_name_target(target_value, protocol):
+            if default_base_uri is None:
+                raise ValueError(
+                    "Bare-name targets require either a fully-qualified URI/path or default_base_uri."
+                )
+
+            product_uri = default_base_uri.join(target_value)
+            cls._validate_driver_protocol(driver_config, product_uri.protocol)
+            return ResolvedProduct(
+                product_name=product_name,
+                product_uri=product_uri,
+                output_base_uri=default_base_uri,
+                control_root_uri=cls._control_root(product_uri),
+                format=output_format,
+            )
+
+        if protocol != "file":
+            product_uri = StorageUri.parse(target_value)
+            cls._validate_driver_protocol(driver_config, product_uri.protocol)
+            return ResolvedProduct(
+                product_name=product_name,
+                product_uri=product_uri,
+                output_base_uri=product_uri.parent(),
+                control_root_uri=cls._control_root(product_uri),
+                format=output_format,
+            )
+
+        if target_value.startswith("file://"):
+            product_uri = StorageUri.parse(target_value)
+            resolved_path = Path(product_uri.path).resolve()
+        else:
+            resolved_path = Path(target_value).resolve()
+        product_uri = StorageUri.from_local_path(resolved_path)
+        cls._validate_driver_protocol(driver_config, product_uri.protocol)
+        return ResolvedProduct(
+            product_name=product_name,
+            product_uri=product_uri,
+            output_base_uri=product_uri.parent(),
+            control_root_uri=cls._control_root(product_uri),
+            format=output_format,
+        )
+
+    @staticmethod
+    def _is_bare_name_target(target_arg: str, protocol: str) -> bool:
+        return (
+            protocol == "file"
+            and "/" not in target_arg
+            and "\\" not in target_arg
+            and not target_arg.startswith("file://")
+        )
+
+    @staticmethod
+    def _control_root(product_uri: StorageUri) -> StorageUri:
+        return product_uri.join(".firecube")
+
+    @staticmethod
+    def _validate_driver_protocol(
+        driver_config: StorageDriverConfig,
+        protocol: str,
+    ) -> None:
+        if driver_config.driver == "obstore" and protocol not in _OBSTORE_SUPPORTED_PROTOCOLS:
+            raise ValueError(
+                f"Storage driver 'obstore' does not support protocol '{protocol}'; "
+                "use --storage-driver=fsspec instead."
+            )

firecube/core/runtime.py

@@ -0,0 +1,187 @@
+# Copyright 2025-2026 EUMETSAT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runtime wiring and global state for Firecube configuration.
+
+This module handles side-effects like setting global variables and exporting
+environment variables. It depends on `firecube.core.config` for types
+and loading logic.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Mapping, MutableMapping
+from pathlib import Path
+from typing import Any
+
+from firecube.core.config import (
+    StorageConfig,
+    build_storage_config,
+    derive_target_uri,
+    load_config_file,
+)
+from firecube.core.product.identity import ProductIdentity
+from firecube.core.storage.uri import StorageUri
+
+log = logging.getLogger("firecube.core.runtime")
+
+_global_storage_config: StorageConfig | None = None
+
+
+def _target_uri_from_config(identity: ProductIdentity) -> str:
+    """Render the FIRECUBE_TARGET_URI value from a `ProductIdentity`.
+
+    Preserves legacy env-var semantics: bare local path for `file://` URIs,
+    canonical URI form for remote (`s3://`, `gs://`) URIs.
+    """
+    if identity.product_uri.protocol == "file":
+        return identity.product_uri.path
+    return identity.product_uri.to_str()
+
+
+def set_global_storage_config(config: StorageConfig) -> None:
+    """Set the global storage configuration."""
+    global _global_storage_config
+    config.validate()
+    _global_storage_config = config
+
+
+def get_global_storage_config() -> StorageConfig:
+    """Return the global storage configuration."""
+    if _global_storage_config is None:
+        raise RuntimeError("Global storage configuration not set. Initialise with CLI args first.")
+    return _global_storage_config
+
+
+def export_storage_config_to_env(
+    config: StorageConfig,
+    identity: ProductIdentity | None = None,
+    *,
+    env: MutableMapping[str, str] | None = None,
+) -> None:
+    """Publish storage configuration to environment variables for child processes.
+
+    Parameters
+    ----------
+    config:
+        Non-location storage config (storage_type, endpoint_url, credentials,
+        region, path_style, storage_driver).
+    identity:
+        Optional product identity providing location fields (bucket /
+        target_path / target_uri). When ``None``, location vars are omitted.
+    env:
+        Optional mutable mapping to write into; defaults to ``os.environ``.
+
+    Notes
+    -----
+    We intentionally export both FIRECUBE_* and AWS_* style variables:
+      - FIRECUBE_* is used by Firecube internals and helpers.
+      - AWS_* allows libraries such as fsspec/xarray/zarr to discover
+        credentials and endpoints when working with S3-compatible storage.
+    """
+    target_env: MutableMapping[str, str] = os.environ if env is None else env
+
+    target_env["FIRECUBE_STORAGE_TYPE"] = config.storage_type
+    if config.endpoint_url:
+        target_env["FIRECUBE_ENDPOINT_URL"] = config.endpoint_url
+    if config.access_key:
+        target_env["FIRECUBE_ACCESS_KEY"] = config.access_key
+    if config.secret_key:
+        target_env["FIRECUBE_SECRET_KEY"] = config.secret_key
+    if config.region:
+        target_env["FIRECUBE_REGION"] = config.region
+    target_env["FIRECUBE_PATH_STYLE"] = str(config.path_style).lower()
+    target_env["FIRECUBE_STORAGE_DRIVER"] = config.storage_driver
+
+    if identity is not None:
+        if identity.product_uri.authority:
+            target_env["FIRECUBE_BUCKET"] = identity.product_uri.authority
+        if identity.product_uri.protocol == "file":
+            target_env["FIRECUBE_TARGET_PATH"] = identity.product_uri.path
+        target_env["FIRECUBE_TARGET_URI"] = _target_uri_from_config(identity)
+
+    if config.access_key:
+        target_env["AWS_ACCESS_KEY_ID"] = config.access_key
+    if config.secret_key:
+        target_env["AWS_SECRET_ACCESS_KEY"] = config.secret_key
+    if config.region:
+        target_env["AWS_DEFAULT_REGION"] = config.region
+    if config.endpoint_url:
+        target_env["AWS_ENDPOINT_URL"] = config.endpoint_url
+    target_env["AWS_S3_ADDRESSING_STYLE"] = "path" if config.path_style else "virtual"
+
+
+def identity_from_storage_config(storage_config: StorageConfig) -> ProductIdentity | None:
+    """Synthesize a `ProductIdentity` from a resolved `StorageConfig`.
+
+    Bridges the legacy bridge-attr storage view (``target_path`` / ``bucket``
+    set on ``StorageConfig`` by ``build_storage_config``) to the
+    ``ProductIdentity`` shape consumed by env-export and CLI bridge sites.
+    The remaining bridge-attr reads live inside ``derive_target_uri`` (see
+    config.py); this helper is the single seam between resolved config and
+    identity.
+
+    Returns ``None`` when the resolved config lacks a base URI (e.g. local
+    storage_type with no ``target_path`` set yet).
+    """
+    try:
+        base_uri_str = derive_target_uri(storage_config)
+    except ValueError:
+        return None
+
+    if storage_config.storage_type == "local":
+        base_uri = StorageUri.from_local_path(base_uri_str)
+    else:
+        base_uri = StorageUri.parse(base_uri_str)
+
+    name = base_uri.path.rstrip("/").rsplit("/", 1)[-1]
+    return ProductIdentity(
+        product_name=name,
+        product_uri=base_uri,
+        control_root_uri=base_uri.join(".firecube"),
+        format="zarr",
+    )
+
+
+def resolve_storage_config(
+    *,
+    config_file: Path | str | None = None,
+    env: Mapping[str, str] = os.environ,  # type: ignore[assignment]
+    overrides: dict[str, Any] | None = None,
+    export_env: bool = True,
+    set_global: bool = True,
+) -> StorageConfig:
+    """Resolve StorageConfig from config.toml + env + overrides.
+
+    This is a convenience wrapper used by CLI entrypoints to keep the
+    load/build/validate/export wiring consistent across commands.
+
+    Notes
+    -----
+    - Precedence is implemented in `build_storage_config(...)`.
+    - This helper does not depend on click (so it can be used by non-CLI code).
+    """
+    cfg = load_config_file(config_file)
+    storage_config = build_storage_config(cfg, env, overrides or {})
+    storage_config.validate()
+
+    if set_global:
+        set_global_storage_config(storage_config)
+    if export_env:
+        identity = identity_from_storage_config(storage_config)
+        export_storage_config_to_env(storage_config, identity)
+
+    return storage_config