firecube 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- firecube/__init__.py +23 -0
- firecube/__main__.py +20 -0
- firecube/_version.py +46 -0
- firecube/cli/_audience.py +135 -0
- firecube/cli/_command_schemas.py +112 -0
- firecube/cli/_ctx.py +85 -0
- firecube/cli/_errors.py +115 -0
- firecube/cli/_formatter.py +195 -0
- firecube/cli/_product.py +47 -0
- firecube/cli/_rename_hints.py +123 -0
- firecube/cli/_shared_options.py +247 -0
- firecube/cli/_slot_env.py +102 -0
- firecube/cli/_slot_planning.py +95 -0
- firecube/cli/_typed_options.py +181 -0
- firecube/cli/_uri_policy.py +110 -0
- firecube/cli/advise.py +321 -0
- firecube/cli/archive.py +870 -0
- firecube/cli/catalog.py +196 -0
- firecube/cli/chunks/__init__.py +74 -0
- firecube/cli/chunks/_claims.py +229 -0
- firecube/cli/chunks/_common.py +84 -0
- firecube/cli/chunks/_delete.py +376 -0
- firecube/cli/chunks/_list.py +334 -0
- firecube/cli/chunks/_manager.py +161 -0
- firecube/cli/chunks/_runs.py +294 -0
- firecube/cli/chunks/_snapshots.py +165 -0
- firecube/cli/completion.py +73 -0
- firecube/cli/main.py +866 -0
- firecube/cli/parquet.py +289 -0
- firecube/cli/plugins/__init__.py +28 -0
- firecube/cli/plugins/commands.py +262 -0
- firecube/cli/plugins/introspect.py +304 -0
- firecube/cli/plugins/mgmt.py +198 -0
- firecube/cli/plugins/registry.py +132 -0
- firecube/cli/zarr.py +1043 -0
- firecube/core/__init__.py +14 -0
- firecube/core/api.py +88 -0
- firecube/core/cf/__init__.py +23 -0
- firecube/core/cf/check_ids.py +126 -0
- firecube/core/cf/report.py +57 -0
- firecube/core/cf/validator.py +313 -0
- firecube/core/config.py +307 -0
- firecube/core/controlplane/__init__.py +47 -0
- firecube/core/controlplane/_event_processor.py +136 -0
- firecube/core/controlplane/_helpers.py +114 -0
- firecube/core/controlplane/_projection.py +253 -0
- firecube/core/controlplane/_snapshot.py +215 -0
- firecube/core/controlplane/_wal_reader.py +402 -0
- firecube/core/controlplane/_wal_writer.py +589 -0
- firecube/core/controlplane/claims.py +294 -0
- firecube/core/controlplane/deletion.py +704 -0
- firecube/core/controlplane/events.py +219 -0
- firecube/core/controlplane/manager.py +990 -0
- firecube/core/controlplane/metrics.py +97 -0
- firecube/core/controlplane/repo.py +804 -0
- firecube/core/controlplane/repo_utils.py +73 -0
- firecube/core/controlplane/time_dim.py +111 -0
- firecube/core/controlplane/types.py +445 -0
- firecube/core/credentials.py +36 -0
- firecube/core/duckdb/__init__.py +27 -0
- firecube/core/duckdb/bridge.py +82 -0
- firecube/core/duckdb/io.py +30 -0
- firecube/core/errors.py +74 -0
- firecube/core/filesystem/__init__.py +60 -0
- firecube/core/filesystem/_obstore_compat.py +60 -0
- firecube/core/filesystem/fsspec_backend.py +308 -0
- firecube/core/filesystem/instrumentation.py +260 -0
- firecube/core/filesystem/obstore_backend.py +406 -0
- firecube/core/filesystem/ops.py +412 -0
- firecube/core/filesystem/protocol.py +130 -0
- firecube/core/filesystem/store_factory.py +149 -0
- firecube/core/formats/__init__.py +46 -0
- firecube/core/formats/_iso.py +84 -0
- firecube/core/formats/discovery.py +165 -0
- firecube/core/formats/hdf5.py +137 -0
- firecube/core/formats/netcdf.py +226 -0
- firecube/core/formats/zip.py +192 -0
- firecube/core/intake.py +530 -0
- firecube/core/observability/__init__.py +72 -0
- firecube/core/observability/_state.py +24 -0
- firecube/core/observability/logging.py +147 -0
- firecube/core/observability/metrics.py +230 -0
- firecube/core/observability/telemetry/__init__.py +87 -0
- firecube/core/observability/telemetry/pushgateway.py +339 -0
- firecube/core/observability/telemetry/sinks.py +155 -0
- firecube/core/observability/tracing.py +126 -0
- firecube/core/product/__init__.py +45 -0
- firecube/core/product/identity.py +76 -0
- firecube/core/product/resolver.py +121 -0
- firecube/core/product/target.py +120 -0
- firecube/core/runtime.py +187 -0
- firecube/core/slot_index.py +205 -0
- firecube/core/storage/__init__.py +45 -0
- firecube/core/storage/binding.py +39 -0
- firecube/core/storage/cache_key.py +40 -0
- firecube/core/storage/completion.py +245 -0
- firecube/core/storage/driver_config.py +125 -0
- firecube/core/storage/results.py +30 -0
- firecube/core/storage/session.py +431 -0
- firecube/core/storage/transfer.py +142 -0
- firecube/core/storage/uri.py +173 -0
- firecube/core/tensogram/__init__.py +37 -0
- firecube/core/tensogram/_compat.py +36 -0
- firecube/core/tensogram/controlplane_codec.py +230 -0
- firecube/core/tensogram/converter.py +390 -0
- firecube/core/tensogram/metadata.py +274 -0
- firecube/core/tensogram/restore.py +303 -0
- firecube/core/tensogram/schema.py +93 -0
- firecube/core/uris.py +182 -0
- firecube/core/workspaces.py +68 -0
- firecube/core/zarr/__init__.py +69 -0
- firecube/core/zarr/_reserved_attrs.py +47 -0
- firecube/core/zarr/_reserved_root_attrs.py +74 -0
- firecube/core/zarr/io.py +88 -0
- firecube/core/zarr/layers.py +184 -0
- firecube/core/zarr/multires.py +178 -0
- firecube/core/zarr/region_writer.py +782 -0
- firecube/core/zarr/scrub.py +200 -0
- firecube/core/zarr/state.py +258 -0
- firecube/core/zarr/time_decode.py +71 -0
- firecube/core/zarr/validation.py +622 -0
- firecube/ingestor/__init__.py +27 -0
- firecube/ingestor/api.py +215 -0
- firecube/ingestor/config/__init__.py +19 -0
- firecube/ingestor/config/coercion.py +157 -0
- firecube/ingestor/config/engine.py +188 -0
- firecube/ingestor/contracts/__init__.py +14 -0
- firecube/ingestor/contracts/interfaces.py +235 -0
- firecube/ingestor/devtools/_templates/__init__.py +14 -0
- firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
- firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
- firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
- firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
- firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
- firecube/ingestor/devtools/scaffolding.py +229 -0
- firecube/ingestor/errors.py +79 -0
- firecube/ingestor/extensions/_binning.py +152 -0
- firecube/ingestor/extensions/duck.py +153 -0
- firecube/ingestor/extensions/grid.py +287 -0
- firecube/ingestor/extensions/healpix.py +330 -0
- firecube/ingestor/registry/__init__.py +14 -0
- firecube/ingestor/registry/loader.py +144 -0
- firecube/ingestor/registry/metadata.py +97 -0
- firecube/ingestor/registry/version_compat.py +95 -0
- firecube/ingestor/runtime/__init__.py +14 -0
- firecube/ingestor/runtime/aggregation.py +112 -0
- firecube/ingestor/runtime/base.py +836 -0
- firecube/ingestor/runtime/base_hooks.py +129 -0
- firecube/ingestor/runtime/batching.py +161 -0
- firecube/ingestor/runtime/configure.py +195 -0
- firecube/ingestor/runtime/coverage.py +177 -0
- firecube/ingestor/runtime/engine.py +793 -0
- firecube/ingestor/runtime/parallel_evidence.py +75 -0
- firecube/ingestor/runtime/parallel_execution_state.py +28 -0
- firecube/ingestor/runtime/parallel_gate.py +164 -0
- firecube/ingestor/runtime/parallel_run_id.py +48 -0
- firecube/ingestor/runtime/preflight.py +40 -0
- firecube/ingestor/runtime/recording.py +425 -0
- firecube/ingestor/runtime/resume_guard.py +556 -0
- firecube/ingestor/runtime/resume_types.py +65 -0
- firecube/ingestor/runtime/telemetry.py +171 -0
- firecube/ingestor/runtime/tensogram/__init__.py +14 -0
- firecube/ingestor/runtime/tensogram/strategy.py +164 -0
- firecube/ingestor/runtime/workspace.py +220 -0
- firecube/ingestor/runtime/zarr/__init__.py +34 -0
- firecube/ingestor/runtime/zarr/append.py +466 -0
- firecube/ingestor/runtime/zarr/append_services.py +650 -0
- firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
- firecube/ingestor/runtime/zarr/contracts.py +58 -0
- firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
- firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
- firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
- firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
- firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
- firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
- firecube/ingestor/runtime/zarr/write.py +212 -0
- firecube/ingestor/runtime/zarr/write_context.py +139 -0
- firecube/ingestor/templates/__init__.py +19 -0
- firecube/ingestor/templates/config.py +118 -0
- firecube/ingestor/templates/direct_zarr.py +790 -0
- firecube/ingestor/templates/generic.py +462 -0
- firecube/ingestor/templates/generic_tensogram.py +170 -0
- firecube/ingestor/types/__init__.py +19 -0
- firecube/ingestor/types/config.py +89 -0
- firecube/ingestor/types/context.py +409 -0
- firecube/ingestor/types/manifest.py +53 -0
- firecube/ingestor/types/planned_range.py +152 -0
- firecube/ingestor/types/result_metrics.py +386 -0
- firecube/ingestor/utils/__init__.py +14 -0
- firecube/ingestor/utils/duckdb_utils.py +235 -0
- firecube/ingestor/validation.py +55 -0
- firecube-0.1.4.dist-info/METADATA +242 -0
- firecube-0.1.4.dist-info/RECORD +197 -0
- firecube-0.1.4.dist-info/WHEEL +4 -0
- firecube-0.1.4.dist-info/entry_points.txt +2 -0
- firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
- firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Generic input format helpers (ZIP, HDF5, NetCDF, file discovery).
|
|
16
|
+
|
|
17
|
+
Public imports remain stable at ``firecube.core.formats``.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from firecube.core.formats.discovery import KNOWN_EXTENSIONS, discover_input_files
|
|
21
|
+
from firecube.core.formats.hdf5 import materialize_hdf5_path, read_hdf5_array
|
|
22
|
+
from firecube.core.formats.netcdf import (
|
|
23
|
+
clean_netcdf_encoding,
|
|
24
|
+
normalize_string_vars,
|
|
25
|
+
prepare_netcdf_for_zarr,
|
|
26
|
+
rename_time_dim,
|
|
27
|
+
)
|
|
28
|
+
from firecube.core.formats.zip import (
|
|
29
|
+
extract_hdf5_from_zip,
|
|
30
|
+
extract_zip_files_parallel,
|
|
31
|
+
stream_hdf5_from_zip,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"KNOWN_EXTENSIONS",
|
|
36
|
+
"clean_netcdf_encoding",
|
|
37
|
+
"discover_input_files",
|
|
38
|
+
"extract_hdf5_from_zip",
|
|
39
|
+
"extract_zip_files_parallel",
|
|
40
|
+
"materialize_hdf5_path",
|
|
41
|
+
"normalize_string_vars",
|
|
42
|
+
"prepare_netcdf_for_zarr",
|
|
43
|
+
"read_hdf5_array",
|
|
44
|
+
"rename_time_dim",
|
|
45
|
+
"stream_hdf5_from_zip",
|
|
46
|
+
]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Strict UTC ISO-8601 → numpy.datetime64 helpers. Package-private."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Iterable
|
|
20
|
+
from datetime import datetime, timedelta
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parse_iso_utc(s: str) -> datetime:
|
|
26
|
+
"""Parse a strict UTC ISO-8601 string to a naive UTC datetime.
|
|
27
|
+
|
|
28
|
+
Accepts Z suffix or explicit +00:00 offset. Rejects non-UTC offsets
|
|
29
|
+
with NotImplementedError. Rejects timezone-naive, malformed, or empty strings
|
|
30
|
+
with ValueError.
|
|
31
|
+
"""
|
|
32
|
+
if not isinstance(s, str):
|
|
33
|
+
raise ValueError(f"expected str, got {type(s).__name__!r}")
|
|
34
|
+
if not s:
|
|
35
|
+
raise ValueError("empty ISO string")
|
|
36
|
+
if _fractional_second_digits(s) > 6:
|
|
37
|
+
raise ValueError(f"unparseable ISO string: {s!r}")
|
|
38
|
+
|
|
39
|
+
normalized = s[:-1] + "+00:00" if s.endswith("Z") else s
|
|
40
|
+
try:
|
|
41
|
+
dt = datetime.fromisoformat(normalized)
|
|
42
|
+
except ValueError as exc:
|
|
43
|
+
raise ValueError(f"unparseable ISO string: {s!r}") from exc
|
|
44
|
+
|
|
45
|
+
if dt.tzinfo is None:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"timezone-naive ISO string: {s!r}; expected UTC with 'Z' or '+00:00' suffix"
|
|
48
|
+
)
|
|
49
|
+
if dt.utcoffset() != timedelta(0):
|
|
50
|
+
raise NotImplementedError(f"non-UTC ISO string: {s!r} (offset={dt.utcoffset()})")
|
|
51
|
+
dt = dt.replace(tzinfo=None)
|
|
52
|
+
|
|
53
|
+
return dt
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _fractional_second_digits(s: str) -> int:
|
|
57
|
+
if "." not in s:
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
fraction = s.rsplit(".", maxsplit=1)[1]
|
|
61
|
+
for separator in ("Z", "+", "-"):
|
|
62
|
+
fraction = fraction.split(separator, maxsplit=1)[0]
|
|
63
|
+
return len(fraction)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _iso_strs_to_datetime64(strs: Iterable[str]) -> np.ndarray:
|
|
67
|
+
"""Convert an iterable of strict UTC ISO-8601 strings to datetime64.
|
|
68
|
+
|
|
69
|
+
Precision is auto-detected from the input: datetime64[s] when no string
|
|
70
|
+
contains a fractional-second component, datetime64[us] otherwise.
|
|
71
|
+
The whole array uses a single unit (upgraded to [us] if any element has
|
|
72
|
+
fractional-second syntax).
|
|
73
|
+
|
|
74
|
+
Raises ValueError for malformed or non-UTC strings, and for strings with
|
|
75
|
+
more than 6 fractional digits (nanosecond precision; stdlib limit).
|
|
76
|
+
"""
|
|
77
|
+
strs_list = list(strs)
|
|
78
|
+
if not strs_list:
|
|
79
|
+
return np.array([], dtype="datetime64[s]")
|
|
80
|
+
|
|
81
|
+
has_frac = any("." in s for s in strs_list)
|
|
82
|
+
unit = "us" if has_frac else "s"
|
|
83
|
+
parsed = [_parse_iso_utc(s) for s in strs_list]
|
|
84
|
+
return np.asarray(parsed, dtype=f"datetime64[{unit}]")
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Input file discovery helpers for ingestion plugins.
|
|
16
|
+
|
|
17
|
+
Format-agnostic discovery utilities that walk a local path or remote URI
|
|
18
|
+
and return candidate input file URIs (ZIP, HDF5, NetCDF, ...).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import contextlib
|
|
24
|
+
import fnmatch
|
|
25
|
+
from collections.abc import Iterable, Sequence
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from firecube.core.formats.hdf5 import looks_like_hdf5
|
|
30
|
+
from firecube.core.uris import is_remote_target, parse_uri
|
|
31
|
+
|
|
32
|
+
# All file extensions recognised by the firecube ecosystem.
|
|
33
|
+
# The first three are the defaults for ``discover_input_files``; additional
|
|
34
|
+
# formats (e.g. ``.tgm``) must be requested explicitly via *include_suffixes*.
|
|
35
|
+
KNOWN_EXTENSIONS: frozenset[str] = frozenset({".zip", ".h5", ".nc", ".tgm"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _path_matches_any_glob(
|
|
39
|
+
path: str, patterns: Iterable[str], *, candidates: Iterable[str]
|
|
40
|
+
) -> bool:
|
|
41
|
+
"""Return True when any candidate path matches any glob pattern."""
|
|
42
|
+
normalized_candidates = {path, *candidates}
|
|
43
|
+
return any(
|
|
44
|
+
fnmatch.fnmatch(candidate, pattern)
|
|
45
|
+
for pattern in patterns
|
|
46
|
+
for candidate in normalized_candidates
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _filter_discovered_paths(
|
|
51
|
+
paths: Iterable[str],
|
|
52
|
+
*,
|
|
53
|
+
source_uri: str,
|
|
54
|
+
include_suffixes: Sequence[str],
|
|
55
|
+
preferred_globs: Iterable[str] | None,
|
|
56
|
+
recursive: bool,
|
|
57
|
+
sniff_hdf5: bool,
|
|
58
|
+
exclude: Iterable[str] | None,
|
|
59
|
+
fs: Any,
|
|
60
|
+
root: str,
|
|
61
|
+
) -> list[str]:
|
|
62
|
+
"""Filter discovered filesystem paths down to candidate input files."""
|
|
63
|
+
remote_source = is_remote_target(source_uri)
|
|
64
|
+
suffixes = {s.lower() for s in include_suffixes}
|
|
65
|
+
preferred_patterns = tuple(preferred_globs or ())
|
|
66
|
+
excluded_patterns = tuple(exclude or ())
|
|
67
|
+
source_is_dir = False
|
|
68
|
+
with contextlib.suppress(Exception):
|
|
69
|
+
source_is_dir = bool(fs.isdir(root))
|
|
70
|
+
|
|
71
|
+
root_prefix = root.rstrip("/")
|
|
72
|
+
resolved: list[str] = []
|
|
73
|
+
|
|
74
|
+
for path in paths:
|
|
75
|
+
normalized = str(path)
|
|
76
|
+
path_for_match = parse_uri(normalized)["path"] if remote_source else normalized
|
|
77
|
+
basename = Path(path_for_match).name
|
|
78
|
+
relative = path_for_match
|
|
79
|
+
|
|
80
|
+
if root_prefix and path_for_match.startswith(f"{root_prefix}/"):
|
|
81
|
+
relative = path_for_match[len(root_prefix) + 1 :]
|
|
82
|
+
elif path_for_match == root_prefix:
|
|
83
|
+
relative = basename
|
|
84
|
+
|
|
85
|
+
if not recursive and "/" in relative:
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
glob_candidates = {basename, relative, normalized}
|
|
89
|
+
|
|
90
|
+
if excluded_patterns and _path_matches_any_glob(
|
|
91
|
+
normalized, excluded_patterns, candidates=glob_candidates
|
|
92
|
+
):
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
suffix = Path(path_for_match).suffix.lower()
|
|
96
|
+
include_by_suffix = suffix in suffixes
|
|
97
|
+
include_by_sniff = False
|
|
98
|
+
if sniff_hdf5 and (suffix == "" or suffix not in suffixes):
|
|
99
|
+
include_by_sniff = looks_like_hdf5(Path(normalized))
|
|
100
|
+
|
|
101
|
+
include_by_glob = False
|
|
102
|
+
if preferred_patterns and source_is_dir:
|
|
103
|
+
include_by_glob = _path_matches_any_glob(
|
|
104
|
+
normalized, preferred_patterns, candidates=glob_candidates
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
if include_by_suffix or include_by_sniff or include_by_glob:
|
|
108
|
+
resolved.append(normalized)
|
|
109
|
+
|
|
110
|
+
dedup = dict.fromkeys(resolved)
|
|
111
|
+
return sorted(dedup, key=lambda path: Path(path).name)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def discover_input_files(
|
|
115
|
+
source: str | Path,
|
|
116
|
+
*,
|
|
117
|
+
storage_config: Any | None = None,
|
|
118
|
+
include_suffixes: Sequence[str] = (".zip", ".h5", ".nc"),
|
|
119
|
+
preferred_globs: Iterable[str] | None = None,
|
|
120
|
+
recursive: bool = True,
|
|
121
|
+
sniff_hdf5: bool = True,
|
|
122
|
+
exclude: Iterable[str] | None = None,
|
|
123
|
+
) -> list[str]:
|
|
124
|
+
"""Discover input files from a local path or remote URI.
|
|
125
|
+
|
|
126
|
+
Selection is intentionally conservative and format-agnostic:
|
|
127
|
+
- Accept files matching `include_suffixes`
|
|
128
|
+
- Optionally accept extensionless files that look like HDF5
|
|
129
|
+
- Optionally add files matched by `preferred_globs` (glob patterns)
|
|
130
|
+
|
|
131
|
+
Returns URI/path strings (for example ``/tmp/data/file.nc`` or
|
|
132
|
+
``s3://bucket/prefix/file.nc``). ``Path`` inputs are accepted for backward
|
|
133
|
+
compatibility and are converted to strings internally.
|
|
134
|
+
"""
|
|
135
|
+
from firecube.core.filesystem.ops import _open_fsspec_url
|
|
136
|
+
from firecube.core.uris import is_remote_target, parse_uri
|
|
137
|
+
|
|
138
|
+
source_uri = str(source)
|
|
139
|
+
is_remote = is_remote_target(source_uri)
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
fs, root = _open_fsspec_url(source_uri, storage_config=storage_config)
|
|
143
|
+
except Exception as exc:
|
|
144
|
+
raise ValueError(f"Cannot open source location {source_uri!r}: {exc}") from exc
|
|
145
|
+
|
|
146
|
+
try:
|
|
147
|
+
all_paths = [str(path) for path in fs.find(root)]
|
|
148
|
+
except Exception as exc:
|
|
149
|
+
raise ValueError(f"Cannot list source location {source_uri!r}: {exc}") from exc
|
|
150
|
+
|
|
151
|
+
if is_remote:
|
|
152
|
+
protocol = parse_uri(source_uri)["protocol"]
|
|
153
|
+
all_paths = [path if "://" in path else f"{protocol}://{path}" for path in all_paths]
|
|
154
|
+
|
|
155
|
+
return _filter_discovered_paths(
|
|
156
|
+
all_paths,
|
|
157
|
+
source_uri=source_uri,
|
|
158
|
+
include_suffixes=include_suffixes,
|
|
159
|
+
preferred_globs=preferred_globs,
|
|
160
|
+
recursive=recursive,
|
|
161
|
+
sniff_hdf5=sniff_hdf5 and not is_remote,
|
|
162
|
+
exclude=exclude,
|
|
163
|
+
fs=fs,
|
|
164
|
+
root=root,
|
|
165
|
+
)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""HDF5 helpers for ingestion plugins.
|
|
16
|
+
|
|
17
|
+
Reusable helpers for sniffing, reading, and materializing HDF5 inputs
|
|
18
|
+
(either standalone files or members embedded inside ZIP archives).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import contextlib
|
|
24
|
+
import logging
|
|
25
|
+
import tempfile
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from firecube.core.formats.zip import extract_hdf5_from_zip
|
|
30
|
+
|
|
31
|
+
log = logging.getLogger("firecube.core.formats")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def looks_like_hdf5(path: Path) -> bool:
|
|
35
|
+
"""Best-effort magic-number check for HDF5 files without extensions."""
|
|
36
|
+
try:
|
|
37
|
+
with path.open("rb") as fh:
|
|
38
|
+
sig = fh.read(8)
|
|
39
|
+
return sig == b"\x89HDF\r\n\x1a\n"
|
|
40
|
+
except Exception:
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read_hdf5_array(
|
|
45
|
+
hdf5_path: Path,
|
|
46
|
+
*,
|
|
47
|
+
variable: str,
|
|
48
|
+
logger: logging.Logger | None = None,
|
|
49
|
+
) -> Any:
|
|
50
|
+
"""Read a named array from a local HDF5(-like) file, with xarray fallback.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
hdf5_path: Path to the HDF5 file.
|
|
54
|
+
variable: Name of the variable/dataset to read.
|
|
55
|
+
logger: Optional logger for debug messages.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Numpy array of the data (float32).
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
KeyError: If the variable is not found.
|
|
62
|
+
RuntimeError: If dependencies are missing or read fails.
|
|
63
|
+
"""
|
|
64
|
+
logger = logger or log
|
|
65
|
+
try:
|
|
66
|
+
import numpy as np
|
|
67
|
+
except Exception as exc:
|
|
68
|
+
raise RuntimeError("numpy is required to read HDF5 inputs") from exc
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
import h5py
|
|
72
|
+
except Exception as exc:
|
|
73
|
+
raise RuntimeError("h5py is required to read HDF5 inputs") from exc
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
with h5py.File(hdf5_path, "r") as handle:
|
|
77
|
+
if variable not in handle:
|
|
78
|
+
raise KeyError(f"{variable} dataset not found in HDF5 file")
|
|
79
|
+
return np.asarray(handle[variable], dtype=np.float32)
|
|
80
|
+
except Exception as h5_exc:
|
|
81
|
+
try:
|
|
82
|
+
import xarray as xr
|
|
83
|
+
|
|
84
|
+
ds = xr.open_dataset(hdf5_path, phony_dims="sort")
|
|
85
|
+
if variable not in ds:
|
|
86
|
+
raise KeyError(f"{variable} variable not found in dataset")
|
|
87
|
+
arr = ds[variable].values.astype(np.float32, copy=False)
|
|
88
|
+
ds.close()
|
|
89
|
+
return arr
|
|
90
|
+
except Exception as xr_exc:
|
|
91
|
+
raise RuntimeError(
|
|
92
|
+
f"Failed to read {variable} from {hdf5_path.name}: {h5_exc}"
|
|
93
|
+
) from xr_exc
|
|
94
|
+
finally:
|
|
95
|
+
with contextlib.suppress(Exception):
|
|
96
|
+
logger.debug("Fell back to xarray open_dataset for %s", hdf5_path.name)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def materialize_hdf5_path(
|
|
100
|
+
file_path: Path,
|
|
101
|
+
*,
|
|
102
|
+
extract_root: Path | None = None,
|
|
103
|
+
logger: logging.Logger | None = None,
|
|
104
|
+
) -> tuple[Path, tempfile.TemporaryDirectory | None]:
|
|
105
|
+
"""Return a local HDF5 path for a source path (ZIP or direct HDF5-like file).
|
|
106
|
+
|
|
107
|
+
If the input is a ZIP file, it extracts the content to a temporary directory.
|
|
108
|
+
If it is already an HDF5-like file, it is returned as-is.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
file_path: Source file path.
|
|
112
|
+
extract_root: Optional root directory for temporary extraction.
|
|
113
|
+
logger: Optional logger.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
Tuple of (path_to_hdf5_file, temporary_directory_object_or_None).
|
|
117
|
+
"""
|
|
118
|
+
logger = logger or log
|
|
119
|
+
if file_path.suffix.lower() != ".zip":
|
|
120
|
+
return file_path, None
|
|
121
|
+
|
|
122
|
+
tmp_dir: tempfile.TemporaryDirectory | None
|
|
123
|
+
if extract_root is None:
|
|
124
|
+
tmp_dir = tempfile.TemporaryDirectory()
|
|
125
|
+
else:
|
|
126
|
+
extract_root.mkdir(parents=True, exist_ok=True)
|
|
127
|
+
tmp_dir = tempfile.TemporaryDirectory(dir=str(extract_root))
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
extracted = extract_hdf5_from_zip(file_path, Path(tmp_dir.name), logger=logger)
|
|
131
|
+
if extracted is None:
|
|
132
|
+
raise FileNotFoundError(f"No HDF5 member found in archive {file_path.name}")
|
|
133
|
+
return extracted, tmp_dir
|
|
134
|
+
except Exception:
|
|
135
|
+
with contextlib.suppress(Exception):
|
|
136
|
+
tmp_dir.cleanup()
|
|
137
|
+
raise
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""NetCDF-to-Zarr preparation utilities.
|
|
16
|
+
|
|
17
|
+
Composable helpers that normalize xarray Datasets loaded from NetCDF
|
|
18
|
+
sources into the shape expected by Firecube's Zarr writer. Plugin-agnostic.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
from collections.abc import Collection
|
|
25
|
+
from contextlib import suppress
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
import xarray as xr
|
|
29
|
+
|
|
30
|
+
from firecube.core.formats._iso import _iso_strs_to_datetime64
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def clean_netcdf_encoding(ds: xr.Dataset) -> xr.Dataset:
|
|
34
|
+
"""Strip HDF5 chunk hints from all variable encodings.
|
|
35
|
+
|
|
36
|
+
Removes 'chunks', 'chunksizes', and 'preferred_chunks' from each
|
|
37
|
+
variable's encoding dict so they do not conflict with the Zarr chunk
|
|
38
|
+
layout chosen by the core writer. Modifies encoding dicts in place;
|
|
39
|
+
returns the same dataset object.
|
|
40
|
+
"""
|
|
41
|
+
for var_name in list(ds.variables):
|
|
42
|
+
enc = ds[var_name].encoding
|
|
43
|
+
enc.pop("chunks", None)
|
|
44
|
+
enc.pop("chunksizes", None)
|
|
45
|
+
enc.pop("preferred_chunks", None)
|
|
46
|
+
return ds
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def rename_time_dim(
|
|
50
|
+
ds: xr.Dataset,
|
|
51
|
+
source: str = "time",
|
|
52
|
+
target: str = "timestamp",
|
|
53
|
+
) -> xr.Dataset:
|
|
54
|
+
"""Rename the time dimension to Firecube's append convention.
|
|
55
|
+
|
|
56
|
+
If *source* exists as a dimension or coordinate, rename it to *target*.
|
|
57
|
+
If *source* is not present, return the dataset unchanged (no error).
|
|
58
|
+
"""
|
|
59
|
+
if source in ds.dims or source in ds.coords:
|
|
60
|
+
return ds.rename({source: target})
|
|
61
|
+
return ds
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prepare_netcdf_for_zarr(
|
|
65
|
+
ds: xr.Dataset,
|
|
66
|
+
time_dim: str = "time",
|
|
67
|
+
target_time_dim: str = "timestamp",
|
|
68
|
+
) -> xr.Dataset:
|
|
69
|
+
"""Convenience wrapper: rename time dimension then clean encodings.
|
|
70
|
+
|
|
71
|
+
Applies all standard NetCDF→Zarr V3 preparation steps in order:
|
|
72
|
+
1. Rename time dimension from *time_dim* to *target_time_dim*.
|
|
73
|
+
2. Strip HDF5 chunk hints from all variable encodings.
|
|
74
|
+
"""
|
|
75
|
+
ds = rename_time_dim(ds, source=time_dim, target=target_time_dim)
|
|
76
|
+
ds = clean_netcdf_encoding(ds)
|
|
77
|
+
return ds
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# ─── string variable normalization ─────────────────────────
|
|
81
|
+
|
|
82
|
+
_ACCEPTED_ELEMENT_TYPES: tuple[type, ...] = (str, bytes, np.str_, np.bytes_)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _is_string_var_pure(values: np.ndarray) -> bool:
|
|
86
|
+
"""Pure predicate: True when values holds a string-typed var we can normalize.
|
|
87
|
+
|
|
88
|
+
Returns False for anything we cannot confidently classify; never raises.
|
|
89
|
+
"""
|
|
90
|
+
kind = values.dtype.kind
|
|
91
|
+
if kind in ("S", "U"):
|
|
92
|
+
return True
|
|
93
|
+
if kind != "O" or values.size == 0:
|
|
94
|
+
return False
|
|
95
|
+
return all(isinstance(x, _ACCEPTED_ELEMENT_TYPES) for x in values.ravel())
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _values_to_str_list(values: np.ndarray) -> list[str]:
|
|
99
|
+
"""Convert a string-shaped ndarray to a flat list of Python str."""
|
|
100
|
+
kind = values.dtype.kind
|
|
101
|
+
if kind == "U":
|
|
102
|
+
return values.ravel().tolist()
|
|
103
|
+
if kind == "S":
|
|
104
|
+
try:
|
|
105
|
+
return np.char.decode(values, "utf-8").ravel().tolist()
|
|
106
|
+
except UnicodeDecodeError as exc:
|
|
107
|
+
raise ValueError(f"invalid UTF-8 bytes in string var: {exc}") from exc
|
|
108
|
+
out: list[str] = []
|
|
109
|
+
for x in values.ravel():
|
|
110
|
+
if isinstance(x, str):
|
|
111
|
+
out.append(x)
|
|
112
|
+
elif isinstance(x, (bytes, np.bytes_)):
|
|
113
|
+
try:
|
|
114
|
+
out.append(x.decode("utf-8"))
|
|
115
|
+
except UnicodeDecodeError as exc:
|
|
116
|
+
raise ValueError(f"invalid UTF-8 bytes in string var: {exc}") from exc
|
|
117
|
+
else:
|
|
118
|
+
out.append(str(x))
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def normalize_string_vars(
|
|
123
|
+
ds: xr.Dataset,
|
|
124
|
+
*,
|
|
125
|
+
iso_targets: Collection[str] | None = None,
|
|
126
|
+
logger: logging.Logger | None = None,
|
|
127
|
+
preserve_cf_time_attrs: bool = False,
|
|
128
|
+
) -> xr.Dataset:
|
|
129
|
+
"""Normalize string-typed variables in a NetCDF-loaded Dataset for Zarr writes.
|
|
130
|
+
|
|
131
|
+
Detects string vars by dtype kind (``S``, ``U``, or ``O`` with all-string
|
|
132
|
+
content) and applies one of:
|
|
133
|
+
|
|
134
|
+
- **ISO conversion**: vars named in ``iso_targets`` are converted from ISO-8601
|
|
135
|
+
UTC strings to ``datetime64[s]`` or ``datetime64[us]`` (auto-detected by
|
|
136
|
+
fractional-second presence). By default, ``units`` and ``calendar`` attrs (if
|
|
137
|
+
present on the source variable) are moved from ``.attrs`` to ``.encoding`` on
|
|
138
|
+
the converted variable — matching xarray's convention for decoded CF time
|
|
139
|
+
metadata (metadata consumed during conversion is relocated to encoding rather
|
|
140
|
+
than deleted). Set ``preserve_cf_time_attrs=True`` to keep them verbatim in
|
|
141
|
+
``.attrs``; note that firecube's CF advisor may then flag the variable. The
|
|
142
|
+
returned ISO-converted variable always receives fresh ``.encoding`` — source
|
|
143
|
+
encoding hints (chunks, dtype, pre-existing time-decode keys) are NOT copied.
|
|
144
|
+
- **UTF-8 decode**: other string vars are decoded to ``<U*>`` (bytes decoded,
|
|
145
|
+
object arrays consolidated to fixed-width).
|
|
146
|
+
|
|
147
|
+
Variables whose dtype is ``datetime64`` are silently skipped even if named in
|
|
148
|
+
``iso_targets`` (already decoded — nothing to do). Variables with non-string
|
|
149
|
+
object content are also silently skipped, with a DEBUG log when ``logger`` is
|
|
150
|
+
provided.
|
|
151
|
+
|
|
152
|
+
Raises ``ValueError`` only when a name in ``iso_targets`` exists in the dataset
|
|
153
|
+
AND its data cannot be processed (empty object array, non-string object content,
|
|
154
|
+
unparseable ISO string, or invalid UTF-8 bytes). A name in ``iso_targets`` that
|
|
155
|
+
is absent from the dataset is silently skipped.
|
|
156
|
+
|
|
157
|
+
This utility normalizes detectable string variables. It does NOT validate
|
|
158
|
+
whole-dataset Zarr writability; callers who require strict schemas should run a
|
|
159
|
+
separate validator.
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
ds: Input dataset (post-concat is fine; handles vlen-string widening).
|
|
163
|
+
iso_targets: Variable names to convert to ``datetime64``. ``None`` skips
|
|
164
|
+
ISO conversion (UTF-8 decode is still applied to other string vars).
|
|
165
|
+
logger: Optional logger for DEBUG-level skip diagnostics.
|
|
166
|
+
preserve_cf_time_attrs: If True, keep ``units`` and ``calendar`` attrs on
|
|
167
|
+
ISO-converted variables verbatim in ``.attrs`` and do NOT write them to
|
|
168
|
+
``.encoding``. Default False moves them to ``.encoding`` (xarray convention).
|
|
169
|
+
Ignored for CF-time attr handling when no variable is ISO-converted; other
|
|
170
|
+
string normalization still runs.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
A new dataset. The input is not mutated.
|
|
174
|
+
"""
|
|
175
|
+
iso_target_set: frozenset[str] = frozenset(iso_targets or ())
|
|
176
|
+
replacements: dict[str, xr.DataArray] = {}
|
|
177
|
+
|
|
178
|
+
for name, var in ds.data_vars.items():
|
|
179
|
+
var_name = str(name)
|
|
180
|
+
values = np.asarray(var.values)
|
|
181
|
+
|
|
182
|
+
if values.dtype.kind == "M":
|
|
183
|
+
continue
|
|
184
|
+
|
|
185
|
+
is_str = _is_string_var_pure(values)
|
|
186
|
+
in_targets = var_name in iso_target_set
|
|
187
|
+
|
|
188
|
+
if in_targets and not is_str:
|
|
189
|
+
raise ValueError(
|
|
190
|
+
f"variable {var_name!r} is in iso_targets but is not a normalizable "
|
|
191
|
+
f"string var (dtype.kind={values.dtype.kind!r}, size={values.size})"
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
if not is_str:
|
|
195
|
+
if values.dtype.kind == "O" and logger is not None:
|
|
196
|
+
logger.debug(
|
|
197
|
+
"normalize_string_vars: skipping object-dtype var %r (size=%d)",
|
|
198
|
+
var_name,
|
|
199
|
+
values.size,
|
|
200
|
+
)
|
|
201
|
+
continue
|
|
202
|
+
|
|
203
|
+
strs = _values_to_str_list(values)
|
|
204
|
+
if in_targets:
|
|
205
|
+
new_values = _iso_strs_to_datetime64(strs).reshape(values.shape)
|
|
206
|
+
new_attrs = dict(var.attrs)
|
|
207
|
+
new_encoding: dict[str, object] = {}
|
|
208
|
+
if not preserve_cf_time_attrs:
|
|
209
|
+
# Match xarray's pop_to() convention (xarray/coding/times.py:1407):
|
|
210
|
+
# attrs consumed during type conversion move to .encoding, preserving
|
|
211
|
+
# metadata without polluting .attrs on the decoded array.
|
|
212
|
+
for key in ("units", "calendar"):
|
|
213
|
+
if key in new_attrs:
|
|
214
|
+
new_encoding[key] = new_attrs.pop(key)
|
|
215
|
+
new_da = xr.DataArray(new_values, dims=var.dims, attrs=new_attrs, name=var_name)
|
|
216
|
+
new_da.encoding = new_encoding
|
|
217
|
+
replacements[var_name] = new_da
|
|
218
|
+
else:
|
|
219
|
+
new_values = np.asarray(strs, dtype=object).reshape(values.shape)
|
|
220
|
+
with suppress(UnicodeDecodeError, ValueError):
|
|
221
|
+
new_values = new_values.astype(str)
|
|
222
|
+
replacements[var_name] = xr.DataArray(
|
|
223
|
+
new_values, dims=var.dims, attrs=dict(var.attrs), name=var_name
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
return ds.assign(replacements)
|