firecube 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. firecube/__init__.py +23 -0
  2. firecube/__main__.py +20 -0
  3. firecube/_version.py +46 -0
  4. firecube/cli/_audience.py +135 -0
  5. firecube/cli/_command_schemas.py +112 -0
  6. firecube/cli/_ctx.py +85 -0
  7. firecube/cli/_errors.py +115 -0
  8. firecube/cli/_formatter.py +195 -0
  9. firecube/cli/_product.py +47 -0
  10. firecube/cli/_rename_hints.py +123 -0
  11. firecube/cli/_shared_options.py +247 -0
  12. firecube/cli/_slot_env.py +102 -0
  13. firecube/cli/_slot_planning.py +95 -0
  14. firecube/cli/_typed_options.py +181 -0
  15. firecube/cli/_uri_policy.py +110 -0
  16. firecube/cli/advise.py +321 -0
  17. firecube/cli/archive.py +870 -0
  18. firecube/cli/catalog.py +196 -0
  19. firecube/cli/chunks/__init__.py +74 -0
  20. firecube/cli/chunks/_claims.py +229 -0
  21. firecube/cli/chunks/_common.py +84 -0
  22. firecube/cli/chunks/_delete.py +376 -0
  23. firecube/cli/chunks/_list.py +334 -0
  24. firecube/cli/chunks/_manager.py +161 -0
  25. firecube/cli/chunks/_runs.py +294 -0
  26. firecube/cli/chunks/_snapshots.py +165 -0
  27. firecube/cli/completion.py +73 -0
  28. firecube/cli/main.py +866 -0
  29. firecube/cli/parquet.py +289 -0
  30. firecube/cli/plugins/__init__.py +28 -0
  31. firecube/cli/plugins/commands.py +262 -0
  32. firecube/cli/plugins/introspect.py +304 -0
  33. firecube/cli/plugins/mgmt.py +198 -0
  34. firecube/cli/plugins/registry.py +132 -0
  35. firecube/cli/zarr.py +1043 -0
  36. firecube/core/__init__.py +14 -0
  37. firecube/core/api.py +88 -0
  38. firecube/core/cf/__init__.py +23 -0
  39. firecube/core/cf/check_ids.py +126 -0
  40. firecube/core/cf/report.py +57 -0
  41. firecube/core/cf/validator.py +313 -0
  42. firecube/core/config.py +307 -0
  43. firecube/core/controlplane/__init__.py +47 -0
  44. firecube/core/controlplane/_event_processor.py +136 -0
  45. firecube/core/controlplane/_helpers.py +114 -0
  46. firecube/core/controlplane/_projection.py +253 -0
  47. firecube/core/controlplane/_snapshot.py +215 -0
  48. firecube/core/controlplane/_wal_reader.py +402 -0
  49. firecube/core/controlplane/_wal_writer.py +589 -0
  50. firecube/core/controlplane/claims.py +294 -0
  51. firecube/core/controlplane/deletion.py +704 -0
  52. firecube/core/controlplane/events.py +219 -0
  53. firecube/core/controlplane/manager.py +990 -0
  54. firecube/core/controlplane/metrics.py +97 -0
  55. firecube/core/controlplane/repo.py +804 -0
  56. firecube/core/controlplane/repo_utils.py +73 -0
  57. firecube/core/controlplane/time_dim.py +111 -0
  58. firecube/core/controlplane/types.py +445 -0
  59. firecube/core/credentials.py +36 -0
  60. firecube/core/duckdb/__init__.py +27 -0
  61. firecube/core/duckdb/bridge.py +82 -0
  62. firecube/core/duckdb/io.py +30 -0
  63. firecube/core/errors.py +74 -0
  64. firecube/core/filesystem/__init__.py +60 -0
  65. firecube/core/filesystem/_obstore_compat.py +60 -0
  66. firecube/core/filesystem/fsspec_backend.py +308 -0
  67. firecube/core/filesystem/instrumentation.py +260 -0
  68. firecube/core/filesystem/obstore_backend.py +406 -0
  69. firecube/core/filesystem/ops.py +412 -0
  70. firecube/core/filesystem/protocol.py +130 -0
  71. firecube/core/filesystem/store_factory.py +149 -0
  72. firecube/core/formats/__init__.py +46 -0
  73. firecube/core/formats/_iso.py +84 -0
  74. firecube/core/formats/discovery.py +165 -0
  75. firecube/core/formats/hdf5.py +137 -0
  76. firecube/core/formats/netcdf.py +226 -0
  77. firecube/core/formats/zip.py +192 -0
  78. firecube/core/intake.py +530 -0
  79. firecube/core/observability/__init__.py +72 -0
  80. firecube/core/observability/_state.py +24 -0
  81. firecube/core/observability/logging.py +147 -0
  82. firecube/core/observability/metrics.py +230 -0
  83. firecube/core/observability/telemetry/__init__.py +87 -0
  84. firecube/core/observability/telemetry/pushgateway.py +339 -0
  85. firecube/core/observability/telemetry/sinks.py +155 -0
  86. firecube/core/observability/tracing.py +126 -0
  87. firecube/core/product/__init__.py +45 -0
  88. firecube/core/product/identity.py +76 -0
  89. firecube/core/product/resolver.py +121 -0
  90. firecube/core/product/target.py +120 -0
  91. firecube/core/runtime.py +187 -0
  92. firecube/core/slot_index.py +205 -0
  93. firecube/core/storage/__init__.py +45 -0
  94. firecube/core/storage/binding.py +39 -0
  95. firecube/core/storage/cache_key.py +40 -0
  96. firecube/core/storage/completion.py +245 -0
  97. firecube/core/storage/driver_config.py +125 -0
  98. firecube/core/storage/results.py +30 -0
  99. firecube/core/storage/session.py +431 -0
  100. firecube/core/storage/transfer.py +142 -0
  101. firecube/core/storage/uri.py +173 -0
  102. firecube/core/tensogram/__init__.py +37 -0
  103. firecube/core/tensogram/_compat.py +36 -0
  104. firecube/core/tensogram/controlplane_codec.py +230 -0
  105. firecube/core/tensogram/converter.py +390 -0
  106. firecube/core/tensogram/metadata.py +274 -0
  107. firecube/core/tensogram/restore.py +303 -0
  108. firecube/core/tensogram/schema.py +93 -0
  109. firecube/core/uris.py +182 -0
  110. firecube/core/workspaces.py +68 -0
  111. firecube/core/zarr/__init__.py +69 -0
  112. firecube/core/zarr/_reserved_attrs.py +47 -0
  113. firecube/core/zarr/_reserved_root_attrs.py +74 -0
  114. firecube/core/zarr/io.py +88 -0
  115. firecube/core/zarr/layers.py +184 -0
  116. firecube/core/zarr/multires.py +178 -0
  117. firecube/core/zarr/region_writer.py +782 -0
  118. firecube/core/zarr/scrub.py +200 -0
  119. firecube/core/zarr/state.py +258 -0
  120. firecube/core/zarr/time_decode.py +71 -0
  121. firecube/core/zarr/validation.py +622 -0
  122. firecube/ingestor/__init__.py +27 -0
  123. firecube/ingestor/api.py +215 -0
  124. firecube/ingestor/config/__init__.py +19 -0
  125. firecube/ingestor/config/coercion.py +157 -0
  126. firecube/ingestor/config/engine.py +188 -0
  127. firecube/ingestor/contracts/__init__.py +14 -0
  128. firecube/ingestor/contracts/interfaces.py +235 -0
  129. firecube/ingestor/devtools/_templates/__init__.py +14 -0
  130. firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
  131. firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
  132. firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
  133. firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
  134. firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
  135. firecube/ingestor/devtools/scaffolding.py +229 -0
  136. firecube/ingestor/errors.py +79 -0
  137. firecube/ingestor/extensions/_binning.py +152 -0
  138. firecube/ingestor/extensions/duck.py +153 -0
  139. firecube/ingestor/extensions/grid.py +287 -0
  140. firecube/ingestor/extensions/healpix.py +330 -0
  141. firecube/ingestor/registry/__init__.py +14 -0
  142. firecube/ingestor/registry/loader.py +144 -0
  143. firecube/ingestor/registry/metadata.py +97 -0
  144. firecube/ingestor/registry/version_compat.py +95 -0
  145. firecube/ingestor/runtime/__init__.py +14 -0
  146. firecube/ingestor/runtime/aggregation.py +112 -0
  147. firecube/ingestor/runtime/base.py +836 -0
  148. firecube/ingestor/runtime/base_hooks.py +129 -0
  149. firecube/ingestor/runtime/batching.py +161 -0
  150. firecube/ingestor/runtime/configure.py +195 -0
  151. firecube/ingestor/runtime/coverage.py +177 -0
  152. firecube/ingestor/runtime/engine.py +793 -0
  153. firecube/ingestor/runtime/parallel_evidence.py +75 -0
  154. firecube/ingestor/runtime/parallel_execution_state.py +28 -0
  155. firecube/ingestor/runtime/parallel_gate.py +164 -0
  156. firecube/ingestor/runtime/parallel_run_id.py +48 -0
  157. firecube/ingestor/runtime/preflight.py +40 -0
  158. firecube/ingestor/runtime/recording.py +425 -0
  159. firecube/ingestor/runtime/resume_guard.py +556 -0
  160. firecube/ingestor/runtime/resume_types.py +65 -0
  161. firecube/ingestor/runtime/telemetry.py +171 -0
  162. firecube/ingestor/runtime/tensogram/__init__.py +14 -0
  163. firecube/ingestor/runtime/tensogram/strategy.py +164 -0
  164. firecube/ingestor/runtime/workspace.py +220 -0
  165. firecube/ingestor/runtime/zarr/__init__.py +34 -0
  166. firecube/ingestor/runtime/zarr/append.py +466 -0
  167. firecube/ingestor/runtime/zarr/append_services.py +650 -0
  168. firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
  169. firecube/ingestor/runtime/zarr/contracts.py +58 -0
  170. firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
  171. firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
  172. firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
  173. firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
  174. firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
  175. firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
  176. firecube/ingestor/runtime/zarr/write.py +212 -0
  177. firecube/ingestor/runtime/zarr/write_context.py +139 -0
  178. firecube/ingestor/templates/__init__.py +19 -0
  179. firecube/ingestor/templates/config.py +118 -0
  180. firecube/ingestor/templates/direct_zarr.py +790 -0
  181. firecube/ingestor/templates/generic.py +462 -0
  182. firecube/ingestor/templates/generic_tensogram.py +170 -0
  183. firecube/ingestor/types/__init__.py +19 -0
  184. firecube/ingestor/types/config.py +89 -0
  185. firecube/ingestor/types/context.py +409 -0
  186. firecube/ingestor/types/manifest.py +53 -0
  187. firecube/ingestor/types/planned_range.py +152 -0
  188. firecube/ingestor/types/result_metrics.py +386 -0
  189. firecube/ingestor/utils/__init__.py +14 -0
  190. firecube/ingestor/utils/duckdb_utils.py +235 -0
  191. firecube/ingestor/validation.py +55 -0
  192. firecube-0.1.4.dist-info/METADATA +242 -0
  193. firecube-0.1.4.dist-info/RECORD +197 -0
  194. firecube-0.1.4.dist-info/WHEEL +4 -0
  195. firecube-0.1.4.dist-info/entry_points.txt +2 -0
  196. firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
  197. firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,46 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Generic input format helpers (ZIP, HDF5, NetCDF, file discovery).
16
+
17
+ Public imports remain stable at ``firecube.core.formats``.
18
+ """
19
+
20
+ from firecube.core.formats.discovery import KNOWN_EXTENSIONS, discover_input_files
21
+ from firecube.core.formats.hdf5 import materialize_hdf5_path, read_hdf5_array
22
+ from firecube.core.formats.netcdf import (
23
+ clean_netcdf_encoding,
24
+ normalize_string_vars,
25
+ prepare_netcdf_for_zarr,
26
+ rename_time_dim,
27
+ )
28
+ from firecube.core.formats.zip import (
29
+ extract_hdf5_from_zip,
30
+ extract_zip_files_parallel,
31
+ stream_hdf5_from_zip,
32
+ )
33
+
34
+ __all__ = [
35
+ "KNOWN_EXTENSIONS",
36
+ "clean_netcdf_encoding",
37
+ "discover_input_files",
38
+ "extract_hdf5_from_zip",
39
+ "extract_zip_files_parallel",
40
+ "materialize_hdf5_path",
41
+ "normalize_string_vars",
42
+ "prepare_netcdf_for_zarr",
43
+ "read_hdf5_array",
44
+ "rename_time_dim",
45
+ "stream_hdf5_from_zip",
46
+ ]
@@ -0,0 +1,84 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Strict UTC ISO-8601 → numpy.datetime64 helpers. Package-private."""
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Iterable
20
+ from datetime import datetime, timedelta
21
+
22
+ import numpy as np
23
+
24
+
25
+ def _parse_iso_utc(s: str) -> datetime:
26
+ """Parse a strict UTC ISO-8601 string to a naive UTC datetime.
27
+
28
+ Accepts Z suffix or explicit +00:00 offset. Rejects non-UTC offsets
29
+ with NotImplementedError. Rejects timezone-naive, malformed, or empty strings
30
+ with ValueError.
31
+ """
32
+ if not isinstance(s, str):
33
+ raise ValueError(f"expected str, got {type(s).__name__!r}")
34
+ if not s:
35
+ raise ValueError("empty ISO string")
36
+ if _fractional_second_digits(s) > 6:
37
+ raise ValueError(f"unparseable ISO string: {s!r}")
38
+
39
+ normalized = s[:-1] + "+00:00" if s.endswith("Z") else s
40
+ try:
41
+ dt = datetime.fromisoformat(normalized)
42
+ except ValueError as exc:
43
+ raise ValueError(f"unparseable ISO string: {s!r}") from exc
44
+
45
+ if dt.tzinfo is None:
46
+ raise ValueError(
47
+ f"timezone-naive ISO string: {s!r}; expected UTC with 'Z' or '+00:00' suffix"
48
+ )
49
+ if dt.utcoffset() != timedelta(0):
50
+ raise NotImplementedError(f"non-UTC ISO string: {s!r} (offset={dt.utcoffset()})")
51
+ dt = dt.replace(tzinfo=None)
52
+
53
+ return dt
54
+
55
+
56
+ def _fractional_second_digits(s: str) -> int:
57
+ if "." not in s:
58
+ return 0
59
+
60
+ fraction = s.rsplit(".", maxsplit=1)[1]
61
+ for separator in ("Z", "+", "-"):
62
+ fraction = fraction.split(separator, maxsplit=1)[0]
63
+ return len(fraction)
64
+
65
+
66
+ def _iso_strs_to_datetime64(strs: Iterable[str]) -> np.ndarray:
67
+ """Convert an iterable of strict UTC ISO-8601 strings to datetime64.
68
+
69
+ Precision is auto-detected from the input: datetime64[s] when no string
70
+ contains a fractional-second component, datetime64[us] otherwise.
71
+ The whole array uses a single unit (upgraded to [us] if any element has
72
+ fractional-second syntax).
73
+
74
+ Raises ValueError for malformed or non-UTC strings, and for strings with
75
+ more than 6 fractional digits (nanosecond precision; stdlib limit).
76
+ """
77
+ strs_list = list(strs)
78
+ if not strs_list:
79
+ return np.array([], dtype="datetime64[s]")
80
+
81
+ has_frac = any("." in s for s in strs_list)
82
+ unit = "us" if has_frac else "s"
83
+ parsed = [_parse_iso_utc(s) for s in strs_list]
84
+ return np.asarray(parsed, dtype=f"datetime64[{unit}]")
@@ -0,0 +1,165 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Input file discovery helpers for ingestion plugins.
16
+
17
+ Format-agnostic discovery utilities that walk a local path or remote URI
18
+ and return candidate input file URIs (ZIP, HDF5, NetCDF, ...).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import contextlib
24
+ import fnmatch
25
+ from collections.abc import Iterable, Sequence
26
+ from pathlib import Path
27
+ from typing import Any
28
+
29
+ from firecube.core.formats.hdf5 import looks_like_hdf5
30
+ from firecube.core.uris import is_remote_target, parse_uri
31
+
32
+ # All file extensions recognised by the firecube ecosystem.
33
+ # The first three are the defaults for ``discover_input_files``; additional
34
+ # formats (e.g. ``.tgm``) must be requested explicitly via *include_suffixes*.
35
+ KNOWN_EXTENSIONS: frozenset[str] = frozenset({".zip", ".h5", ".nc", ".tgm"})
36
+
37
+
38
+ def _path_matches_any_glob(
39
+ path: str, patterns: Iterable[str], *, candidates: Iterable[str]
40
+ ) -> bool:
41
+ """Return True when any candidate path matches any glob pattern."""
42
+ normalized_candidates = {path, *candidates}
43
+ return any(
44
+ fnmatch.fnmatch(candidate, pattern)
45
+ for pattern in patterns
46
+ for candidate in normalized_candidates
47
+ )
48
+
49
+
50
+ def _filter_discovered_paths(
51
+ paths: Iterable[str],
52
+ *,
53
+ source_uri: str,
54
+ include_suffixes: Sequence[str],
55
+ preferred_globs: Iterable[str] | None,
56
+ recursive: bool,
57
+ sniff_hdf5: bool,
58
+ exclude: Iterable[str] | None,
59
+ fs: Any,
60
+ root: str,
61
+ ) -> list[str]:
62
+ """Filter discovered filesystem paths down to candidate input files."""
63
+ remote_source = is_remote_target(source_uri)
64
+ suffixes = {s.lower() for s in include_suffixes}
65
+ preferred_patterns = tuple(preferred_globs or ())
66
+ excluded_patterns = tuple(exclude or ())
67
+ source_is_dir = False
68
+ with contextlib.suppress(Exception):
69
+ source_is_dir = bool(fs.isdir(root))
70
+
71
+ root_prefix = root.rstrip("/")
72
+ resolved: list[str] = []
73
+
74
+ for path in paths:
75
+ normalized = str(path)
76
+ path_for_match = parse_uri(normalized)["path"] if remote_source else normalized
77
+ basename = Path(path_for_match).name
78
+ relative = path_for_match
79
+
80
+ if root_prefix and path_for_match.startswith(f"{root_prefix}/"):
81
+ relative = path_for_match[len(root_prefix) + 1 :]
82
+ elif path_for_match == root_prefix:
83
+ relative = basename
84
+
85
+ if not recursive and "/" in relative:
86
+ continue
87
+
88
+ glob_candidates = {basename, relative, normalized}
89
+
90
+ if excluded_patterns and _path_matches_any_glob(
91
+ normalized, excluded_patterns, candidates=glob_candidates
92
+ ):
93
+ continue
94
+
95
+ suffix = Path(path_for_match).suffix.lower()
96
+ include_by_suffix = suffix in suffixes
97
+ include_by_sniff = False
98
+ if sniff_hdf5 and (suffix == "" or suffix not in suffixes):
99
+ include_by_sniff = looks_like_hdf5(Path(normalized))
100
+
101
+ include_by_glob = False
102
+ if preferred_patterns and source_is_dir:
103
+ include_by_glob = _path_matches_any_glob(
104
+ normalized, preferred_patterns, candidates=glob_candidates
105
+ )
106
+
107
+ if include_by_suffix or include_by_sniff or include_by_glob:
108
+ resolved.append(normalized)
109
+
110
+ dedup = dict.fromkeys(resolved)
111
+ return sorted(dedup, key=lambda path: Path(path).name)
112
+
113
+
114
+ def discover_input_files(
115
+ source: str | Path,
116
+ *,
117
+ storage_config: Any | None = None,
118
+ include_suffixes: Sequence[str] = (".zip", ".h5", ".nc"),
119
+ preferred_globs: Iterable[str] | None = None,
120
+ recursive: bool = True,
121
+ sniff_hdf5: bool = True,
122
+ exclude: Iterable[str] | None = None,
123
+ ) -> list[str]:
124
+ """Discover input files from a local path or remote URI.
125
+
126
+ Selection is intentionally conservative and format-agnostic:
127
+ - Accept files matching `include_suffixes`
128
+ - Optionally accept extensionless files that look like HDF5
129
+ - Optionally add files matched by `preferred_globs` (glob patterns)
130
+
131
+ Returns URI/path strings (for example ``/tmp/data/file.nc`` or
132
+ ``s3://bucket/prefix/file.nc``). ``Path`` inputs are accepted for backward
133
+ compatibility and are converted to strings internally.
134
+ """
135
+ from firecube.core.filesystem.ops import _open_fsspec_url
136
+ from firecube.core.uris import is_remote_target, parse_uri
137
+
138
+ source_uri = str(source)
139
+ is_remote = is_remote_target(source_uri)
140
+
141
+ try:
142
+ fs, root = _open_fsspec_url(source_uri, storage_config=storage_config)
143
+ except Exception as exc:
144
+ raise ValueError(f"Cannot open source location {source_uri!r}: {exc}") from exc
145
+
146
+ try:
147
+ all_paths = [str(path) for path in fs.find(root)]
148
+ except Exception as exc:
149
+ raise ValueError(f"Cannot list source location {source_uri!r}: {exc}") from exc
150
+
151
+ if is_remote:
152
+ protocol = parse_uri(source_uri)["protocol"]
153
+ all_paths = [path if "://" in path else f"{protocol}://{path}" for path in all_paths]
154
+
155
+ return _filter_discovered_paths(
156
+ all_paths,
157
+ source_uri=source_uri,
158
+ include_suffixes=include_suffixes,
159
+ preferred_globs=preferred_globs,
160
+ recursive=recursive,
161
+ sniff_hdf5=sniff_hdf5 and not is_remote,
162
+ exclude=exclude,
163
+ fs=fs,
164
+ root=root,
165
+ )
@@ -0,0 +1,137 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """HDF5 helpers for ingestion plugins.
16
+
17
+ Reusable helpers for sniffing, reading, and materializing HDF5 inputs
18
+ (either standalone files or members embedded inside ZIP archives).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import contextlib
24
+ import logging
25
+ import tempfile
26
+ from pathlib import Path
27
+ from typing import Any
28
+
29
+ from firecube.core.formats.zip import extract_hdf5_from_zip
30
+
31
+ log = logging.getLogger("firecube.core.formats")
32
+
33
+
34
+ def looks_like_hdf5(path: Path) -> bool:
35
+ """Best-effort magic-number check for HDF5 files without extensions."""
36
+ try:
37
+ with path.open("rb") as fh:
38
+ sig = fh.read(8)
39
+ return sig == b"\x89HDF\r\n\x1a\n"
40
+ except Exception:
41
+ return False
42
+
43
+
44
+ def read_hdf5_array(
45
+ hdf5_path: Path,
46
+ *,
47
+ variable: str,
48
+ logger: logging.Logger | None = None,
49
+ ) -> Any:
50
+ """Read a named array from a local HDF5(-like) file, with xarray fallback.
51
+
52
+ Args:
53
+ hdf5_path: Path to the HDF5 file.
54
+ variable: Name of the variable/dataset to read.
55
+ logger: Optional logger for debug messages.
56
+
57
+ Returns:
58
+ Numpy array of the data (float32).
59
+
60
+ Raises:
61
+ KeyError: If the variable is not found.
62
+ RuntimeError: If dependencies are missing or read fails.
63
+ """
64
+ logger = logger or log
65
+ try:
66
+ import numpy as np
67
+ except Exception as exc:
68
+ raise RuntimeError("numpy is required to read HDF5 inputs") from exc
69
+
70
+ try:
71
+ import h5py
72
+ except Exception as exc:
73
+ raise RuntimeError("h5py is required to read HDF5 inputs") from exc
74
+
75
+ try:
76
+ with h5py.File(hdf5_path, "r") as handle:
77
+ if variable not in handle:
78
+ raise KeyError(f"{variable} dataset not found in HDF5 file")
79
+ return np.asarray(handle[variable], dtype=np.float32)
80
+ except Exception as h5_exc:
81
+ try:
82
+ import xarray as xr
83
+
84
+ ds = xr.open_dataset(hdf5_path, phony_dims="sort")
85
+ if variable not in ds:
86
+ raise KeyError(f"{variable} variable not found in dataset")
87
+ arr = ds[variable].values.astype(np.float32, copy=False)
88
+ ds.close()
89
+ return arr
90
+ except Exception as xr_exc:
91
+ raise RuntimeError(
92
+ f"Failed to read {variable} from {hdf5_path.name}: {h5_exc}"
93
+ ) from xr_exc
94
+ finally:
95
+ with contextlib.suppress(Exception):
96
+ logger.debug("Fell back to xarray open_dataset for %s", hdf5_path.name)
97
+
98
+
99
+ def materialize_hdf5_path(
100
+ file_path: Path,
101
+ *,
102
+ extract_root: Path | None = None,
103
+ logger: logging.Logger | None = None,
104
+ ) -> tuple[Path, tempfile.TemporaryDirectory | None]:
105
+ """Return a local HDF5 path for a source path (ZIP or direct HDF5-like file).
106
+
107
+ If the input is a ZIP file, it extracts the content to a temporary directory.
108
+ If it is already an HDF5-like file, it is returned as-is.
109
+
110
+ Args:
111
+ file_path: Source file path.
112
+ extract_root: Optional root directory for temporary extraction.
113
+ logger: Optional logger.
114
+
115
+ Returns:
116
+ Tuple of (path_to_hdf5_file, temporary_directory_object_or_None).
117
+ """
118
+ logger = logger or log
119
+ if file_path.suffix.lower() != ".zip":
120
+ return file_path, None
121
+
122
+ tmp_dir: tempfile.TemporaryDirectory | None
123
+ if extract_root is None:
124
+ tmp_dir = tempfile.TemporaryDirectory()
125
+ else:
126
+ extract_root.mkdir(parents=True, exist_ok=True)
127
+ tmp_dir = tempfile.TemporaryDirectory(dir=str(extract_root))
128
+
129
+ try:
130
+ extracted = extract_hdf5_from_zip(file_path, Path(tmp_dir.name), logger=logger)
131
+ if extracted is None:
132
+ raise FileNotFoundError(f"No HDF5 member found in archive {file_path.name}")
133
+ return extracted, tmp_dir
134
+ except Exception:
135
+ with contextlib.suppress(Exception):
136
+ tmp_dir.cleanup()
137
+ raise
@@ -0,0 +1,226 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """NetCDF-to-Zarr preparation utilities.
16
+
17
+ Composable helpers that normalize xarray Datasets loaded from NetCDF
18
+ sources into the shape expected by Firecube's Zarr writer. Plugin-agnostic.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import logging
24
+ from collections.abc import Collection
25
+ from contextlib import suppress
26
+
27
+ import numpy as np
28
+ import xarray as xr
29
+
30
+ from firecube.core.formats._iso import _iso_strs_to_datetime64
31
+
32
+
33
+ def clean_netcdf_encoding(ds: xr.Dataset) -> xr.Dataset:
34
+ """Strip HDF5 chunk hints from all variable encodings.
35
+
36
+ Removes 'chunks', 'chunksizes', and 'preferred_chunks' from each
37
+ variable's encoding dict so they do not conflict with the Zarr chunk
38
+ layout chosen by the core writer. Modifies encoding dicts in place;
39
+ returns the same dataset object.
40
+ """
41
+ for var_name in list(ds.variables):
42
+ enc = ds[var_name].encoding
43
+ enc.pop("chunks", None)
44
+ enc.pop("chunksizes", None)
45
+ enc.pop("preferred_chunks", None)
46
+ return ds
47
+
48
+
49
+ def rename_time_dim(
50
+ ds: xr.Dataset,
51
+ source: str = "time",
52
+ target: str = "timestamp",
53
+ ) -> xr.Dataset:
54
+ """Rename the time dimension to Firecube's append convention.
55
+
56
+ If *source* exists as a dimension or coordinate, rename it to *target*.
57
+ If *source* is not present, return the dataset unchanged (no error).
58
+ """
59
+ if source in ds.dims or source in ds.coords:
60
+ return ds.rename({source: target})
61
+ return ds
62
+
63
+
64
+ def prepare_netcdf_for_zarr(
65
+ ds: xr.Dataset,
66
+ time_dim: str = "time",
67
+ target_time_dim: str = "timestamp",
68
+ ) -> xr.Dataset:
69
+ """Convenience wrapper: rename time dimension then clean encodings.
70
+
71
+ Applies all standard NetCDF→Zarr V3 preparation steps in order:
72
+ 1. Rename time dimension from *time_dim* to *target_time_dim*.
73
+ 2. Strip HDF5 chunk hints from all variable encodings.
74
+ """
75
+ ds = rename_time_dim(ds, source=time_dim, target=target_time_dim)
76
+ ds = clean_netcdf_encoding(ds)
77
+ return ds
78
+
79
+
80
+ # ─── string variable normalization ─────────────────────────
81
+
82
+ _ACCEPTED_ELEMENT_TYPES: tuple[type, ...] = (str, bytes, np.str_, np.bytes_)
83
+
84
+
85
+ def _is_string_var_pure(values: np.ndarray) -> bool:
86
+ """Pure predicate: True when values holds a string-typed var we can normalize.
87
+
88
+ Returns False for anything we cannot confidently classify; never raises.
89
+ """
90
+ kind = values.dtype.kind
91
+ if kind in ("S", "U"):
92
+ return True
93
+ if kind != "O" or values.size == 0:
94
+ return False
95
+ return all(isinstance(x, _ACCEPTED_ELEMENT_TYPES) for x in values.ravel())
96
+
97
+
98
+ def _values_to_str_list(values: np.ndarray) -> list[str]:
99
+ """Convert a string-shaped ndarray to a flat list of Python str."""
100
+ kind = values.dtype.kind
101
+ if kind == "U":
102
+ return values.ravel().tolist()
103
+ if kind == "S":
104
+ try:
105
+ return np.char.decode(values, "utf-8").ravel().tolist()
106
+ except UnicodeDecodeError as exc:
107
+ raise ValueError(f"invalid UTF-8 bytes in string var: {exc}") from exc
108
+ out: list[str] = []
109
+ for x in values.ravel():
110
+ if isinstance(x, str):
111
+ out.append(x)
112
+ elif isinstance(x, (bytes, np.bytes_)):
113
+ try:
114
+ out.append(x.decode("utf-8"))
115
+ except UnicodeDecodeError as exc:
116
+ raise ValueError(f"invalid UTF-8 bytes in string var: {exc}") from exc
117
+ else:
118
+ out.append(str(x))
119
+ return out
120
+
121
+
122
+ def normalize_string_vars(
123
+ ds: xr.Dataset,
124
+ *,
125
+ iso_targets: Collection[str] | None = None,
126
+ logger: logging.Logger | None = None,
127
+ preserve_cf_time_attrs: bool = False,
128
+ ) -> xr.Dataset:
129
+ """Normalize string-typed variables in a NetCDF-loaded Dataset for Zarr writes.
130
+
131
+ Detects string vars by dtype kind (``S``, ``U``, or ``O`` with all-string
132
+ content) and applies one of:
133
+
134
+ - **ISO conversion**: vars named in ``iso_targets`` are converted from ISO-8601
135
+ UTC strings to ``datetime64[s]`` or ``datetime64[us]`` (auto-detected by
136
+ fractional-second presence). By default, ``units`` and ``calendar`` attrs (if
137
+ present on the source variable) are moved from ``.attrs`` to ``.encoding`` on
138
+ the converted variable — matching xarray's convention for decoded CF time
139
+ metadata (metadata consumed during conversion is relocated to encoding rather
140
+ than deleted). Set ``preserve_cf_time_attrs=True`` to keep them verbatim in
141
+ ``.attrs``; note that firecube's CF advisor may then flag the variable. The
142
+ returned ISO-converted variable always receives fresh ``.encoding`` — source
143
+ encoding hints (chunks, dtype, pre-existing time-decode keys) are NOT copied.
144
+ - **UTF-8 decode**: other string vars are decoded to ``<U*>`` (bytes decoded,
145
+ object arrays consolidated to fixed-width).
146
+
147
+ Variables whose dtype is ``datetime64`` are silently skipped even if named in
148
+ ``iso_targets`` (already decoded — nothing to do). Variables with non-string
149
+ object content are also silently skipped, with a DEBUG log when ``logger`` is
150
+ provided.
151
+
152
+ Raises ``ValueError`` only when a name in ``iso_targets`` exists in the dataset
153
+ AND its data cannot be processed (empty object array, non-string object content,
154
+ unparseable ISO string, or invalid UTF-8 bytes). A name in ``iso_targets`` that
155
+ is absent from the dataset is silently skipped.
156
+
157
+ This utility normalizes detectable string variables. It does NOT validate
158
+ whole-dataset Zarr writability; callers who require strict schemas should run a
159
+ separate validator.
160
+
161
+ Args:
162
+ ds: Input dataset (post-concat is fine; handles vlen-string widening).
163
+ iso_targets: Variable names to convert to ``datetime64``. ``None`` skips
164
+ ISO conversion (UTF-8 decode is still applied to other string vars).
165
+ logger: Optional logger for DEBUG-level skip diagnostics.
166
+ preserve_cf_time_attrs: If True, keep ``units`` and ``calendar`` attrs on
167
+ ISO-converted variables verbatim in ``.attrs`` and do NOT write them to
168
+ ``.encoding``. Default False moves them to ``.encoding`` (xarray convention).
169
+ Ignored for CF-time attr handling when no variable is ISO-converted; other
170
+ string normalization still runs.
171
+
172
+ Returns:
173
+ A new dataset. The input is not mutated.
174
+ """
175
+ iso_target_set: frozenset[str] = frozenset(iso_targets or ())
176
+ replacements: dict[str, xr.DataArray] = {}
177
+
178
+ for name, var in ds.data_vars.items():
179
+ var_name = str(name)
180
+ values = np.asarray(var.values)
181
+
182
+ if values.dtype.kind == "M":
183
+ continue
184
+
185
+ is_str = _is_string_var_pure(values)
186
+ in_targets = var_name in iso_target_set
187
+
188
+ if in_targets and not is_str:
189
+ raise ValueError(
190
+ f"variable {var_name!r} is in iso_targets but is not a normalizable "
191
+ f"string var (dtype.kind={values.dtype.kind!r}, size={values.size})"
192
+ )
193
+
194
+ if not is_str:
195
+ if values.dtype.kind == "O" and logger is not None:
196
+ logger.debug(
197
+ "normalize_string_vars: skipping object-dtype var %r (size=%d)",
198
+ var_name,
199
+ values.size,
200
+ )
201
+ continue
202
+
203
+ strs = _values_to_str_list(values)
204
+ if in_targets:
205
+ new_values = _iso_strs_to_datetime64(strs).reshape(values.shape)
206
+ new_attrs = dict(var.attrs)
207
+ new_encoding: dict[str, object] = {}
208
+ if not preserve_cf_time_attrs:
209
+ # Match xarray's pop_to() convention (xarray/coding/times.py:1407):
210
+ # attrs consumed during type conversion move to .encoding, preserving
211
+ # metadata without polluting .attrs on the decoded array.
212
+ for key in ("units", "calendar"):
213
+ if key in new_attrs:
214
+ new_encoding[key] = new_attrs.pop(key)
215
+ new_da = xr.DataArray(new_values, dims=var.dims, attrs=new_attrs, name=var_name)
216
+ new_da.encoding = new_encoding
217
+ replacements[var_name] = new_da
218
+ else:
219
+ new_values = np.asarray(strs, dtype=object).reshape(values.shape)
220
+ with suppress(UnicodeDecodeError, ValueError):
221
+ new_values = new_values.astype(str)
222
+ replacements[var_name] = xr.DataArray(
223
+ new_values, dims=var.dims, attrs=dict(var.attrs), name=var_name
224
+ )
225
+
226
+ return ds.assign(replacements)