firecube 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. firecube/__init__.py +23 -0
  2. firecube/__main__.py +20 -0
  3. firecube/_version.py +46 -0
  4. firecube/cli/_audience.py +135 -0
  5. firecube/cli/_command_schemas.py +112 -0
  6. firecube/cli/_ctx.py +85 -0
  7. firecube/cli/_errors.py +115 -0
  8. firecube/cli/_formatter.py +195 -0
  9. firecube/cli/_product.py +47 -0
  10. firecube/cli/_rename_hints.py +123 -0
  11. firecube/cli/_shared_options.py +247 -0
  12. firecube/cli/_slot_env.py +102 -0
  13. firecube/cli/_slot_planning.py +95 -0
  14. firecube/cli/_typed_options.py +181 -0
  15. firecube/cli/_uri_policy.py +110 -0
  16. firecube/cli/advise.py +321 -0
  17. firecube/cli/archive.py +870 -0
  18. firecube/cli/catalog.py +196 -0
  19. firecube/cli/chunks/__init__.py +74 -0
  20. firecube/cli/chunks/_claims.py +229 -0
  21. firecube/cli/chunks/_common.py +84 -0
  22. firecube/cli/chunks/_delete.py +376 -0
  23. firecube/cli/chunks/_list.py +334 -0
  24. firecube/cli/chunks/_manager.py +161 -0
  25. firecube/cli/chunks/_runs.py +294 -0
  26. firecube/cli/chunks/_snapshots.py +165 -0
  27. firecube/cli/completion.py +73 -0
  28. firecube/cli/main.py +866 -0
  29. firecube/cli/parquet.py +289 -0
  30. firecube/cli/plugins/__init__.py +28 -0
  31. firecube/cli/plugins/commands.py +262 -0
  32. firecube/cli/plugins/introspect.py +304 -0
  33. firecube/cli/plugins/mgmt.py +198 -0
  34. firecube/cli/plugins/registry.py +132 -0
  35. firecube/cli/zarr.py +1043 -0
  36. firecube/core/__init__.py +14 -0
  37. firecube/core/api.py +88 -0
  38. firecube/core/cf/__init__.py +23 -0
  39. firecube/core/cf/check_ids.py +126 -0
  40. firecube/core/cf/report.py +57 -0
  41. firecube/core/cf/validator.py +313 -0
  42. firecube/core/config.py +307 -0
  43. firecube/core/controlplane/__init__.py +47 -0
  44. firecube/core/controlplane/_event_processor.py +136 -0
  45. firecube/core/controlplane/_helpers.py +114 -0
  46. firecube/core/controlplane/_projection.py +253 -0
  47. firecube/core/controlplane/_snapshot.py +215 -0
  48. firecube/core/controlplane/_wal_reader.py +402 -0
  49. firecube/core/controlplane/_wal_writer.py +589 -0
  50. firecube/core/controlplane/claims.py +294 -0
  51. firecube/core/controlplane/deletion.py +704 -0
  52. firecube/core/controlplane/events.py +219 -0
  53. firecube/core/controlplane/manager.py +990 -0
  54. firecube/core/controlplane/metrics.py +97 -0
  55. firecube/core/controlplane/repo.py +804 -0
  56. firecube/core/controlplane/repo_utils.py +73 -0
  57. firecube/core/controlplane/time_dim.py +111 -0
  58. firecube/core/controlplane/types.py +445 -0
  59. firecube/core/credentials.py +36 -0
  60. firecube/core/duckdb/__init__.py +27 -0
  61. firecube/core/duckdb/bridge.py +82 -0
  62. firecube/core/duckdb/io.py +30 -0
  63. firecube/core/errors.py +74 -0
  64. firecube/core/filesystem/__init__.py +60 -0
  65. firecube/core/filesystem/_obstore_compat.py +60 -0
  66. firecube/core/filesystem/fsspec_backend.py +308 -0
  67. firecube/core/filesystem/instrumentation.py +260 -0
  68. firecube/core/filesystem/obstore_backend.py +406 -0
  69. firecube/core/filesystem/ops.py +412 -0
  70. firecube/core/filesystem/protocol.py +130 -0
  71. firecube/core/filesystem/store_factory.py +149 -0
  72. firecube/core/formats/__init__.py +46 -0
  73. firecube/core/formats/_iso.py +84 -0
  74. firecube/core/formats/discovery.py +165 -0
  75. firecube/core/formats/hdf5.py +137 -0
  76. firecube/core/formats/netcdf.py +226 -0
  77. firecube/core/formats/zip.py +192 -0
  78. firecube/core/intake.py +530 -0
  79. firecube/core/observability/__init__.py +72 -0
  80. firecube/core/observability/_state.py +24 -0
  81. firecube/core/observability/logging.py +147 -0
  82. firecube/core/observability/metrics.py +230 -0
  83. firecube/core/observability/telemetry/__init__.py +87 -0
  84. firecube/core/observability/telemetry/pushgateway.py +339 -0
  85. firecube/core/observability/telemetry/sinks.py +155 -0
  86. firecube/core/observability/tracing.py +126 -0
  87. firecube/core/product/__init__.py +45 -0
  88. firecube/core/product/identity.py +76 -0
  89. firecube/core/product/resolver.py +121 -0
  90. firecube/core/product/target.py +120 -0
  91. firecube/core/runtime.py +187 -0
  92. firecube/core/slot_index.py +205 -0
  93. firecube/core/storage/__init__.py +45 -0
  94. firecube/core/storage/binding.py +39 -0
  95. firecube/core/storage/cache_key.py +40 -0
  96. firecube/core/storage/completion.py +245 -0
  97. firecube/core/storage/driver_config.py +125 -0
  98. firecube/core/storage/results.py +30 -0
  99. firecube/core/storage/session.py +431 -0
  100. firecube/core/storage/transfer.py +142 -0
  101. firecube/core/storage/uri.py +173 -0
  102. firecube/core/tensogram/__init__.py +37 -0
  103. firecube/core/tensogram/_compat.py +36 -0
  104. firecube/core/tensogram/controlplane_codec.py +230 -0
  105. firecube/core/tensogram/converter.py +390 -0
  106. firecube/core/tensogram/metadata.py +274 -0
  107. firecube/core/tensogram/restore.py +303 -0
  108. firecube/core/tensogram/schema.py +93 -0
  109. firecube/core/uris.py +182 -0
  110. firecube/core/workspaces.py +68 -0
  111. firecube/core/zarr/__init__.py +69 -0
  112. firecube/core/zarr/_reserved_attrs.py +47 -0
  113. firecube/core/zarr/_reserved_root_attrs.py +74 -0
  114. firecube/core/zarr/io.py +88 -0
  115. firecube/core/zarr/layers.py +184 -0
  116. firecube/core/zarr/multires.py +178 -0
  117. firecube/core/zarr/region_writer.py +782 -0
  118. firecube/core/zarr/scrub.py +200 -0
  119. firecube/core/zarr/state.py +258 -0
  120. firecube/core/zarr/time_decode.py +71 -0
  121. firecube/core/zarr/validation.py +622 -0
  122. firecube/ingestor/__init__.py +27 -0
  123. firecube/ingestor/api.py +215 -0
  124. firecube/ingestor/config/__init__.py +19 -0
  125. firecube/ingestor/config/coercion.py +157 -0
  126. firecube/ingestor/config/engine.py +188 -0
  127. firecube/ingestor/contracts/__init__.py +14 -0
  128. firecube/ingestor/contracts/interfaces.py +235 -0
  129. firecube/ingestor/devtools/_templates/__init__.py +14 -0
  130. firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
  131. firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
  132. firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
  133. firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
  134. firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
  135. firecube/ingestor/devtools/scaffolding.py +229 -0
  136. firecube/ingestor/errors.py +79 -0
  137. firecube/ingestor/extensions/_binning.py +152 -0
  138. firecube/ingestor/extensions/duck.py +153 -0
  139. firecube/ingestor/extensions/grid.py +287 -0
  140. firecube/ingestor/extensions/healpix.py +330 -0
  141. firecube/ingestor/registry/__init__.py +14 -0
  142. firecube/ingestor/registry/loader.py +144 -0
  143. firecube/ingestor/registry/metadata.py +97 -0
  144. firecube/ingestor/registry/version_compat.py +95 -0
  145. firecube/ingestor/runtime/__init__.py +14 -0
  146. firecube/ingestor/runtime/aggregation.py +112 -0
  147. firecube/ingestor/runtime/base.py +836 -0
  148. firecube/ingestor/runtime/base_hooks.py +129 -0
  149. firecube/ingestor/runtime/batching.py +161 -0
  150. firecube/ingestor/runtime/configure.py +195 -0
  151. firecube/ingestor/runtime/coverage.py +177 -0
  152. firecube/ingestor/runtime/engine.py +793 -0
  153. firecube/ingestor/runtime/parallel_evidence.py +75 -0
  154. firecube/ingestor/runtime/parallel_execution_state.py +28 -0
  155. firecube/ingestor/runtime/parallel_gate.py +164 -0
  156. firecube/ingestor/runtime/parallel_run_id.py +48 -0
  157. firecube/ingestor/runtime/preflight.py +40 -0
  158. firecube/ingestor/runtime/recording.py +425 -0
  159. firecube/ingestor/runtime/resume_guard.py +556 -0
  160. firecube/ingestor/runtime/resume_types.py +65 -0
  161. firecube/ingestor/runtime/telemetry.py +171 -0
  162. firecube/ingestor/runtime/tensogram/__init__.py +14 -0
  163. firecube/ingestor/runtime/tensogram/strategy.py +164 -0
  164. firecube/ingestor/runtime/workspace.py +220 -0
  165. firecube/ingestor/runtime/zarr/__init__.py +34 -0
  166. firecube/ingestor/runtime/zarr/append.py +466 -0
  167. firecube/ingestor/runtime/zarr/append_services.py +650 -0
  168. firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
  169. firecube/ingestor/runtime/zarr/contracts.py +58 -0
  170. firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
  171. firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
  172. firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
  173. firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
  174. firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
  175. firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
  176. firecube/ingestor/runtime/zarr/write.py +212 -0
  177. firecube/ingestor/runtime/zarr/write_context.py +139 -0
  178. firecube/ingestor/templates/__init__.py +19 -0
  179. firecube/ingestor/templates/config.py +118 -0
  180. firecube/ingestor/templates/direct_zarr.py +790 -0
  181. firecube/ingestor/templates/generic.py +462 -0
  182. firecube/ingestor/templates/generic_tensogram.py +170 -0
  183. firecube/ingestor/types/__init__.py +19 -0
  184. firecube/ingestor/types/config.py +89 -0
  185. firecube/ingestor/types/context.py +409 -0
  186. firecube/ingestor/types/manifest.py +53 -0
  187. firecube/ingestor/types/planned_range.py +152 -0
  188. firecube/ingestor/types/result_metrics.py +386 -0
  189. firecube/ingestor/utils/__init__.py +14 -0
  190. firecube/ingestor/utils/duckdb_utils.py +235 -0
  191. firecube/ingestor/validation.py +55 -0
  192. firecube-0.1.4.dist-info/METADATA +242 -0
  193. firecube-0.1.4.dist-info/RECORD +197 -0
  194. firecube-0.1.4.dist-info/WHEEL +4 -0
  195. firecube-0.1.4.dist-info/entry_points.txt +2 -0
  196. firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
  197. firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,870 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Archive CLI commands for Tensogram (.tgm) archival.
16
+
17
+ Archive subcommands distinguish two URI categories:
18
+
19
+ * **Source/target Zarr products** are resolved via
20
+ :class:`firecube.core.product.resolver.ProductResolver` and feed a
21
+ :class:`firecube.core.storage.session.StorageSession` so that driver,
22
+ endpoint, and credential resolution happen at the CLI boundary.
23
+ * **`.tgm` artifact files** are *not* products. They are external archive
24
+ files and use raw :class:`firecube.core.storage.uri.StorageUri` only —
25
+ product resolvers MUST NOT be used for them. ``info``/``validate``/``list``
26
+ operate on artifacts only and never construct a session.
27
+ """
28
+
29
+ # pyright: reportMissingImports=false, reportCallIssue=false
30
+
31
+ from __future__ import annotations
32
+
33
+ import os
34
+ import sys
35
+ import tempfile
36
+ import uuid
37
+ from pathlib import Path
38
+ from typing import Any, cast
39
+
40
+ import click
41
+
42
+ from firecube.cli._ctx import get_config, get_storage_config
43
+ from firecube.cli._product import resolve_product_identity
44
+ from firecube.cli._shared_options import (
45
+ archive_uri_option,
46
+ dry_run_flag,
47
+ target_uri_option,
48
+ yes_flag,
49
+ )
50
+ from firecube.cli._typed_options import display_format_option
51
+ from firecube.cli._uri_policy import (
52
+ apply_smart_default,
53
+ parse_product_uri,
54
+ )
55
+ from firecube.core import observability
56
+ from firecube.core.config import get_archive_defaults
57
+ from firecube.core.controlplane.types import MAINTENANCE_OP_ARCHIVE_RESTORE, WriteDomain
58
+ from firecube.core.errors import ClaimConflictError
59
+ from firecube.core.storage.binding import StorageBinding
60
+ from firecube.core.storage.driver_config import StorageDriverConfig
61
+ from firecube.core.storage.session import StorageSession
62
+ from firecube.core.storage.transfer import copy_file as transfer_copy_file
63
+ from firecube.core.storage.transfer import session_for_uri
64
+ from firecube.core.storage.uri import StorageUri
65
+ from firecube.core.uris import is_remote_target, local_path_from_target
66
+ from firecube.ingestor.runtime.recording import SpanRecorder
67
+ from firecube.ingestor.types.context import (
68
+ IngestResult,
69
+ RuntimeFlags,
70
+ RuntimeIdentity,
71
+ RuntimeIngestContext,
72
+ )
73
+ from firecube.ingestor.types.result_metrics import OutputPaths
74
+
75
+
76
+ def _require_overwrite_confirmation(*, overwrite: bool, yes_i_really_mean_it: bool) -> None:
77
+ """Enforce --yes-i-really-mean-it (or interactive confirm) for --overwrite."""
78
+ if not overwrite or yes_i_really_mean_it:
79
+ return
80
+ if not sys.stdin.isatty():
81
+ raise click.UsageError(
82
+ "OverwriteWithoutConfirmation: --overwrite requires --yes-i-really-mean-it in non-TTY."
83
+ )
84
+ click.confirm("Overwrite existing data?", abort=True)
85
+
86
+
87
+ def _temporary_tgm_path() -> str:
88
+ fd, path = tempfile.mkstemp(suffix=".tgm")
89
+ os.close(fd)
90
+ os.unlink(path)
91
+ return path
92
+
93
+
94
+ def _maintenance_claim_message(*, product: str, operation: str, detail: str) -> str:
95
+ return (
96
+ f"Cannot run {operation} for product {product}: {detail}. "
97
+ "If a prior writer is stuck, resolve it with `firecube chunks runs abandon`."
98
+ )
99
+
100
+
101
+ def _acquire_maintenance_claim(
102
+ *, manager, product: str, operation: str, owner_id: str
103
+ ) -> WriteDomain:
104
+ active_claims = manager.list_claims(product=product)
105
+ if active_claims:
106
+ details = ", ".join(f"{claim.domain} (owner={claim.owner_id})" for claim in active_claims)
107
+ raise click.ClickException(
108
+ _maintenance_claim_message(
109
+ product=product,
110
+ operation=operation,
111
+ detail=f"active write claim(s) exist: {details}",
112
+ )
113
+ )
114
+
115
+ domain = WriteDomain(product=product, category="maintenance", name=operation)
116
+ try:
117
+ manager.acquire_claim(product=product, domain=domain, owner_id=owner_id)
118
+ except ClaimConflictError as exc:
119
+ raise click.ClickException(
120
+ _maintenance_claim_message(
121
+ product=product,
122
+ operation=operation,
123
+ detail=str(exc) or "write claim acquisition failed",
124
+ )
125
+ ) from exc
126
+ return domain
127
+
128
+
129
+ @click.group(context_settings={"help_option_names": ["-h", "--help"]})
130
+ @click.pass_context
131
+ def archive(ctx: click.Context) -> None:
132
+ """create and manage Tensogram archives
133
+
134
+ Archives compress all array groups from a Zarr store into a single portable
135
+ file using configurable codecs. Use archive create to produce an archive
136
+ and archive restore to extract it back to Zarr.
137
+ """
138
+ observability.init_observability("firecube-archive")
139
+ ctx.ensure_object(dict)
140
+
141
+
142
+ @archive.command(
143
+ "create",
144
+ epilog="""\b
145
+ Examples:
146
+ # archive a local Zarr store with default compression
147
+ firecube archive create \\
148
+ --source file:///path/to/product.zarr \\
149
+ --archive file:///tmp/archive.tgm
150
+ # archive a specific time range
151
+ firecube archive create \\
152
+ --source file:///path/to/product.zarr \\
153
+ --archive file:///tmp/archive.tgm \\
154
+ --start-date 2024-01-01 \\
155
+ --end-date 2024-02-01
156
+ # archive a single group with explicit codec
157
+ firecube archive create \\
158
+ --source file:///path/to/product.zarr \\
159
+ --archive file:///tmp/archive.tgm \\
160
+ --group <group> \\
161
+ --compression zstd
162
+ See also: firecube archive restore, firecube archive info, firecube archive validate
163
+ """,
164
+ )
165
+ @click.option(
166
+ "-s", "--source", required=True, help="Zarr store URI (file:///abs/path or s3://bucket/key)"
167
+ )
168
+ @archive_uri_option(tier="write")
169
+ @click.option(
170
+ "--storage-type",
171
+ "storage_type",
172
+ required=False,
173
+ type=click.Choice(["local", "s3"], case_sensitive=False),
174
+ )
175
+ @click.option(
176
+ "--storage-driver",
177
+ "storage_driver",
178
+ required=False,
179
+ type=click.Choice(["fsspec", "obstore"], case_sensitive=False),
180
+ )
181
+ @click.option("--start-date", default=None, help="ISO 8601 start date for time range filter")
182
+ @click.option("--end-date", default=None, help="ISO 8601 end date for time range filter")
183
+ @click.option("-g", "--group", default=None, help="product group to archive (e.g. F024, NORDLIS)")
184
+ @click.option("--variables", default=None, help="comma-separated variable names to include")
185
+ @click.option(
186
+ "--compression",
187
+ default=None,
188
+ type=click.Choice(["blosc2", "szip", "zstd", "lz4", "zfp", "sz3"]),
189
+ help="compression codec for .tgm encoding (default: zstd)",
190
+ )
191
+ @click.option("--overwrite", is_flag=True, default=False, help="overwrite existing target file")
192
+ @click.option(
193
+ "--allow-nan/--no-allow-nan",
194
+ default=None,
195
+ help="allow NaN values in archive (default: true)",
196
+ )
197
+ @click.option(
198
+ "--allow-inf/--no-allow-inf",
199
+ default=None,
200
+ help="allow Inf values in archive (default: true)",
201
+ )
202
+ @dry_run_flag
203
+ @yes_flag
204
+ @click.pass_context
205
+ def create(
206
+ ctx: click.Context,
207
+ source: str,
208
+ archive: str,
209
+ storage_type: str | None,
210
+ storage_driver: str | None,
211
+ start_date: str | None,
212
+ end_date: str | None,
213
+ group: str | None,
214
+ variables: str | None,
215
+ compression: str | None,
216
+ overwrite: bool,
217
+ allow_nan: bool | None,
218
+ allow_inf: bool | None,
219
+ dry_run: bool,
220
+ yes_i_really_mean_it: bool,
221
+ ) -> None:
222
+ """convert a Zarr store to a .tgm archive
223
+
224
+ reads all selected array groups, encodes them with the chosen codec
225
+ (default: zstd), and writes a single portable .tgm file. use --start-date
226
+ and --end-date to archive a specific time window, or --group to select one
227
+ product group.
228
+ """
229
+ from firecube.core.tensogram.converter import zarr_to_tgm
230
+
231
+ parsed_source = parse_product_uri(source)
232
+ storage_type = apply_smart_default(parsed_source, storage_type)
233
+ if is_remote_target(archive):
234
+ raise click.ClickException("Remote .tgm artifacts not yet supported")
235
+ archive_abs = str(local_path_from_target(archive))
236
+
237
+ storage_config = get_storage_config(
238
+ ctx,
239
+ overrides={"storage_type": storage_type, "storage_driver": storage_driver},
240
+ cache=False,
241
+ )
242
+
243
+ cfg = get_config(ctx)
244
+ archive_defaults = get_archive_defaults(cfg)
245
+
246
+ if compression is None:
247
+ compression = str(archive_defaults.get("compression", "zstd"))
248
+ if not overwrite:
249
+ overwrite = bool(archive_defaults.get("overwrite", False))
250
+ if allow_nan is None:
251
+ allow_nan = bool(archive_defaults.get("allow_nan", True))
252
+ if allow_inf is None:
253
+ allow_inf = bool(archive_defaults.get("allow_inf", True))
254
+
255
+ _require_overwrite_confirmation(overwrite=overwrite, yes_i_really_mean_it=yes_i_really_mean_it)
256
+
257
+ if dry_run:
258
+ click.echo(f"[dry-run] Would archive: {source} -> {archive_abs}")
259
+ if group:
260
+ click.echo(f" Group: {group}")
261
+ if start_date or end_date:
262
+ click.echo(f" Time range: {start_date or '*'} -> {end_date or '*'}")
263
+ click.echo(f" Compression: {compression or 'zstd'}")
264
+ return
265
+
266
+ variable_list = [v.strip() for v in variables.split(",")] if variables else None
267
+
268
+ driver_config = StorageDriverConfig.from_storage_config(storage_config)
269
+ source_identity = resolve_product_identity(
270
+ source,
271
+ format="zarr",
272
+ product_name=source.rstrip("/").rsplit("/", 1)[-1],
273
+ option_name="--source",
274
+ )
275
+ session = StorageSession(
276
+ StorageBinding(
277
+ identity=source_identity,
278
+ driver=driver_config,
279
+ )
280
+ )
281
+
282
+ archive_uri = StorageUri.from_local_path(archive_abs)
283
+ create_target = archive_abs
284
+ temp_archive_path: str | None = None
285
+
286
+ if archive_uri.is_remote():
287
+ if session.exists(archive_uri) and not overwrite:
288
+ raise click.ClickException(
289
+ f"Archive file already exists: {archive}. Use overwrite=True to replace it."
290
+ )
291
+ temp_archive_path = _temporary_tgm_path()
292
+ create_target = temp_archive_path
293
+
294
+ try:
295
+ result = zarr_to_tgm(
296
+ source_identity.product_uri.to_str(),
297
+ create_target,
298
+ group=group,
299
+ variables=variable_list,
300
+ start_date=start_date,
301
+ end_date=end_date,
302
+ compression=compression,
303
+ overwrite=overwrite,
304
+ session=session,
305
+ allow_nan=allow_nan,
306
+ allow_inf=allow_inf,
307
+ )
308
+ if temp_archive_path is not None:
309
+ if session.exists(archive_uri) and overwrite:
310
+ session.delete(archive_uri)
311
+ # Cross-endpoint: product session is bound to source zarr, not archive_uri.
312
+ target_session = (
313
+ session_for_uri(archive_uri, session.driver) if archive_uri.is_remote() else None
314
+ )
315
+ transfer_copy_file(
316
+ StorageUri.from_local_path(temp_archive_path),
317
+ archive_uri,
318
+ target_session=target_session,
319
+ )
320
+ result["target"] = archive_uri.to_str()
321
+ except (FileExistsError, ValueError, ImportError) as exc:
322
+ raise click.ClickException(str(exc)) from exc
323
+ finally:
324
+ if temp_archive_path is not None and os.path.exists(temp_archive_path):
325
+ os.unlink(temp_archive_path)
326
+
327
+ click.echo(f"Archive created: {result['target']}")
328
+ if result.get("groups"):
329
+ click.echo(f" Groups: {', '.join(result['groups'])}")
330
+ click.echo(f" Variables: {', '.join(result['variables'])}")
331
+ if result.get("time_range"):
332
+ tr = result["time_range"]
333
+ click.echo(f" Time range: {tr.get('start')} → {tr.get('end')} ({tr.get('n')} steps)")
334
+ elif any(result.get("time_ranges", {}).values()):
335
+ click.echo(f" Time ranges: {len(result['time_ranges'])} groups")
336
+ size_mb = result["file_size_bytes"] / (1024 * 1024)
337
+ click.echo(f" Size: {size_mb:.2f} MB | Codec: {result['compression']}")
338
+ if result.get("skipped"):
339
+ click.echo(f" Skipped: {', '.join(result['skipped'])} (unsupported dtype)", err=True)
340
+
341
+
342
+ @archive.command(
343
+ "restore",
344
+ epilog="""\b
345
+ Examples:
346
+ # restore an archive to a local Zarr store
347
+ firecube archive restore \\
348
+ --archive file:///tmp/archive.tgm \\
349
+ --target file:///path/to/restored.zarr
350
+ # restore and overwrite an existing store
351
+ firecube archive restore \\
352
+ --archive file:///tmp/archive.tgm \\
353
+ --target file:///path/to/product.zarr \\
354
+ --overwrite
355
+ See also: firecube archive create, firecube archive info
356
+ """,
357
+ )
358
+ @archive_uri_option(tier="write")
359
+ @target_uri_option(tier="write")
360
+ @click.option(
361
+ "--storage-type",
362
+ "storage_type",
363
+ required=False,
364
+ type=click.Choice(["local", "s3"], case_sensitive=False),
365
+ )
366
+ @click.option(
367
+ "--storage-driver",
368
+ "storage_driver",
369
+ required=False,
370
+ type=click.Choice(["fsspec", "obstore"], case_sensitive=False),
371
+ )
372
+ @click.option("--overwrite", is_flag=True, default=False, help="overwrite existing Zarr store")
373
+ @dry_run_flag
374
+ @yes_flag
375
+ @click.pass_context
376
+ def restore(
377
+ ctx: click.Context,
378
+ archive: str,
379
+ target: str,
380
+ storage_type: str | None,
381
+ storage_driver: str | None,
382
+ overwrite: bool,
383
+ dry_run: bool,
384
+ yes_i_really_mean_it: bool,
385
+ ) -> None:
386
+ """restore a .tgm archive to a Zarr store
387
+
388
+ decodes all encoded array groups from the .tgm file and writes them back
389
+ to the target Zarr store. archive groups restore to their original paths.
390
+ """
391
+ parsed_target = parse_product_uri(target)
392
+ storage_type = apply_smart_default(parsed_target, storage_type)
393
+ if is_remote_target(archive):
394
+ raise click.ClickException("Remote .tgm artifacts not yet supported")
395
+ archive_abs = str(local_path_from_target(archive))
396
+
397
+ _require_overwrite_confirmation(overwrite=overwrite, yes_i_really_mean_it=yes_i_really_mean_it)
398
+
399
+ if dry_run:
400
+ click.echo(f"[dry-run] Would restore: {archive_abs} -> {target}")
401
+ if overwrite:
402
+ click.echo(" Overwrite: yes")
403
+ return
404
+
405
+ from firecube.core.tensogram.restore import tgm_to_zarr
406
+
407
+ storage_config = get_storage_config(
408
+ ctx,
409
+ overrides={"storage_type": storage_type, "storage_driver": storage_driver},
410
+ cache=False,
411
+ )
412
+
413
+ driver_config = StorageDriverConfig.from_storage_config(storage_config)
414
+ restore_identity = resolve_product_identity(
415
+ target,
416
+ format="zarr",
417
+ product_name=target.rstrip("/").rsplit("/", 1)[-1],
418
+ option_name="--target",
419
+ )
420
+ session = StorageSession(
421
+ StorageBinding(
422
+ identity=restore_identity,
423
+ driver=driver_config,
424
+ )
425
+ )
426
+
427
+ # W3.4: Archive restore is recorded as a first-class WAL run so the audit
428
+ # trail (`firecube chunks runs list`) shows when a product was restored
429
+ # from an archive, what the source archive was, and whether the restore
430
+ # succeeded or failed. We construct the SpanRecorder lazily per phase
431
+ # because tgm_to_zarr's `_prepare_target` may rmtree the entire target
432
+ # directory (including any `.firecube/` we created), which would
433
+ # invalidate a cached WAL writer.
434
+ restore_run_id = f"archive-restore-{uuid.uuid4()}"
435
+ product_name = restore_identity.product_name
436
+ target_uri_str = restore_identity.product_uri.to_str()
437
+ slice_meta: dict[str, Any] = {
438
+ "run_kind": "archive_restore",
439
+ "source_archive": archive,
440
+ "target_product": target_uri_str,
441
+ }
442
+
443
+ def _record_failure(error: str) -> None:
444
+ manager = session.control_plane()
445
+ try:
446
+ recorder = SpanRecorder(manager)
447
+ recorder.register_run_started(
448
+ run_id=restore_run_id,
449
+ product=product_name,
450
+ output_path=target_uri_str,
451
+ output_format="zarr",
452
+ slice_meta=slice_meta,
453
+ )
454
+ manager.record_maintenance_started(
455
+ product=product_name,
456
+ run_id=restore_run_id,
457
+ op=MAINTENANCE_OP_ARCHIVE_RESTORE,
458
+ scope_meta=slice_meta,
459
+ )
460
+ recorder.register_run_failure(
461
+ run_id=restore_run_id,
462
+ product=product_name,
463
+ output_path=target_uri_str,
464
+ output_format="zarr",
465
+ slice_meta=slice_meta,
466
+ error=error,
467
+ )
468
+ manager.record_maintenance_failed(
469
+ product=product_name,
470
+ run_id=restore_run_id,
471
+ op=MAINTENANCE_OP_ARCHIVE_RESTORE,
472
+ scope_meta=slice_meta,
473
+ error=error,
474
+ )
475
+ finally:
476
+ manager.close()
477
+
478
+ archive_uri = StorageUri.from_local_path(archive_abs)
479
+ restore_source = archive_abs
480
+ temp_archive_path: str | None = None
481
+
482
+ if archive_uri.is_remote():
483
+ temp_archive_path = _temporary_tgm_path()
484
+ try:
485
+ # Cross-endpoint: product session is bound to restore target, not archive_uri.
486
+ source_archive_session = session_for_uri(archive_uri, session.driver)
487
+ transfer_copy_file(
488
+ archive_uri,
489
+ StorageUri.from_local_path(temp_archive_path),
490
+ source_session=source_archive_session,
491
+ )
492
+ except Exception as exc:
493
+ _record_failure(str(exc))
494
+ raise
495
+ restore_source = temp_archive_path
496
+
497
+ claim_manager = session.control_plane()
498
+ claim_domain: WriteDomain | None = None
499
+ try:
500
+ claim_domain = _acquire_maintenance_claim(
501
+ manager=claim_manager,
502
+ product=product_name,
503
+ operation="archive_restore",
504
+ owner_id=restore_run_id,
505
+ )
506
+ try:
507
+ result = tgm_to_zarr(
508
+ restore_source,
509
+ target_uri_str,
510
+ overwrite=overwrite,
511
+ session=session,
512
+ on_group_restored=lambda restored_group: click.echo(
513
+ f"restored group: {restored_group or '/'}"
514
+ ),
515
+ )
516
+ if not result.get("variables") and not result.get("groups"):
517
+ raise ValueError(
518
+ f"Archive restore produced no output: {restore_source!r} "
519
+ "may be empty or not a valid tensogram archive."
520
+ )
521
+ finally:
522
+ if temp_archive_path is not None and os.path.exists(temp_archive_path):
523
+ os.unlink(temp_archive_path)
524
+ except (FileExistsError, ImportError, ValueError) as exc:
525
+ _record_failure(str(exc))
526
+ raise click.ClickException(str(exc)) from exc
527
+ except Exception as exc:
528
+ _record_failure(str(exc))
529
+ raise
530
+ else:
531
+ success_manager = session.control_plane()
532
+ try:
533
+ success_recorder = SpanRecorder(success_manager)
534
+ success_recorder.register_run_started(
535
+ run_id=restore_run_id,
536
+ product=product_name,
537
+ output_path=target_uri_str,
538
+ output_format="zarr",
539
+ slice_meta=slice_meta,
540
+ )
541
+ success_manager.record_maintenance_started(
542
+ product=product_name,
543
+ run_id=restore_run_id,
544
+ op=MAINTENANCE_OP_ARCHIVE_RESTORE,
545
+ scope_meta=slice_meta,
546
+ )
547
+ synthetic_ctx = RuntimeIngestContext(
548
+ source=archive,
549
+ target=target_uri_str,
550
+ output_format="zarr",
551
+ run_id=restore_run_id,
552
+ identity=RuntimeIdentity(run_id=restore_run_id),
553
+ flags=RuntimeFlags(force_reingest=False),
554
+ )
555
+ synthetic_result = IngestResult(
556
+ outputs=OutputPaths(primary=target_uri_str),
557
+ output_format="zarr",
558
+ metrics={},
559
+ )
560
+ success_recorder.register_run(
561
+ ctx=synthetic_ctx,
562
+ result=synthetic_result,
563
+ run_id=restore_run_id,
564
+ product=product_name,
565
+ slice_meta=slice_meta,
566
+ )
567
+ success_manager.record_maintenance_completed(
568
+ product=product_name,
569
+ run_id=restore_run_id,
570
+ op=MAINTENANCE_OP_ARCHIVE_RESTORE,
571
+ scope_meta=slice_meta,
572
+ )
573
+ finally:
574
+ success_manager.close()
575
+ finally:
576
+ try:
577
+ if claim_domain is not None:
578
+ claim_manager.clear_claim(
579
+ product=product_name,
580
+ domain_id=claim_domain.identifier,
581
+ force=True,
582
+ )
583
+ finally:
584
+ claim_manager.close()
585
+
586
+ click.echo(f"Zarr restored: {result['target']}")
587
+ if result.get("groups"):
588
+ click.echo(f" Groups: {', '.join(result['groups'])}")
589
+ click.echo(f" Variables: {', '.join(result['variables'])}")
590
+ click.echo(f" Coordinates: {', '.join(result['coordinates'])}")
591
+ click.echo(
592
+ " Control-plane: restored"
593
+ if result.get("controlplane_restored")
594
+ else " Control-plane: not present"
595
+ )
596
+
597
+
598
+ @archive.command(
599
+ "info",
600
+ epilog="""\b
601
+ Examples:
602
+ # show metadata for an archive
603
+ firecube archive info --archive file:///tmp/archive.tgm
604
+
605
+ \b
606
+ # output metadata as JSON
607
+ firecube archive info --archive file:///tmp/archive.tgm -f json
608
+
609
+ See also: firecube archive list, firecube archive validate
610
+ """,
611
+ )
612
+ @archive_uri_option(tier="inspect")
613
+ @display_format_option(default="table")
614
+ @click.pass_context
615
+ def info(ctx: click.Context, archive: str, output_format: str) -> None:
616
+ """show metadata for a .tgm archive
617
+
618
+ reads the archive header to display variable names, dimension sizes, codec,
619
+ and time coverage. no data is decoded or written; suitable for quick
620
+ inspection.
621
+ """
622
+ from firecube.core.tensogram._compat import require_tensogram
623
+ from firecube.core.tensogram.schema import (
624
+ ARCHIVE_VERSION,
625
+ KEY_ARCHIVE_VERSION,
626
+ ROLE_CONTROLPLANE,
627
+ )
628
+
629
+ parsed = parse_product_uri(archive)
630
+ if parsed.scheme == "s3":
631
+ raise click.ClickException("Remote .tgm artifacts not yet supported")
632
+ local_path = parsed.normalized.removeprefix("file://")
633
+ if not Path(local_path).exists():
634
+ raise click.ClickException(f"Archive not found: {archive}")
635
+
636
+ require_tensogram("firecube archive info")
637
+
638
+ import json as _json
639
+
640
+ import tensogram as _tensogram
641
+
642
+ tensogram = cast(Any, _tensogram)
643
+ as_json = output_format == "json"
644
+
645
+ with tensogram.TensogramFile.open(local_path) as f:
646
+ count = f.message_count()
647
+ if count == 0:
648
+ raise click.ClickException("Archive contains no messages.")
649
+
650
+ first_meta = f.file_decode_metadata(0)
651
+ first_extra = first_meta.extra or {}
652
+ firecube_meta = first_extra.get("firecube", {})
653
+ archive_version = str(firecube_meta.get(KEY_ARCHIVE_VERSION, ""))
654
+ if archive_version != ARCHIVE_VERSION:
655
+ raise click.ClickException(
656
+ f"Unsupported Firecube archive version: {archive_version or 'missing'}. "
657
+ f"Expected {ARCHIVE_VERSION}."
658
+ )
659
+
660
+ groups_info: list[dict[str, Any]] = []
661
+ has_controlplane = False
662
+
663
+ for i in range(count):
664
+ meta = f.file_decode_metadata(i)
665
+ extra = meta.extra or {}
666
+ fc = extra.get("firecube", {})
667
+ role = fc.get("role", "data")
668
+
669
+ if role == ROLE_CONTROLPLANE:
670
+ has_controlplane = True
671
+ continue
672
+
673
+ group_name = fc.get("group", "")
674
+ base_entries = list(meta.base) if meta.base else []
675
+ result = f.file_decode_descriptors(i)
676
+ descriptors = result.get("descriptors", [])
677
+ archived_coords = set(fc.get("coordinates", []))
678
+
679
+ group_vars: dict[str, dict] = {}
680
+ for j, desc in enumerate(descriptors):
681
+ base_entry = base_entries[j] if j < len(base_entries) else {}
682
+ obj_name = str(base_entry.get("name") or f"object_{j}")
683
+ if obj_name not in archived_coords:
684
+ group_vars[obj_name] = {
685
+ "shape": list(desc.shape) if desc.shape else [],
686
+ "dtype": str(desc.dtype),
687
+ "dims": base_entry.get("dim_names", []),
688
+ "zarr_chunks": base_entry.get("zarr_chunks"),
689
+ }
690
+
691
+ groups_info.append(
692
+ {
693
+ "group": group_name,
694
+ "variables": group_vars,
695
+ "compression": fc.get("compression", ""),
696
+ "source_uri": fc.get("source_uri", ""),
697
+ "archived_at": fc.get("archived_at", ""),
698
+ }
699
+ )
700
+
701
+ info_dict: dict[str, Any] = {
702
+ "path": local_path,
703
+ "format": archive_version,
704
+ "size_bytes": os.path.getsize(local_path),
705
+ "groups": [g["group"] for g in groups_info],
706
+ "has_controlplane": has_controlplane,
707
+ "group_details": groups_info,
708
+ }
709
+
710
+ if as_json:
711
+ click.echo(_json.dumps(info_dict, indent=2))
712
+ else:
713
+ click.echo(f"Path: {local_path}")
714
+ click.echo(f"Format: {archive_version} (multi-group)")
715
+ size_mb = os.path.getsize(local_path) / (1024 * 1024)
716
+ click.echo(f"Size: {size_mb:.2f} MB")
717
+ click.echo(
718
+ "Groups: " + (", ".join(g["group"] for g in groups_info) or "(none)")
719
+ )
720
+ click.echo("Control-plane: " + ("present" if has_controlplane else "not present"))
721
+ for g in groups_info:
722
+ click.echo(f"\nGroup: {g['group'] or '(root)'}")
723
+ click.echo(f" Source: {g['source_uri']}")
724
+ click.echo(f" Archived: {g['archived_at']}")
725
+ click.echo(f" Codec: {g['compression']}")
726
+ click.echo(" Variables:")
727
+ for vname, vmeta in g["variables"].items():
728
+ chunks_str = (
729
+ f" chunks={vmeta['zarr_chunks']}" if vmeta.get("zarr_chunks") else ""
730
+ )
731
+ click.echo(
732
+ f" {vname}: shape={vmeta['shape']} dtype={vmeta['dtype']}{chunks_str}"
733
+ )
734
+
735
+
736
+ @archive.command(
737
+ "validate",
738
+ epilog="""\b
739
+ Examples:
740
+ # validate an archive and check exit code
741
+ firecube archive validate --archive file:///tmp/archive.tgm && echo "ok"
742
+
743
+ \b
744
+ # quick structure-only validation
745
+ firecube archive validate --archive file:///tmp/archive.tgm --quick
746
+
747
+ See also: firecube archive info, firecube archive create
748
+ """,
749
+ )
750
+ @archive_uri_option(tier="inspect")
751
+ @click.option(
752
+ "--quick", is_flag=True, default=False, help="structure-only check (no hash verification)"
753
+ )
754
+ @click.pass_context
755
+ def validate(ctx: click.Context, archive: str, quick: bool) -> None:
756
+ """check integrity of a .tgm archive
757
+
758
+ reads the archive header and validates internal structure without fully
759
+ decoding data. exits with code 0 if the archive is valid, code 1 if it is
760
+ corrupted or incomplete.
761
+ """
762
+ from firecube.core.tensogram._compat import require_tensogram
763
+
764
+ parsed = parse_product_uri(archive)
765
+ if parsed.scheme == "s3":
766
+ raise click.ClickException("Remote .tgm artifacts not yet supported")
767
+ local_path = parsed.normalized.removeprefix("file://")
768
+ if not Path(local_path).exists():
769
+ raise click.ClickException(f"Archive not found: {archive}")
770
+
771
+ require_tensogram("firecube archive validate")
772
+
773
+ import tensogram as _tensogram
774
+
775
+ tensogram = cast(Any, _tensogram)
776
+
777
+ level = "quick" if quick else "default"
778
+ report = tensogram.validate_file(local_path, level=level)
779
+
780
+ file_issues = report.get("file_issues", [])
781
+ messages = report.get("messages", [])
782
+
783
+ hard_issues: list[tuple[str, dict[str, Any]]] = [("file", i) for i in file_issues]
784
+ hard_issues.extend(
785
+ (f"msg {i}", issue)
786
+ for i, msg_report in enumerate(messages)
787
+ for issue in msg_report.get("issues", [])
788
+ )
789
+
790
+ if not hard_issues:
791
+ click.echo(f"VALID: {local_path} ({len(messages)} message(s), no issues)")
792
+ else:
793
+ click.echo(f"INVALID: {local_path} ({len(hard_issues)} issue(s))")
794
+ for scope, issue in hard_issues:
795
+ if scope == "file":
796
+ click.echo(
797
+ f" [file] offset={issue.get('byte_offset')}: {issue.get('description')}"
798
+ )
799
+ else:
800
+ click.echo(f" [{scope}] {issue}")
801
+ raise SystemExit(1)
802
+
803
+
804
+ @archive.command(
805
+ "list",
806
+ epilog="""\b
807
+ Examples:
808
+ # list archive contents
809
+ firecube archive list --archive file:///tmp/archive.tgm
810
+
811
+ See also: firecube archive info, firecube archive restore
812
+ """,
813
+ )
814
+ @archive_uri_option(tier="inspect")
815
+ @click.pass_context
816
+ def list_cmd(ctx: click.Context, archive: str) -> None:
817
+ """list contents of a .tgm archive
818
+
819
+ displays per-variable group entries including name, dimensions, and time
820
+ range. no data is decoded; useful for reviewing archive contents before
821
+ restoring.
822
+ """
823
+ from firecube.core.tensogram._compat import require_tensogram
824
+ from firecube.core.tensogram.schema import ROLE_CONTROLPLANE
825
+
826
+ parsed = parse_product_uri(archive)
827
+ if parsed.scheme == "s3":
828
+ raise click.ClickException("Remote .tgm artifacts not yet supported")
829
+ local_path = parsed.normalized.removeprefix("file://")
830
+ if not Path(local_path).exists():
831
+ raise click.ClickException(f"Archive not found: {archive}")
832
+
833
+ require_tensogram("firecube archive list")
834
+
835
+ import tensogram as _tensogram
836
+
837
+ tensogram = cast(Any, _tensogram)
838
+
839
+ with tensogram.TensogramFile.open(local_path) as f:
840
+ count = f.message_count()
841
+ click.echo(f"Messages: {count}")
842
+ for i in range(count):
843
+ meta = f.file_decode_metadata(i)
844
+ extra = meta.extra or {}
845
+ fc = extra.get("firecube", {})
846
+ role = fc.get("role", "data")
847
+ base_entries = list(meta.base) if meta.base else []
848
+
849
+ if role == ROLE_CONTROLPLANE:
850
+ click.echo("\n[control-plane]")
851
+ click.echo(f" Product: {fc.get('product', '')}")
852
+ continue
853
+
854
+ group_name = fc.get("group") or f"message {i}"
855
+ click.echo(f"\nGroup: {group_name}")
856
+ if fc:
857
+ click.echo(f" Source: {fc.get('source_uri', '')}")
858
+ click.echo(f" Codec: {fc.get('compression', '')}")
859
+
860
+ result = f.file_decode_descriptors(i)
861
+ descriptors = result.get("descriptors", [])
862
+ click.echo(f" Objects: {len(descriptors)}")
863
+ for j, desc in enumerate(descriptors):
864
+ base_entry = base_entries[j] if j < len(base_entries) else {}
865
+ shape = list(desc.shape) if desc.shape else []
866
+ click.echo(
867
+ f" [{j}] {base_entry.get('name') or desc.params.get('name', desc.obj_type)}:"
868
+ f" shape={shape} dtype={desc.dtype}"
869
+ f" compression={desc.compression}"
870
+ )