firecube 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- firecube/__init__.py +23 -0
- firecube/__main__.py +20 -0
- firecube/_version.py +46 -0
- firecube/cli/_audience.py +135 -0
- firecube/cli/_command_schemas.py +112 -0
- firecube/cli/_ctx.py +85 -0
- firecube/cli/_errors.py +115 -0
- firecube/cli/_formatter.py +195 -0
- firecube/cli/_product.py +47 -0
- firecube/cli/_rename_hints.py +123 -0
- firecube/cli/_shared_options.py +247 -0
- firecube/cli/_slot_env.py +102 -0
- firecube/cli/_slot_planning.py +95 -0
- firecube/cli/_typed_options.py +181 -0
- firecube/cli/_uri_policy.py +110 -0
- firecube/cli/advise.py +321 -0
- firecube/cli/archive.py +870 -0
- firecube/cli/catalog.py +196 -0
- firecube/cli/chunks/__init__.py +74 -0
- firecube/cli/chunks/_claims.py +229 -0
- firecube/cli/chunks/_common.py +84 -0
- firecube/cli/chunks/_delete.py +376 -0
- firecube/cli/chunks/_list.py +334 -0
- firecube/cli/chunks/_manager.py +161 -0
- firecube/cli/chunks/_runs.py +294 -0
- firecube/cli/chunks/_snapshots.py +165 -0
- firecube/cli/completion.py +73 -0
- firecube/cli/main.py +866 -0
- firecube/cli/parquet.py +289 -0
- firecube/cli/plugins/__init__.py +28 -0
- firecube/cli/plugins/commands.py +262 -0
- firecube/cli/plugins/introspect.py +304 -0
- firecube/cli/plugins/mgmt.py +198 -0
- firecube/cli/plugins/registry.py +132 -0
- firecube/cli/zarr.py +1043 -0
- firecube/core/__init__.py +14 -0
- firecube/core/api.py +88 -0
- firecube/core/cf/__init__.py +23 -0
- firecube/core/cf/check_ids.py +126 -0
- firecube/core/cf/report.py +57 -0
- firecube/core/cf/validator.py +313 -0
- firecube/core/config.py +307 -0
- firecube/core/controlplane/__init__.py +47 -0
- firecube/core/controlplane/_event_processor.py +136 -0
- firecube/core/controlplane/_helpers.py +114 -0
- firecube/core/controlplane/_projection.py +253 -0
- firecube/core/controlplane/_snapshot.py +215 -0
- firecube/core/controlplane/_wal_reader.py +402 -0
- firecube/core/controlplane/_wal_writer.py +589 -0
- firecube/core/controlplane/claims.py +294 -0
- firecube/core/controlplane/deletion.py +704 -0
- firecube/core/controlplane/events.py +219 -0
- firecube/core/controlplane/manager.py +990 -0
- firecube/core/controlplane/metrics.py +97 -0
- firecube/core/controlplane/repo.py +804 -0
- firecube/core/controlplane/repo_utils.py +73 -0
- firecube/core/controlplane/time_dim.py +111 -0
- firecube/core/controlplane/types.py +445 -0
- firecube/core/credentials.py +36 -0
- firecube/core/duckdb/__init__.py +27 -0
- firecube/core/duckdb/bridge.py +82 -0
- firecube/core/duckdb/io.py +30 -0
- firecube/core/errors.py +74 -0
- firecube/core/filesystem/__init__.py +60 -0
- firecube/core/filesystem/_obstore_compat.py +60 -0
- firecube/core/filesystem/fsspec_backend.py +308 -0
- firecube/core/filesystem/instrumentation.py +260 -0
- firecube/core/filesystem/obstore_backend.py +406 -0
- firecube/core/filesystem/ops.py +412 -0
- firecube/core/filesystem/protocol.py +130 -0
- firecube/core/filesystem/store_factory.py +149 -0
- firecube/core/formats/__init__.py +46 -0
- firecube/core/formats/_iso.py +84 -0
- firecube/core/formats/discovery.py +165 -0
- firecube/core/formats/hdf5.py +137 -0
- firecube/core/formats/netcdf.py +226 -0
- firecube/core/formats/zip.py +192 -0
- firecube/core/intake.py +530 -0
- firecube/core/observability/__init__.py +72 -0
- firecube/core/observability/_state.py +24 -0
- firecube/core/observability/logging.py +147 -0
- firecube/core/observability/metrics.py +230 -0
- firecube/core/observability/telemetry/__init__.py +87 -0
- firecube/core/observability/telemetry/pushgateway.py +339 -0
- firecube/core/observability/telemetry/sinks.py +155 -0
- firecube/core/observability/tracing.py +126 -0
- firecube/core/product/__init__.py +45 -0
- firecube/core/product/identity.py +76 -0
- firecube/core/product/resolver.py +121 -0
- firecube/core/product/target.py +120 -0
- firecube/core/runtime.py +187 -0
- firecube/core/slot_index.py +205 -0
- firecube/core/storage/__init__.py +45 -0
- firecube/core/storage/binding.py +39 -0
- firecube/core/storage/cache_key.py +40 -0
- firecube/core/storage/completion.py +245 -0
- firecube/core/storage/driver_config.py +125 -0
- firecube/core/storage/results.py +30 -0
- firecube/core/storage/session.py +431 -0
- firecube/core/storage/transfer.py +142 -0
- firecube/core/storage/uri.py +173 -0
- firecube/core/tensogram/__init__.py +37 -0
- firecube/core/tensogram/_compat.py +36 -0
- firecube/core/tensogram/controlplane_codec.py +230 -0
- firecube/core/tensogram/converter.py +390 -0
- firecube/core/tensogram/metadata.py +274 -0
- firecube/core/tensogram/restore.py +303 -0
- firecube/core/tensogram/schema.py +93 -0
- firecube/core/uris.py +182 -0
- firecube/core/workspaces.py +68 -0
- firecube/core/zarr/__init__.py +69 -0
- firecube/core/zarr/_reserved_attrs.py +47 -0
- firecube/core/zarr/_reserved_root_attrs.py +74 -0
- firecube/core/zarr/io.py +88 -0
- firecube/core/zarr/layers.py +184 -0
- firecube/core/zarr/multires.py +178 -0
- firecube/core/zarr/region_writer.py +782 -0
- firecube/core/zarr/scrub.py +200 -0
- firecube/core/zarr/state.py +258 -0
- firecube/core/zarr/time_decode.py +71 -0
- firecube/core/zarr/validation.py +622 -0
- firecube/ingestor/__init__.py +27 -0
- firecube/ingestor/api.py +215 -0
- firecube/ingestor/config/__init__.py +19 -0
- firecube/ingestor/config/coercion.py +157 -0
- firecube/ingestor/config/engine.py +188 -0
- firecube/ingestor/contracts/__init__.py +14 -0
- firecube/ingestor/contracts/interfaces.py +235 -0
- firecube/ingestor/devtools/_templates/__init__.py +14 -0
- firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
- firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
- firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
- firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
- firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
- firecube/ingestor/devtools/scaffolding.py +229 -0
- firecube/ingestor/errors.py +79 -0
- firecube/ingestor/extensions/_binning.py +152 -0
- firecube/ingestor/extensions/duck.py +153 -0
- firecube/ingestor/extensions/grid.py +287 -0
- firecube/ingestor/extensions/healpix.py +330 -0
- firecube/ingestor/registry/__init__.py +14 -0
- firecube/ingestor/registry/loader.py +144 -0
- firecube/ingestor/registry/metadata.py +97 -0
- firecube/ingestor/registry/version_compat.py +95 -0
- firecube/ingestor/runtime/__init__.py +14 -0
- firecube/ingestor/runtime/aggregation.py +112 -0
- firecube/ingestor/runtime/base.py +836 -0
- firecube/ingestor/runtime/base_hooks.py +129 -0
- firecube/ingestor/runtime/batching.py +161 -0
- firecube/ingestor/runtime/configure.py +195 -0
- firecube/ingestor/runtime/coverage.py +177 -0
- firecube/ingestor/runtime/engine.py +793 -0
- firecube/ingestor/runtime/parallel_evidence.py +75 -0
- firecube/ingestor/runtime/parallel_execution_state.py +28 -0
- firecube/ingestor/runtime/parallel_gate.py +164 -0
- firecube/ingestor/runtime/parallel_run_id.py +48 -0
- firecube/ingestor/runtime/preflight.py +40 -0
- firecube/ingestor/runtime/recording.py +425 -0
- firecube/ingestor/runtime/resume_guard.py +556 -0
- firecube/ingestor/runtime/resume_types.py +65 -0
- firecube/ingestor/runtime/telemetry.py +171 -0
- firecube/ingestor/runtime/tensogram/__init__.py +14 -0
- firecube/ingestor/runtime/tensogram/strategy.py +164 -0
- firecube/ingestor/runtime/workspace.py +220 -0
- firecube/ingestor/runtime/zarr/__init__.py +34 -0
- firecube/ingestor/runtime/zarr/append.py +466 -0
- firecube/ingestor/runtime/zarr/append_services.py +650 -0
- firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
- firecube/ingestor/runtime/zarr/contracts.py +58 -0
- firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
- firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
- firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
- firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
- firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
- firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
- firecube/ingestor/runtime/zarr/write.py +212 -0
- firecube/ingestor/runtime/zarr/write_context.py +139 -0
- firecube/ingestor/templates/__init__.py +19 -0
- firecube/ingestor/templates/config.py +118 -0
- firecube/ingestor/templates/direct_zarr.py +790 -0
- firecube/ingestor/templates/generic.py +462 -0
- firecube/ingestor/templates/generic_tensogram.py +170 -0
- firecube/ingestor/types/__init__.py +19 -0
- firecube/ingestor/types/config.py +89 -0
- firecube/ingestor/types/context.py +409 -0
- firecube/ingestor/types/manifest.py +53 -0
- firecube/ingestor/types/planned_range.py +152 -0
- firecube/ingestor/types/result_metrics.py +386 -0
- firecube/ingestor/utils/__init__.py +14 -0
- firecube/ingestor/utils/duckdb_utils.py +235 -0
- firecube/ingestor/validation.py +55 -0
- firecube-0.1.4.dist-info/METADATA +242 -0
- firecube-0.1.4.dist-info/RECORD +197 -0
- firecube-0.1.4.dist-info/WHEEL +4 -0
- firecube-0.1.4.dist-info/entry_points.txt +2 -0
- firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
- firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
firecube/cli/archive.py
ADDED
|
@@ -0,0 +1,870 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Archive CLI commands for Tensogram (.tgm) archival.
|
|
16
|
+
|
|
17
|
+
Archive subcommands distinguish two URI categories:
|
|
18
|
+
|
|
19
|
+
* **Source/target Zarr products** are resolved via
|
|
20
|
+
:class:`firecube.core.product.resolver.ProductResolver` and feed a
|
|
21
|
+
:class:`firecube.core.storage.session.StorageSession` so that driver,
|
|
22
|
+
endpoint, and credential resolution happen at the CLI boundary.
|
|
23
|
+
* **`.tgm` artifact files** are *not* products. They are external archive
|
|
24
|
+
files and use raw :class:`firecube.core.storage.uri.StorageUri` only —
|
|
25
|
+
product resolvers MUST NOT be used for them. ``info``/``validate``/``list``
|
|
26
|
+
operate on artifacts only and never construct a session.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
# pyright: reportMissingImports=false, reportCallIssue=false
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import os
|
|
34
|
+
import sys
|
|
35
|
+
import tempfile
|
|
36
|
+
import uuid
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
from typing import Any, cast
|
|
39
|
+
|
|
40
|
+
import click
|
|
41
|
+
|
|
42
|
+
from firecube.cli._ctx import get_config, get_storage_config
|
|
43
|
+
from firecube.cli._product import resolve_product_identity
|
|
44
|
+
from firecube.cli._shared_options import (
|
|
45
|
+
archive_uri_option,
|
|
46
|
+
dry_run_flag,
|
|
47
|
+
target_uri_option,
|
|
48
|
+
yes_flag,
|
|
49
|
+
)
|
|
50
|
+
from firecube.cli._typed_options import display_format_option
|
|
51
|
+
from firecube.cli._uri_policy import (
|
|
52
|
+
apply_smart_default,
|
|
53
|
+
parse_product_uri,
|
|
54
|
+
)
|
|
55
|
+
from firecube.core import observability
|
|
56
|
+
from firecube.core.config import get_archive_defaults
|
|
57
|
+
from firecube.core.controlplane.types import MAINTENANCE_OP_ARCHIVE_RESTORE, WriteDomain
|
|
58
|
+
from firecube.core.errors import ClaimConflictError
|
|
59
|
+
from firecube.core.storage.binding import StorageBinding
|
|
60
|
+
from firecube.core.storage.driver_config import StorageDriverConfig
|
|
61
|
+
from firecube.core.storage.session import StorageSession
|
|
62
|
+
from firecube.core.storage.transfer import copy_file as transfer_copy_file
|
|
63
|
+
from firecube.core.storage.transfer import session_for_uri
|
|
64
|
+
from firecube.core.storage.uri import StorageUri
|
|
65
|
+
from firecube.core.uris import is_remote_target, local_path_from_target
|
|
66
|
+
from firecube.ingestor.runtime.recording import SpanRecorder
|
|
67
|
+
from firecube.ingestor.types.context import (
|
|
68
|
+
IngestResult,
|
|
69
|
+
RuntimeFlags,
|
|
70
|
+
RuntimeIdentity,
|
|
71
|
+
RuntimeIngestContext,
|
|
72
|
+
)
|
|
73
|
+
from firecube.ingestor.types.result_metrics import OutputPaths
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _require_overwrite_confirmation(*, overwrite: bool, yes_i_really_mean_it: bool) -> None:
|
|
77
|
+
"""Enforce --yes-i-really-mean-it (or interactive confirm) for --overwrite."""
|
|
78
|
+
if not overwrite or yes_i_really_mean_it:
|
|
79
|
+
return
|
|
80
|
+
if not sys.stdin.isatty():
|
|
81
|
+
raise click.UsageError(
|
|
82
|
+
"OverwriteWithoutConfirmation: --overwrite requires --yes-i-really-mean-it in non-TTY."
|
|
83
|
+
)
|
|
84
|
+
click.confirm("Overwrite existing data?", abort=True)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _temporary_tgm_path() -> str:
|
|
88
|
+
fd, path = tempfile.mkstemp(suffix=".tgm")
|
|
89
|
+
os.close(fd)
|
|
90
|
+
os.unlink(path)
|
|
91
|
+
return path
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _maintenance_claim_message(*, product: str, operation: str, detail: str) -> str:
|
|
95
|
+
return (
|
|
96
|
+
f"Cannot run {operation} for product {product}: {detail}. "
|
|
97
|
+
"If a prior writer is stuck, resolve it with `firecube chunks runs abandon`."
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _acquire_maintenance_claim(
|
|
102
|
+
*, manager, product: str, operation: str, owner_id: str
|
|
103
|
+
) -> WriteDomain:
|
|
104
|
+
active_claims = manager.list_claims(product=product)
|
|
105
|
+
if active_claims:
|
|
106
|
+
details = ", ".join(f"{claim.domain} (owner={claim.owner_id})" for claim in active_claims)
|
|
107
|
+
raise click.ClickException(
|
|
108
|
+
_maintenance_claim_message(
|
|
109
|
+
product=product,
|
|
110
|
+
operation=operation,
|
|
111
|
+
detail=f"active write claim(s) exist: {details}",
|
|
112
|
+
)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
domain = WriteDomain(product=product, category="maintenance", name=operation)
|
|
116
|
+
try:
|
|
117
|
+
manager.acquire_claim(product=product, domain=domain, owner_id=owner_id)
|
|
118
|
+
except ClaimConflictError as exc:
|
|
119
|
+
raise click.ClickException(
|
|
120
|
+
_maintenance_claim_message(
|
|
121
|
+
product=product,
|
|
122
|
+
operation=operation,
|
|
123
|
+
detail=str(exc) or "write claim acquisition failed",
|
|
124
|
+
)
|
|
125
|
+
) from exc
|
|
126
|
+
return domain
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
|
|
130
|
+
@click.pass_context
|
|
131
|
+
def archive(ctx: click.Context) -> None:
|
|
132
|
+
"""create and manage Tensogram archives
|
|
133
|
+
|
|
134
|
+
Archives compress all array groups from a Zarr store into a single portable
|
|
135
|
+
file using configurable codecs. Use archive create to produce an archive
|
|
136
|
+
and archive restore to extract it back to Zarr.
|
|
137
|
+
"""
|
|
138
|
+
observability.init_observability("firecube-archive")
|
|
139
|
+
ctx.ensure_object(dict)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@archive.command(
|
|
143
|
+
"create",
|
|
144
|
+
epilog="""\b
|
|
145
|
+
Examples:
|
|
146
|
+
# archive a local Zarr store with default compression
|
|
147
|
+
firecube archive create \\
|
|
148
|
+
--source file:///path/to/product.zarr \\
|
|
149
|
+
--archive file:///tmp/archive.tgm
|
|
150
|
+
# archive a specific time range
|
|
151
|
+
firecube archive create \\
|
|
152
|
+
--source file:///path/to/product.zarr \\
|
|
153
|
+
--archive file:///tmp/archive.tgm \\
|
|
154
|
+
--start-date 2024-01-01 \\
|
|
155
|
+
--end-date 2024-02-01
|
|
156
|
+
# archive a single group with explicit codec
|
|
157
|
+
firecube archive create \\
|
|
158
|
+
--source file:///path/to/product.zarr \\
|
|
159
|
+
--archive file:///tmp/archive.tgm \\
|
|
160
|
+
--group <group> \\
|
|
161
|
+
--compression zstd
|
|
162
|
+
See also: firecube archive restore, firecube archive info, firecube archive validate
|
|
163
|
+
""",
|
|
164
|
+
)
|
|
165
|
+
@click.option(
|
|
166
|
+
"-s", "--source", required=True, help="Zarr store URI (file:///abs/path or s3://bucket/key)"
|
|
167
|
+
)
|
|
168
|
+
@archive_uri_option(tier="write")
|
|
169
|
+
@click.option(
|
|
170
|
+
"--storage-type",
|
|
171
|
+
"storage_type",
|
|
172
|
+
required=False,
|
|
173
|
+
type=click.Choice(["local", "s3"], case_sensitive=False),
|
|
174
|
+
)
|
|
175
|
+
@click.option(
|
|
176
|
+
"--storage-driver",
|
|
177
|
+
"storage_driver",
|
|
178
|
+
required=False,
|
|
179
|
+
type=click.Choice(["fsspec", "obstore"], case_sensitive=False),
|
|
180
|
+
)
|
|
181
|
+
@click.option("--start-date", default=None, help="ISO 8601 start date for time range filter")
|
|
182
|
+
@click.option("--end-date", default=None, help="ISO 8601 end date for time range filter")
|
|
183
|
+
@click.option("-g", "--group", default=None, help="product group to archive (e.g. F024, NORDLIS)")
|
|
184
|
+
@click.option("--variables", default=None, help="comma-separated variable names to include")
|
|
185
|
+
@click.option(
|
|
186
|
+
"--compression",
|
|
187
|
+
default=None,
|
|
188
|
+
type=click.Choice(["blosc2", "szip", "zstd", "lz4", "zfp", "sz3"]),
|
|
189
|
+
help="compression codec for .tgm encoding (default: zstd)",
|
|
190
|
+
)
|
|
191
|
+
@click.option("--overwrite", is_flag=True, default=False, help="overwrite existing target file")
|
|
192
|
+
@click.option(
|
|
193
|
+
"--allow-nan/--no-allow-nan",
|
|
194
|
+
default=None,
|
|
195
|
+
help="allow NaN values in archive (default: true)",
|
|
196
|
+
)
|
|
197
|
+
@click.option(
|
|
198
|
+
"--allow-inf/--no-allow-inf",
|
|
199
|
+
default=None,
|
|
200
|
+
help="allow Inf values in archive (default: true)",
|
|
201
|
+
)
|
|
202
|
+
@dry_run_flag
|
|
203
|
+
@yes_flag
|
|
204
|
+
@click.pass_context
|
|
205
|
+
def create(
|
|
206
|
+
ctx: click.Context,
|
|
207
|
+
source: str,
|
|
208
|
+
archive: str,
|
|
209
|
+
storage_type: str | None,
|
|
210
|
+
storage_driver: str | None,
|
|
211
|
+
start_date: str | None,
|
|
212
|
+
end_date: str | None,
|
|
213
|
+
group: str | None,
|
|
214
|
+
variables: str | None,
|
|
215
|
+
compression: str | None,
|
|
216
|
+
overwrite: bool,
|
|
217
|
+
allow_nan: bool | None,
|
|
218
|
+
allow_inf: bool | None,
|
|
219
|
+
dry_run: bool,
|
|
220
|
+
yes_i_really_mean_it: bool,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""convert a Zarr store to a .tgm archive
|
|
223
|
+
|
|
224
|
+
reads all selected array groups, encodes them with the chosen codec
|
|
225
|
+
(default: zstd), and writes a single portable .tgm file. use --start-date
|
|
226
|
+
and --end-date to archive a specific time window, or --group to select one
|
|
227
|
+
product group.
|
|
228
|
+
"""
|
|
229
|
+
from firecube.core.tensogram.converter import zarr_to_tgm
|
|
230
|
+
|
|
231
|
+
parsed_source = parse_product_uri(source)
|
|
232
|
+
storage_type = apply_smart_default(parsed_source, storage_type)
|
|
233
|
+
if is_remote_target(archive):
|
|
234
|
+
raise click.ClickException("Remote .tgm artifacts not yet supported")
|
|
235
|
+
archive_abs = str(local_path_from_target(archive))
|
|
236
|
+
|
|
237
|
+
storage_config = get_storage_config(
|
|
238
|
+
ctx,
|
|
239
|
+
overrides={"storage_type": storage_type, "storage_driver": storage_driver},
|
|
240
|
+
cache=False,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
cfg = get_config(ctx)
|
|
244
|
+
archive_defaults = get_archive_defaults(cfg)
|
|
245
|
+
|
|
246
|
+
if compression is None:
|
|
247
|
+
compression = str(archive_defaults.get("compression", "zstd"))
|
|
248
|
+
if not overwrite:
|
|
249
|
+
overwrite = bool(archive_defaults.get("overwrite", False))
|
|
250
|
+
if allow_nan is None:
|
|
251
|
+
allow_nan = bool(archive_defaults.get("allow_nan", True))
|
|
252
|
+
if allow_inf is None:
|
|
253
|
+
allow_inf = bool(archive_defaults.get("allow_inf", True))
|
|
254
|
+
|
|
255
|
+
_require_overwrite_confirmation(overwrite=overwrite, yes_i_really_mean_it=yes_i_really_mean_it)
|
|
256
|
+
|
|
257
|
+
if dry_run:
|
|
258
|
+
click.echo(f"[dry-run] Would archive: {source} -> {archive_abs}")
|
|
259
|
+
if group:
|
|
260
|
+
click.echo(f" Group: {group}")
|
|
261
|
+
if start_date or end_date:
|
|
262
|
+
click.echo(f" Time range: {start_date or '*'} -> {end_date or '*'}")
|
|
263
|
+
click.echo(f" Compression: {compression or 'zstd'}")
|
|
264
|
+
return
|
|
265
|
+
|
|
266
|
+
variable_list = [v.strip() for v in variables.split(",")] if variables else None
|
|
267
|
+
|
|
268
|
+
driver_config = StorageDriverConfig.from_storage_config(storage_config)
|
|
269
|
+
source_identity = resolve_product_identity(
|
|
270
|
+
source,
|
|
271
|
+
format="zarr",
|
|
272
|
+
product_name=source.rstrip("/").rsplit("/", 1)[-1],
|
|
273
|
+
option_name="--source",
|
|
274
|
+
)
|
|
275
|
+
session = StorageSession(
|
|
276
|
+
StorageBinding(
|
|
277
|
+
identity=source_identity,
|
|
278
|
+
driver=driver_config,
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
archive_uri = StorageUri.from_local_path(archive_abs)
|
|
283
|
+
create_target = archive_abs
|
|
284
|
+
temp_archive_path: str | None = None
|
|
285
|
+
|
|
286
|
+
if archive_uri.is_remote():
|
|
287
|
+
if session.exists(archive_uri) and not overwrite:
|
|
288
|
+
raise click.ClickException(
|
|
289
|
+
f"Archive file already exists: {archive}. Use overwrite=True to replace it."
|
|
290
|
+
)
|
|
291
|
+
temp_archive_path = _temporary_tgm_path()
|
|
292
|
+
create_target = temp_archive_path
|
|
293
|
+
|
|
294
|
+
try:
|
|
295
|
+
result = zarr_to_tgm(
|
|
296
|
+
source_identity.product_uri.to_str(),
|
|
297
|
+
create_target,
|
|
298
|
+
group=group,
|
|
299
|
+
variables=variable_list,
|
|
300
|
+
start_date=start_date,
|
|
301
|
+
end_date=end_date,
|
|
302
|
+
compression=compression,
|
|
303
|
+
overwrite=overwrite,
|
|
304
|
+
session=session,
|
|
305
|
+
allow_nan=allow_nan,
|
|
306
|
+
allow_inf=allow_inf,
|
|
307
|
+
)
|
|
308
|
+
if temp_archive_path is not None:
|
|
309
|
+
if session.exists(archive_uri) and overwrite:
|
|
310
|
+
session.delete(archive_uri)
|
|
311
|
+
# Cross-endpoint: product session is bound to source zarr, not archive_uri.
|
|
312
|
+
target_session = (
|
|
313
|
+
session_for_uri(archive_uri, session.driver) if archive_uri.is_remote() else None
|
|
314
|
+
)
|
|
315
|
+
transfer_copy_file(
|
|
316
|
+
StorageUri.from_local_path(temp_archive_path),
|
|
317
|
+
archive_uri,
|
|
318
|
+
target_session=target_session,
|
|
319
|
+
)
|
|
320
|
+
result["target"] = archive_uri.to_str()
|
|
321
|
+
except (FileExistsError, ValueError, ImportError) as exc:
|
|
322
|
+
raise click.ClickException(str(exc)) from exc
|
|
323
|
+
finally:
|
|
324
|
+
if temp_archive_path is not None and os.path.exists(temp_archive_path):
|
|
325
|
+
os.unlink(temp_archive_path)
|
|
326
|
+
|
|
327
|
+
click.echo(f"Archive created: {result['target']}")
|
|
328
|
+
if result.get("groups"):
|
|
329
|
+
click.echo(f" Groups: {', '.join(result['groups'])}")
|
|
330
|
+
click.echo(f" Variables: {', '.join(result['variables'])}")
|
|
331
|
+
if result.get("time_range"):
|
|
332
|
+
tr = result["time_range"]
|
|
333
|
+
click.echo(f" Time range: {tr.get('start')} → {tr.get('end')} ({tr.get('n')} steps)")
|
|
334
|
+
elif any(result.get("time_ranges", {}).values()):
|
|
335
|
+
click.echo(f" Time ranges: {len(result['time_ranges'])} groups")
|
|
336
|
+
size_mb = result["file_size_bytes"] / (1024 * 1024)
|
|
337
|
+
click.echo(f" Size: {size_mb:.2f} MB | Codec: {result['compression']}")
|
|
338
|
+
if result.get("skipped"):
|
|
339
|
+
click.echo(f" Skipped: {', '.join(result['skipped'])} (unsupported dtype)", err=True)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
@archive.command(
|
|
343
|
+
"restore",
|
|
344
|
+
epilog="""\b
|
|
345
|
+
Examples:
|
|
346
|
+
# restore an archive to a local Zarr store
|
|
347
|
+
firecube archive restore \\
|
|
348
|
+
--archive file:///tmp/archive.tgm \\
|
|
349
|
+
--target file:///path/to/restored.zarr
|
|
350
|
+
# restore and overwrite an existing store
|
|
351
|
+
firecube archive restore \\
|
|
352
|
+
--archive file:///tmp/archive.tgm \\
|
|
353
|
+
--target file:///path/to/product.zarr \\
|
|
354
|
+
--overwrite
|
|
355
|
+
See also: firecube archive create, firecube archive info
|
|
356
|
+
""",
|
|
357
|
+
)
|
|
358
|
+
@archive_uri_option(tier="write")
|
|
359
|
+
@target_uri_option(tier="write")
|
|
360
|
+
@click.option(
|
|
361
|
+
"--storage-type",
|
|
362
|
+
"storage_type",
|
|
363
|
+
required=False,
|
|
364
|
+
type=click.Choice(["local", "s3"], case_sensitive=False),
|
|
365
|
+
)
|
|
366
|
+
@click.option(
|
|
367
|
+
"--storage-driver",
|
|
368
|
+
"storage_driver",
|
|
369
|
+
required=False,
|
|
370
|
+
type=click.Choice(["fsspec", "obstore"], case_sensitive=False),
|
|
371
|
+
)
|
|
372
|
+
@click.option("--overwrite", is_flag=True, default=False, help="overwrite existing Zarr store")
|
|
373
|
+
@dry_run_flag
|
|
374
|
+
@yes_flag
|
|
375
|
+
@click.pass_context
|
|
376
|
+
def restore(
|
|
377
|
+
ctx: click.Context,
|
|
378
|
+
archive: str,
|
|
379
|
+
target: str,
|
|
380
|
+
storage_type: str | None,
|
|
381
|
+
storage_driver: str | None,
|
|
382
|
+
overwrite: bool,
|
|
383
|
+
dry_run: bool,
|
|
384
|
+
yes_i_really_mean_it: bool,
|
|
385
|
+
) -> None:
|
|
386
|
+
"""restore a .tgm archive to a Zarr store
|
|
387
|
+
|
|
388
|
+
decodes all encoded array groups from the .tgm file and writes them back
|
|
389
|
+
to the target Zarr store. archive groups restore to their original paths.
|
|
390
|
+
"""
|
|
391
|
+
parsed_target = parse_product_uri(target)
|
|
392
|
+
storage_type = apply_smart_default(parsed_target, storage_type)
|
|
393
|
+
if is_remote_target(archive):
|
|
394
|
+
raise click.ClickException("Remote .tgm artifacts not yet supported")
|
|
395
|
+
archive_abs = str(local_path_from_target(archive))
|
|
396
|
+
|
|
397
|
+
_require_overwrite_confirmation(overwrite=overwrite, yes_i_really_mean_it=yes_i_really_mean_it)
|
|
398
|
+
|
|
399
|
+
if dry_run:
|
|
400
|
+
click.echo(f"[dry-run] Would restore: {archive_abs} -> {target}")
|
|
401
|
+
if overwrite:
|
|
402
|
+
click.echo(" Overwrite: yes")
|
|
403
|
+
return
|
|
404
|
+
|
|
405
|
+
from firecube.core.tensogram.restore import tgm_to_zarr
|
|
406
|
+
|
|
407
|
+
storage_config = get_storage_config(
|
|
408
|
+
ctx,
|
|
409
|
+
overrides={"storage_type": storage_type, "storage_driver": storage_driver},
|
|
410
|
+
cache=False,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
driver_config = StorageDriverConfig.from_storage_config(storage_config)
|
|
414
|
+
restore_identity = resolve_product_identity(
|
|
415
|
+
target,
|
|
416
|
+
format="zarr",
|
|
417
|
+
product_name=target.rstrip("/").rsplit("/", 1)[-1],
|
|
418
|
+
option_name="--target",
|
|
419
|
+
)
|
|
420
|
+
session = StorageSession(
|
|
421
|
+
StorageBinding(
|
|
422
|
+
identity=restore_identity,
|
|
423
|
+
driver=driver_config,
|
|
424
|
+
)
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
# W3.4: Archive restore is recorded as a first-class WAL run so the audit
|
|
428
|
+
# trail (`firecube chunks runs list`) shows when a product was restored
|
|
429
|
+
# from an archive, what the source archive was, and whether the restore
|
|
430
|
+
# succeeded or failed. We construct the SpanRecorder lazily per phase
|
|
431
|
+
# because tgm_to_zarr's `_prepare_target` may rmtree the entire target
|
|
432
|
+
# directory (including any `.firecube/` we created), which would
|
|
433
|
+
# invalidate a cached WAL writer.
|
|
434
|
+
restore_run_id = f"archive-restore-{uuid.uuid4()}"
|
|
435
|
+
product_name = restore_identity.product_name
|
|
436
|
+
target_uri_str = restore_identity.product_uri.to_str()
|
|
437
|
+
slice_meta: dict[str, Any] = {
|
|
438
|
+
"run_kind": "archive_restore",
|
|
439
|
+
"source_archive": archive,
|
|
440
|
+
"target_product": target_uri_str,
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
def _record_failure(error: str) -> None:
|
|
444
|
+
manager = session.control_plane()
|
|
445
|
+
try:
|
|
446
|
+
recorder = SpanRecorder(manager)
|
|
447
|
+
recorder.register_run_started(
|
|
448
|
+
run_id=restore_run_id,
|
|
449
|
+
product=product_name,
|
|
450
|
+
output_path=target_uri_str,
|
|
451
|
+
output_format="zarr",
|
|
452
|
+
slice_meta=slice_meta,
|
|
453
|
+
)
|
|
454
|
+
manager.record_maintenance_started(
|
|
455
|
+
product=product_name,
|
|
456
|
+
run_id=restore_run_id,
|
|
457
|
+
op=MAINTENANCE_OP_ARCHIVE_RESTORE,
|
|
458
|
+
scope_meta=slice_meta,
|
|
459
|
+
)
|
|
460
|
+
recorder.register_run_failure(
|
|
461
|
+
run_id=restore_run_id,
|
|
462
|
+
product=product_name,
|
|
463
|
+
output_path=target_uri_str,
|
|
464
|
+
output_format="zarr",
|
|
465
|
+
slice_meta=slice_meta,
|
|
466
|
+
error=error,
|
|
467
|
+
)
|
|
468
|
+
manager.record_maintenance_failed(
|
|
469
|
+
product=product_name,
|
|
470
|
+
run_id=restore_run_id,
|
|
471
|
+
op=MAINTENANCE_OP_ARCHIVE_RESTORE,
|
|
472
|
+
scope_meta=slice_meta,
|
|
473
|
+
error=error,
|
|
474
|
+
)
|
|
475
|
+
finally:
|
|
476
|
+
manager.close()
|
|
477
|
+
|
|
478
|
+
archive_uri = StorageUri.from_local_path(archive_abs)
|
|
479
|
+
restore_source = archive_abs
|
|
480
|
+
temp_archive_path: str | None = None
|
|
481
|
+
|
|
482
|
+
if archive_uri.is_remote():
|
|
483
|
+
temp_archive_path = _temporary_tgm_path()
|
|
484
|
+
try:
|
|
485
|
+
# Cross-endpoint: product session is bound to restore target, not archive_uri.
|
|
486
|
+
source_archive_session = session_for_uri(archive_uri, session.driver)
|
|
487
|
+
transfer_copy_file(
|
|
488
|
+
archive_uri,
|
|
489
|
+
StorageUri.from_local_path(temp_archive_path),
|
|
490
|
+
source_session=source_archive_session,
|
|
491
|
+
)
|
|
492
|
+
except Exception as exc:
|
|
493
|
+
_record_failure(str(exc))
|
|
494
|
+
raise
|
|
495
|
+
restore_source = temp_archive_path
|
|
496
|
+
|
|
497
|
+
claim_manager = session.control_plane()
|
|
498
|
+
claim_domain: WriteDomain | None = None
|
|
499
|
+
try:
|
|
500
|
+
claim_domain = _acquire_maintenance_claim(
|
|
501
|
+
manager=claim_manager,
|
|
502
|
+
product=product_name,
|
|
503
|
+
operation="archive_restore",
|
|
504
|
+
owner_id=restore_run_id,
|
|
505
|
+
)
|
|
506
|
+
try:
|
|
507
|
+
result = tgm_to_zarr(
|
|
508
|
+
restore_source,
|
|
509
|
+
target_uri_str,
|
|
510
|
+
overwrite=overwrite,
|
|
511
|
+
session=session,
|
|
512
|
+
on_group_restored=lambda restored_group: click.echo(
|
|
513
|
+
f"restored group: {restored_group or '/'}"
|
|
514
|
+
),
|
|
515
|
+
)
|
|
516
|
+
if not result.get("variables") and not result.get("groups"):
|
|
517
|
+
raise ValueError(
|
|
518
|
+
f"Archive restore produced no output: {restore_source!r} "
|
|
519
|
+
"may be empty or not a valid tensogram archive."
|
|
520
|
+
)
|
|
521
|
+
finally:
|
|
522
|
+
if temp_archive_path is not None and os.path.exists(temp_archive_path):
|
|
523
|
+
os.unlink(temp_archive_path)
|
|
524
|
+
except (FileExistsError, ImportError, ValueError) as exc:
|
|
525
|
+
_record_failure(str(exc))
|
|
526
|
+
raise click.ClickException(str(exc)) from exc
|
|
527
|
+
except Exception as exc:
|
|
528
|
+
_record_failure(str(exc))
|
|
529
|
+
raise
|
|
530
|
+
else:
|
|
531
|
+
success_manager = session.control_plane()
|
|
532
|
+
try:
|
|
533
|
+
success_recorder = SpanRecorder(success_manager)
|
|
534
|
+
success_recorder.register_run_started(
|
|
535
|
+
run_id=restore_run_id,
|
|
536
|
+
product=product_name,
|
|
537
|
+
output_path=target_uri_str,
|
|
538
|
+
output_format="zarr",
|
|
539
|
+
slice_meta=slice_meta,
|
|
540
|
+
)
|
|
541
|
+
success_manager.record_maintenance_started(
|
|
542
|
+
product=product_name,
|
|
543
|
+
run_id=restore_run_id,
|
|
544
|
+
op=MAINTENANCE_OP_ARCHIVE_RESTORE,
|
|
545
|
+
scope_meta=slice_meta,
|
|
546
|
+
)
|
|
547
|
+
synthetic_ctx = RuntimeIngestContext(
|
|
548
|
+
source=archive,
|
|
549
|
+
target=target_uri_str,
|
|
550
|
+
output_format="zarr",
|
|
551
|
+
run_id=restore_run_id,
|
|
552
|
+
identity=RuntimeIdentity(run_id=restore_run_id),
|
|
553
|
+
flags=RuntimeFlags(force_reingest=False),
|
|
554
|
+
)
|
|
555
|
+
synthetic_result = IngestResult(
|
|
556
|
+
outputs=OutputPaths(primary=target_uri_str),
|
|
557
|
+
output_format="zarr",
|
|
558
|
+
metrics={},
|
|
559
|
+
)
|
|
560
|
+
success_recorder.register_run(
|
|
561
|
+
ctx=synthetic_ctx,
|
|
562
|
+
result=synthetic_result,
|
|
563
|
+
run_id=restore_run_id,
|
|
564
|
+
product=product_name,
|
|
565
|
+
slice_meta=slice_meta,
|
|
566
|
+
)
|
|
567
|
+
success_manager.record_maintenance_completed(
|
|
568
|
+
product=product_name,
|
|
569
|
+
run_id=restore_run_id,
|
|
570
|
+
op=MAINTENANCE_OP_ARCHIVE_RESTORE,
|
|
571
|
+
scope_meta=slice_meta,
|
|
572
|
+
)
|
|
573
|
+
finally:
|
|
574
|
+
success_manager.close()
|
|
575
|
+
finally:
|
|
576
|
+
try:
|
|
577
|
+
if claim_domain is not None:
|
|
578
|
+
claim_manager.clear_claim(
|
|
579
|
+
product=product_name,
|
|
580
|
+
domain_id=claim_domain.identifier,
|
|
581
|
+
force=True,
|
|
582
|
+
)
|
|
583
|
+
finally:
|
|
584
|
+
claim_manager.close()
|
|
585
|
+
|
|
586
|
+
click.echo(f"Zarr restored: {result['target']}")
|
|
587
|
+
if result.get("groups"):
|
|
588
|
+
click.echo(f" Groups: {', '.join(result['groups'])}")
|
|
589
|
+
click.echo(f" Variables: {', '.join(result['variables'])}")
|
|
590
|
+
click.echo(f" Coordinates: {', '.join(result['coordinates'])}")
|
|
591
|
+
click.echo(
|
|
592
|
+
" Control-plane: restored"
|
|
593
|
+
if result.get("controlplane_restored")
|
|
594
|
+
else " Control-plane: not present"
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
@archive.command(
|
|
599
|
+
"info",
|
|
600
|
+
epilog="""\b
|
|
601
|
+
Examples:
|
|
602
|
+
# show metadata for an archive
|
|
603
|
+
firecube archive info --archive file:///tmp/archive.tgm
|
|
604
|
+
|
|
605
|
+
\b
|
|
606
|
+
# output metadata as JSON
|
|
607
|
+
firecube archive info --archive file:///tmp/archive.tgm -f json
|
|
608
|
+
|
|
609
|
+
See also: firecube archive list, firecube archive validate
|
|
610
|
+
""",
|
|
611
|
+
)
|
|
612
|
+
@archive_uri_option(tier="inspect")
|
|
613
|
+
@display_format_option(default="table")
|
|
614
|
+
@click.pass_context
|
|
615
|
+
def info(ctx: click.Context, archive: str, output_format: str) -> None:
|
|
616
|
+
"""show metadata for a .tgm archive
|
|
617
|
+
|
|
618
|
+
reads the archive header to display variable names, dimension sizes, codec,
|
|
619
|
+
and time coverage. no data is decoded or written; suitable for quick
|
|
620
|
+
inspection.
|
|
621
|
+
"""
|
|
622
|
+
from firecube.core.tensogram._compat import require_tensogram
|
|
623
|
+
from firecube.core.tensogram.schema import (
|
|
624
|
+
ARCHIVE_VERSION,
|
|
625
|
+
KEY_ARCHIVE_VERSION,
|
|
626
|
+
ROLE_CONTROLPLANE,
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
parsed = parse_product_uri(archive)
|
|
630
|
+
if parsed.scheme == "s3":
|
|
631
|
+
raise click.ClickException("Remote .tgm artifacts not yet supported")
|
|
632
|
+
local_path = parsed.normalized.removeprefix("file://")
|
|
633
|
+
if not Path(local_path).exists():
|
|
634
|
+
raise click.ClickException(f"Archive not found: {archive}")
|
|
635
|
+
|
|
636
|
+
require_tensogram("firecube archive info")
|
|
637
|
+
|
|
638
|
+
import json as _json
|
|
639
|
+
|
|
640
|
+
import tensogram as _tensogram
|
|
641
|
+
|
|
642
|
+
tensogram = cast(Any, _tensogram)
|
|
643
|
+
as_json = output_format == "json"
|
|
644
|
+
|
|
645
|
+
with tensogram.TensogramFile.open(local_path) as f:
|
|
646
|
+
count = f.message_count()
|
|
647
|
+
if count == 0:
|
|
648
|
+
raise click.ClickException("Archive contains no messages.")
|
|
649
|
+
|
|
650
|
+
first_meta = f.file_decode_metadata(0)
|
|
651
|
+
first_extra = first_meta.extra or {}
|
|
652
|
+
firecube_meta = first_extra.get("firecube", {})
|
|
653
|
+
archive_version = str(firecube_meta.get(KEY_ARCHIVE_VERSION, ""))
|
|
654
|
+
if archive_version != ARCHIVE_VERSION:
|
|
655
|
+
raise click.ClickException(
|
|
656
|
+
f"Unsupported Firecube archive version: {archive_version or 'missing'}. "
|
|
657
|
+
f"Expected {ARCHIVE_VERSION}."
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
groups_info: list[dict[str, Any]] = []
|
|
661
|
+
has_controlplane = False
|
|
662
|
+
|
|
663
|
+
for i in range(count):
|
|
664
|
+
meta = f.file_decode_metadata(i)
|
|
665
|
+
extra = meta.extra or {}
|
|
666
|
+
fc = extra.get("firecube", {})
|
|
667
|
+
role = fc.get("role", "data")
|
|
668
|
+
|
|
669
|
+
if role == ROLE_CONTROLPLANE:
|
|
670
|
+
has_controlplane = True
|
|
671
|
+
continue
|
|
672
|
+
|
|
673
|
+
group_name = fc.get("group", "")
|
|
674
|
+
base_entries = list(meta.base) if meta.base else []
|
|
675
|
+
result = f.file_decode_descriptors(i)
|
|
676
|
+
descriptors = result.get("descriptors", [])
|
|
677
|
+
archived_coords = set(fc.get("coordinates", []))
|
|
678
|
+
|
|
679
|
+
group_vars: dict[str, dict] = {}
|
|
680
|
+
for j, desc in enumerate(descriptors):
|
|
681
|
+
base_entry = base_entries[j] if j < len(base_entries) else {}
|
|
682
|
+
obj_name = str(base_entry.get("name") or f"object_{j}")
|
|
683
|
+
if obj_name not in archived_coords:
|
|
684
|
+
group_vars[obj_name] = {
|
|
685
|
+
"shape": list(desc.shape) if desc.shape else [],
|
|
686
|
+
"dtype": str(desc.dtype),
|
|
687
|
+
"dims": base_entry.get("dim_names", []),
|
|
688
|
+
"zarr_chunks": base_entry.get("zarr_chunks"),
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
groups_info.append(
|
|
692
|
+
{
|
|
693
|
+
"group": group_name,
|
|
694
|
+
"variables": group_vars,
|
|
695
|
+
"compression": fc.get("compression", ""),
|
|
696
|
+
"source_uri": fc.get("source_uri", ""),
|
|
697
|
+
"archived_at": fc.get("archived_at", ""),
|
|
698
|
+
}
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
info_dict: dict[str, Any] = {
|
|
702
|
+
"path": local_path,
|
|
703
|
+
"format": archive_version,
|
|
704
|
+
"size_bytes": os.path.getsize(local_path),
|
|
705
|
+
"groups": [g["group"] for g in groups_info],
|
|
706
|
+
"has_controlplane": has_controlplane,
|
|
707
|
+
"group_details": groups_info,
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
if as_json:
|
|
711
|
+
click.echo(_json.dumps(info_dict, indent=2))
|
|
712
|
+
else:
|
|
713
|
+
click.echo(f"Path: {local_path}")
|
|
714
|
+
click.echo(f"Format: {archive_version} (multi-group)")
|
|
715
|
+
size_mb = os.path.getsize(local_path) / (1024 * 1024)
|
|
716
|
+
click.echo(f"Size: {size_mb:.2f} MB")
|
|
717
|
+
click.echo(
|
|
718
|
+
"Groups: " + (", ".join(g["group"] for g in groups_info) or "(none)")
|
|
719
|
+
)
|
|
720
|
+
click.echo("Control-plane: " + ("present" if has_controlplane else "not present"))
|
|
721
|
+
for g in groups_info:
|
|
722
|
+
click.echo(f"\nGroup: {g['group'] or '(root)'}")
|
|
723
|
+
click.echo(f" Source: {g['source_uri']}")
|
|
724
|
+
click.echo(f" Archived: {g['archived_at']}")
|
|
725
|
+
click.echo(f" Codec: {g['compression']}")
|
|
726
|
+
click.echo(" Variables:")
|
|
727
|
+
for vname, vmeta in g["variables"].items():
|
|
728
|
+
chunks_str = (
|
|
729
|
+
f" chunks={vmeta['zarr_chunks']}" if vmeta.get("zarr_chunks") else ""
|
|
730
|
+
)
|
|
731
|
+
click.echo(
|
|
732
|
+
f" {vname}: shape={vmeta['shape']} dtype={vmeta['dtype']}{chunks_str}"
|
|
733
|
+
)
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
@archive.command(
|
|
737
|
+
"validate",
|
|
738
|
+
epilog="""\b
|
|
739
|
+
Examples:
|
|
740
|
+
# validate an archive and check exit code
|
|
741
|
+
firecube archive validate --archive file:///tmp/archive.tgm && echo "ok"
|
|
742
|
+
|
|
743
|
+
\b
|
|
744
|
+
# quick structure-only validation
|
|
745
|
+
firecube archive validate --archive file:///tmp/archive.tgm --quick
|
|
746
|
+
|
|
747
|
+
See also: firecube archive info, firecube archive create
|
|
748
|
+
""",
|
|
749
|
+
)
|
|
750
|
+
@archive_uri_option(tier="inspect")
|
|
751
|
+
@click.option(
|
|
752
|
+
"--quick", is_flag=True, default=False, help="structure-only check (no hash verification)"
|
|
753
|
+
)
|
|
754
|
+
@click.pass_context
|
|
755
|
+
def validate(ctx: click.Context, archive: str, quick: bool) -> None:
|
|
756
|
+
"""check integrity of a .tgm archive
|
|
757
|
+
|
|
758
|
+
reads the archive header and validates internal structure without fully
|
|
759
|
+
decoding data. exits with code 0 if the archive is valid, code 1 if it is
|
|
760
|
+
corrupted or incomplete.
|
|
761
|
+
"""
|
|
762
|
+
from firecube.core.tensogram._compat import require_tensogram
|
|
763
|
+
|
|
764
|
+
parsed = parse_product_uri(archive)
|
|
765
|
+
if parsed.scheme == "s3":
|
|
766
|
+
raise click.ClickException("Remote .tgm artifacts not yet supported")
|
|
767
|
+
local_path = parsed.normalized.removeprefix("file://")
|
|
768
|
+
if not Path(local_path).exists():
|
|
769
|
+
raise click.ClickException(f"Archive not found: {archive}")
|
|
770
|
+
|
|
771
|
+
require_tensogram("firecube archive validate")
|
|
772
|
+
|
|
773
|
+
import tensogram as _tensogram
|
|
774
|
+
|
|
775
|
+
tensogram = cast(Any, _tensogram)
|
|
776
|
+
|
|
777
|
+
level = "quick" if quick else "default"
|
|
778
|
+
report = tensogram.validate_file(local_path, level=level)
|
|
779
|
+
|
|
780
|
+
file_issues = report.get("file_issues", [])
|
|
781
|
+
messages = report.get("messages", [])
|
|
782
|
+
|
|
783
|
+
hard_issues: list[tuple[str, dict[str, Any]]] = [("file", i) for i in file_issues]
|
|
784
|
+
hard_issues.extend(
|
|
785
|
+
(f"msg {i}", issue)
|
|
786
|
+
for i, msg_report in enumerate(messages)
|
|
787
|
+
for issue in msg_report.get("issues", [])
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
if not hard_issues:
|
|
791
|
+
click.echo(f"VALID: {local_path} ({len(messages)} message(s), no issues)")
|
|
792
|
+
else:
|
|
793
|
+
click.echo(f"INVALID: {local_path} ({len(hard_issues)} issue(s))")
|
|
794
|
+
for scope, issue in hard_issues:
|
|
795
|
+
if scope == "file":
|
|
796
|
+
click.echo(
|
|
797
|
+
f" [file] offset={issue.get('byte_offset')}: {issue.get('description')}"
|
|
798
|
+
)
|
|
799
|
+
else:
|
|
800
|
+
click.echo(f" [{scope}] {issue}")
|
|
801
|
+
raise SystemExit(1)
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
@archive.command(
|
|
805
|
+
"list",
|
|
806
|
+
epilog="""\b
|
|
807
|
+
Examples:
|
|
808
|
+
# list archive contents
|
|
809
|
+
firecube archive list --archive file:///tmp/archive.tgm
|
|
810
|
+
|
|
811
|
+
See also: firecube archive info, firecube archive restore
|
|
812
|
+
""",
|
|
813
|
+
)
|
|
814
|
+
@archive_uri_option(tier="inspect")
|
|
815
|
+
@click.pass_context
|
|
816
|
+
def list_cmd(ctx: click.Context, archive: str) -> None:
|
|
817
|
+
"""list contents of a .tgm archive
|
|
818
|
+
|
|
819
|
+
displays per-variable group entries including name, dimensions, and time
|
|
820
|
+
range. no data is decoded; useful for reviewing archive contents before
|
|
821
|
+
restoring.
|
|
822
|
+
"""
|
|
823
|
+
from firecube.core.tensogram._compat import require_tensogram
|
|
824
|
+
from firecube.core.tensogram.schema import ROLE_CONTROLPLANE
|
|
825
|
+
|
|
826
|
+
parsed = parse_product_uri(archive)
|
|
827
|
+
if parsed.scheme == "s3":
|
|
828
|
+
raise click.ClickException("Remote .tgm artifacts not yet supported")
|
|
829
|
+
local_path = parsed.normalized.removeprefix("file://")
|
|
830
|
+
if not Path(local_path).exists():
|
|
831
|
+
raise click.ClickException(f"Archive not found: {archive}")
|
|
832
|
+
|
|
833
|
+
require_tensogram("firecube archive list")
|
|
834
|
+
|
|
835
|
+
import tensogram as _tensogram
|
|
836
|
+
|
|
837
|
+
tensogram = cast(Any, _tensogram)
|
|
838
|
+
|
|
839
|
+
with tensogram.TensogramFile.open(local_path) as f:
|
|
840
|
+
count = f.message_count()
|
|
841
|
+
click.echo(f"Messages: {count}")
|
|
842
|
+
for i in range(count):
|
|
843
|
+
meta = f.file_decode_metadata(i)
|
|
844
|
+
extra = meta.extra or {}
|
|
845
|
+
fc = extra.get("firecube", {})
|
|
846
|
+
role = fc.get("role", "data")
|
|
847
|
+
base_entries = list(meta.base) if meta.base else []
|
|
848
|
+
|
|
849
|
+
if role == ROLE_CONTROLPLANE:
|
|
850
|
+
click.echo("\n[control-plane]")
|
|
851
|
+
click.echo(f" Product: {fc.get('product', '')}")
|
|
852
|
+
continue
|
|
853
|
+
|
|
854
|
+
group_name = fc.get("group") or f"message {i}"
|
|
855
|
+
click.echo(f"\nGroup: {group_name}")
|
|
856
|
+
if fc:
|
|
857
|
+
click.echo(f" Source: {fc.get('source_uri', '')}")
|
|
858
|
+
click.echo(f" Codec: {fc.get('compression', '')}")
|
|
859
|
+
|
|
860
|
+
result = f.file_decode_descriptors(i)
|
|
861
|
+
descriptors = result.get("descriptors", [])
|
|
862
|
+
click.echo(f" Objects: {len(descriptors)}")
|
|
863
|
+
for j, desc in enumerate(descriptors):
|
|
864
|
+
base_entry = base_entries[j] if j < len(base_entries) else {}
|
|
865
|
+
shape = list(desc.shape) if desc.shape else []
|
|
866
|
+
click.echo(
|
|
867
|
+
f" [{j}] {base_entry.get('name') or desc.params.get('name', desc.obj_type)}:"
|
|
868
|
+
f" shape={shape} dtype={desc.dtype}"
|
|
869
|
+
f" compression={desc.compression}"
|
|
870
|
+
)
|