firecube 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- firecube/__init__.py +23 -0
- firecube/__main__.py +20 -0
- firecube/_version.py +46 -0
- firecube/cli/_audience.py +135 -0
- firecube/cli/_command_schemas.py +112 -0
- firecube/cli/_ctx.py +85 -0
- firecube/cli/_errors.py +115 -0
- firecube/cli/_formatter.py +195 -0
- firecube/cli/_product.py +47 -0
- firecube/cli/_rename_hints.py +123 -0
- firecube/cli/_shared_options.py +247 -0
- firecube/cli/_slot_env.py +102 -0
- firecube/cli/_slot_planning.py +95 -0
- firecube/cli/_typed_options.py +181 -0
- firecube/cli/_uri_policy.py +110 -0
- firecube/cli/advise.py +321 -0
- firecube/cli/archive.py +870 -0
- firecube/cli/catalog.py +196 -0
- firecube/cli/chunks/__init__.py +74 -0
- firecube/cli/chunks/_claims.py +229 -0
- firecube/cli/chunks/_common.py +84 -0
- firecube/cli/chunks/_delete.py +376 -0
- firecube/cli/chunks/_list.py +334 -0
- firecube/cli/chunks/_manager.py +161 -0
- firecube/cli/chunks/_runs.py +294 -0
- firecube/cli/chunks/_snapshots.py +165 -0
- firecube/cli/completion.py +73 -0
- firecube/cli/main.py +866 -0
- firecube/cli/parquet.py +289 -0
- firecube/cli/plugins/__init__.py +28 -0
- firecube/cli/plugins/commands.py +262 -0
- firecube/cli/plugins/introspect.py +304 -0
- firecube/cli/plugins/mgmt.py +198 -0
- firecube/cli/plugins/registry.py +132 -0
- firecube/cli/zarr.py +1043 -0
- firecube/core/__init__.py +14 -0
- firecube/core/api.py +88 -0
- firecube/core/cf/__init__.py +23 -0
- firecube/core/cf/check_ids.py +126 -0
- firecube/core/cf/report.py +57 -0
- firecube/core/cf/validator.py +313 -0
- firecube/core/config.py +307 -0
- firecube/core/controlplane/__init__.py +47 -0
- firecube/core/controlplane/_event_processor.py +136 -0
- firecube/core/controlplane/_helpers.py +114 -0
- firecube/core/controlplane/_projection.py +253 -0
- firecube/core/controlplane/_snapshot.py +215 -0
- firecube/core/controlplane/_wal_reader.py +402 -0
- firecube/core/controlplane/_wal_writer.py +589 -0
- firecube/core/controlplane/claims.py +294 -0
- firecube/core/controlplane/deletion.py +704 -0
- firecube/core/controlplane/events.py +219 -0
- firecube/core/controlplane/manager.py +990 -0
- firecube/core/controlplane/metrics.py +97 -0
- firecube/core/controlplane/repo.py +804 -0
- firecube/core/controlplane/repo_utils.py +73 -0
- firecube/core/controlplane/time_dim.py +111 -0
- firecube/core/controlplane/types.py +445 -0
- firecube/core/credentials.py +36 -0
- firecube/core/duckdb/__init__.py +27 -0
- firecube/core/duckdb/bridge.py +82 -0
- firecube/core/duckdb/io.py +30 -0
- firecube/core/errors.py +74 -0
- firecube/core/filesystem/__init__.py +60 -0
- firecube/core/filesystem/_obstore_compat.py +60 -0
- firecube/core/filesystem/fsspec_backend.py +308 -0
- firecube/core/filesystem/instrumentation.py +260 -0
- firecube/core/filesystem/obstore_backend.py +406 -0
- firecube/core/filesystem/ops.py +412 -0
- firecube/core/filesystem/protocol.py +130 -0
- firecube/core/filesystem/store_factory.py +149 -0
- firecube/core/formats/__init__.py +46 -0
- firecube/core/formats/_iso.py +84 -0
- firecube/core/formats/discovery.py +165 -0
- firecube/core/formats/hdf5.py +137 -0
- firecube/core/formats/netcdf.py +226 -0
- firecube/core/formats/zip.py +192 -0
- firecube/core/intake.py +530 -0
- firecube/core/observability/__init__.py +72 -0
- firecube/core/observability/_state.py +24 -0
- firecube/core/observability/logging.py +147 -0
- firecube/core/observability/metrics.py +230 -0
- firecube/core/observability/telemetry/__init__.py +87 -0
- firecube/core/observability/telemetry/pushgateway.py +339 -0
- firecube/core/observability/telemetry/sinks.py +155 -0
- firecube/core/observability/tracing.py +126 -0
- firecube/core/product/__init__.py +45 -0
- firecube/core/product/identity.py +76 -0
- firecube/core/product/resolver.py +121 -0
- firecube/core/product/target.py +120 -0
- firecube/core/runtime.py +187 -0
- firecube/core/slot_index.py +205 -0
- firecube/core/storage/__init__.py +45 -0
- firecube/core/storage/binding.py +39 -0
- firecube/core/storage/cache_key.py +40 -0
- firecube/core/storage/completion.py +245 -0
- firecube/core/storage/driver_config.py +125 -0
- firecube/core/storage/results.py +30 -0
- firecube/core/storage/session.py +431 -0
- firecube/core/storage/transfer.py +142 -0
- firecube/core/storage/uri.py +173 -0
- firecube/core/tensogram/__init__.py +37 -0
- firecube/core/tensogram/_compat.py +36 -0
- firecube/core/tensogram/controlplane_codec.py +230 -0
- firecube/core/tensogram/converter.py +390 -0
- firecube/core/tensogram/metadata.py +274 -0
- firecube/core/tensogram/restore.py +303 -0
- firecube/core/tensogram/schema.py +93 -0
- firecube/core/uris.py +182 -0
- firecube/core/workspaces.py +68 -0
- firecube/core/zarr/__init__.py +69 -0
- firecube/core/zarr/_reserved_attrs.py +47 -0
- firecube/core/zarr/_reserved_root_attrs.py +74 -0
- firecube/core/zarr/io.py +88 -0
- firecube/core/zarr/layers.py +184 -0
- firecube/core/zarr/multires.py +178 -0
- firecube/core/zarr/region_writer.py +782 -0
- firecube/core/zarr/scrub.py +200 -0
- firecube/core/zarr/state.py +258 -0
- firecube/core/zarr/time_decode.py +71 -0
- firecube/core/zarr/validation.py +622 -0
- firecube/ingestor/__init__.py +27 -0
- firecube/ingestor/api.py +215 -0
- firecube/ingestor/config/__init__.py +19 -0
- firecube/ingestor/config/coercion.py +157 -0
- firecube/ingestor/config/engine.py +188 -0
- firecube/ingestor/contracts/__init__.py +14 -0
- firecube/ingestor/contracts/interfaces.py +235 -0
- firecube/ingestor/devtools/_templates/__init__.py +14 -0
- firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
- firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
- firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
- firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
- firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
- firecube/ingestor/devtools/scaffolding.py +229 -0
- firecube/ingestor/errors.py +79 -0
- firecube/ingestor/extensions/_binning.py +152 -0
- firecube/ingestor/extensions/duck.py +153 -0
- firecube/ingestor/extensions/grid.py +287 -0
- firecube/ingestor/extensions/healpix.py +330 -0
- firecube/ingestor/registry/__init__.py +14 -0
- firecube/ingestor/registry/loader.py +144 -0
- firecube/ingestor/registry/metadata.py +97 -0
- firecube/ingestor/registry/version_compat.py +95 -0
- firecube/ingestor/runtime/__init__.py +14 -0
- firecube/ingestor/runtime/aggregation.py +112 -0
- firecube/ingestor/runtime/base.py +836 -0
- firecube/ingestor/runtime/base_hooks.py +129 -0
- firecube/ingestor/runtime/batching.py +161 -0
- firecube/ingestor/runtime/configure.py +195 -0
- firecube/ingestor/runtime/coverage.py +177 -0
- firecube/ingestor/runtime/engine.py +793 -0
- firecube/ingestor/runtime/parallel_evidence.py +75 -0
- firecube/ingestor/runtime/parallel_execution_state.py +28 -0
- firecube/ingestor/runtime/parallel_gate.py +164 -0
- firecube/ingestor/runtime/parallel_run_id.py +48 -0
- firecube/ingestor/runtime/preflight.py +40 -0
- firecube/ingestor/runtime/recording.py +425 -0
- firecube/ingestor/runtime/resume_guard.py +556 -0
- firecube/ingestor/runtime/resume_types.py +65 -0
- firecube/ingestor/runtime/telemetry.py +171 -0
- firecube/ingestor/runtime/tensogram/__init__.py +14 -0
- firecube/ingestor/runtime/tensogram/strategy.py +164 -0
- firecube/ingestor/runtime/workspace.py +220 -0
- firecube/ingestor/runtime/zarr/__init__.py +34 -0
- firecube/ingestor/runtime/zarr/append.py +466 -0
- firecube/ingestor/runtime/zarr/append_services.py +650 -0
- firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
- firecube/ingestor/runtime/zarr/contracts.py +58 -0
- firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
- firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
- firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
- firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
- firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
- firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
- firecube/ingestor/runtime/zarr/write.py +212 -0
- firecube/ingestor/runtime/zarr/write_context.py +139 -0
- firecube/ingestor/templates/__init__.py +19 -0
- firecube/ingestor/templates/config.py +118 -0
- firecube/ingestor/templates/direct_zarr.py +790 -0
- firecube/ingestor/templates/generic.py +462 -0
- firecube/ingestor/templates/generic_tensogram.py +170 -0
- firecube/ingestor/types/__init__.py +19 -0
- firecube/ingestor/types/config.py +89 -0
- firecube/ingestor/types/context.py +409 -0
- firecube/ingestor/types/manifest.py +53 -0
- firecube/ingestor/types/planned_range.py +152 -0
- firecube/ingestor/types/result_metrics.py +386 -0
- firecube/ingestor/utils/__init__.py +14 -0
- firecube/ingestor/utils/duckdb_utils.py +235 -0
- firecube/ingestor/validation.py +55 -0
- firecube-0.1.4.dist-info/METADATA +242 -0
- firecube-0.1.4.dist-info/RECORD +197 -0
- firecube-0.1.4.dist-info/WHEEL +4 -0
- firecube-0.1.4.dist-info/entry_points.txt +2 -0
- firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
- firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Gridding utilities for satellite data.
|
|
17
|
+
|
|
18
|
+
These helpers bin irregular (lat, lon, value) samples onto a regular lat/lon
|
|
19
|
+
grid without interpolation. This is intended for curvilinear geolocation
|
|
20
|
+
grids (2D lat/lon) such as MSG products.
|
|
21
|
+
|
|
22
|
+
Design notes
|
|
23
|
+
------------
|
|
24
|
+
- The target grid extent can be **derived** from the data
|
|
25
|
+
(``arange(floor(min), ceil(max) + spacing, spacing)``) or **supplied** via
|
|
26
|
+
``bounds``. Supplying it pins one fixed grid so many granules/timesteps share
|
|
27
|
+
a coordinate and can be appended, instead of each input deriving its own.
|
|
28
|
+
- Output coordinates are *centers* aligned to full degrees. Binning uses
|
|
29
|
+
half-cell edges around centers, so all centers correspond to a real cell.
|
|
30
|
+
This fixes the common off-by-one and "extra empty last row/col" issues that
|
|
31
|
+
arise when treating centers as edges.
|
|
32
|
+
- The binning/aggregation core is shared with :mod:`.healpix` via
|
|
33
|
+
:mod:`._binning`. Performance: mean/min/max/first/last/any are vectorized;
|
|
34
|
+
median is supported but may be expensive on large grids.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
import xarray as xr
|
|
43
|
+
|
|
44
|
+
from ._binning import aggregate_by_position
|
|
45
|
+
from ._binning import as_float_array as _as_float_array
|
|
46
|
+
from ._binning import broadcast_latlon as _broadcast_latlon
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True, slots=True)
|
|
50
|
+
class LatLonBinner:
|
|
51
|
+
"""Precomputed bin mapping from an input grid to a regular lat/lon grid."""
|
|
52
|
+
|
|
53
|
+
lat: np.ndarray # 1D centers
|
|
54
|
+
lon: np.ndarray # 1D centers
|
|
55
|
+
cell_index: np.ndarray # 1D flattened cell indices (len == n_valid_points)
|
|
56
|
+
valid_point_index: np.ndarray # 1D indices into flattened input arrays
|
|
57
|
+
nlat: int
|
|
58
|
+
nlon: int
|
|
59
|
+
spacing: float
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def build_latlon_binner(
|
|
63
|
+
*,
|
|
64
|
+
lat: np.ndarray,
|
|
65
|
+
lon: np.ndarray,
|
|
66
|
+
grid_spacing: float,
|
|
67
|
+
bounds: tuple[float, float, float, float] | None = None,
|
|
68
|
+
fill_value: float | None = None,
|
|
69
|
+
) -> LatLonBinner:
|
|
70
|
+
"""Precompute the mapping from an input lat/lon grid to regular bins.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
lat, lon:
|
|
75
|
+
Input geolocation, 1-D coordinate vectors or matching N-D arrays.
|
|
76
|
+
grid_spacing:
|
|
77
|
+
Target cell size in degrees (> 0).
|
|
78
|
+
bounds:
|
|
79
|
+
Optional ``(lat_min, lat_max, lon_min, lon_max)`` fixing the output
|
|
80
|
+
grid extent. When given, the axis is exactly these bounds and input
|
|
81
|
+
points outside them are dropped -- pin one set of bounds to reuse the
|
|
82
|
+
same grid across granules. When ``None``, the extent is derived from
|
|
83
|
+
the data (``floor(min)``/``ceil(max)``).
|
|
84
|
+
fill_value:
|
|
85
|
+
Accepted for signature parity; data masking is applied by callers.
|
|
86
|
+
"""
|
|
87
|
+
if grid_spacing <= 0:
|
|
88
|
+
raise ValueError("grid_spacing must be > 0")
|
|
89
|
+
|
|
90
|
+
lat_flat, lon_flat, _data_dummy = _broadcast_latlon(
|
|
91
|
+
lat, lon, np.zeros_like(lat, dtype=np.float32)
|
|
92
|
+
)
|
|
93
|
+
lat_flat = _as_float_array(lat_flat)
|
|
94
|
+
lon_flat = _as_float_array(lon_flat)
|
|
95
|
+
|
|
96
|
+
valid_mask = ~(np.isnan(lat_flat) | np.isnan(lon_flat))
|
|
97
|
+
|
|
98
|
+
lat_valid = lat_flat[valid_mask]
|
|
99
|
+
lon_valid = lon_flat[valid_mask]
|
|
100
|
+
valid_point_index = np.flatnonzero(valid_mask)
|
|
101
|
+
|
|
102
|
+
if lat_valid.size == 0:
|
|
103
|
+
raise ValueError("No valid lat/lon points found after filtering NaNs")
|
|
104
|
+
|
|
105
|
+
if bounds is not None:
|
|
106
|
+
lat_min, lat_max, lon_min, lon_max = (float(b) for b in bounds)
|
|
107
|
+
else:
|
|
108
|
+
lat_min = float(np.floor(np.nanmin(lat_valid)))
|
|
109
|
+
lat_max = float(np.ceil(np.nanmax(lat_valid)))
|
|
110
|
+
lon_min = float(np.floor(np.nanmin(lon_valid)))
|
|
111
|
+
lon_max = float(np.ceil(np.nanmax(lon_valid)))
|
|
112
|
+
|
|
113
|
+
lat_centers = np.arange(lat_min, lat_max + grid_spacing, grid_spacing, dtype=np.float64)
|
|
114
|
+
lon_centers = np.arange(lon_min, lon_max + grid_spacing, grid_spacing, dtype=np.float64)
|
|
115
|
+
|
|
116
|
+
# Treat centers as true cell centers; edges are half a spacing away.
|
|
117
|
+
lat0 = lat_centers[0] - (grid_spacing / 2.0)
|
|
118
|
+
lon0 = lon_centers[0] - (grid_spacing / 2.0)
|
|
119
|
+
|
|
120
|
+
lat_idx = np.floor((lat_valid - lat0) / grid_spacing).astype(np.int64, copy=False)
|
|
121
|
+
lon_idx = np.floor((lon_valid - lon0) / grid_spacing).astype(np.int64, copy=False)
|
|
122
|
+
|
|
123
|
+
nlat = int(lat_centers.size)
|
|
124
|
+
nlon = int(lon_centers.size)
|
|
125
|
+
in_bounds = (lat_idx >= 0) & (lat_idx < nlat) & (lon_idx >= 0) & (lon_idx < nlon)
|
|
126
|
+
|
|
127
|
+
lat_idx = lat_idx[in_bounds]
|
|
128
|
+
lon_idx = lon_idx[in_bounds]
|
|
129
|
+
valid_point_index = valid_point_index[in_bounds]
|
|
130
|
+
|
|
131
|
+
cell_index = (lat_idx * nlon + lon_idx).astype(np.int64, copy=False)
|
|
132
|
+
|
|
133
|
+
return LatLonBinner(
|
|
134
|
+
lat=lat_centers.astype(np.float32),
|
|
135
|
+
lon=lon_centers.astype(np.float32),
|
|
136
|
+
cell_index=cell_index,
|
|
137
|
+
valid_point_index=valid_point_index.astype(np.int64, copy=False),
|
|
138
|
+
nlat=nlat,
|
|
139
|
+
nlon=nlon,
|
|
140
|
+
spacing=float(grid_spacing),
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _aggregate_to_grid(
|
|
145
|
+
*,
|
|
146
|
+
values_flat: np.ndarray,
|
|
147
|
+
binner: LatLonBinner,
|
|
148
|
+
aggregation: str,
|
|
149
|
+
) -> np.ndarray:
|
|
150
|
+
"""Aggregate flattened values into a (lat, lon) grid using a precomputed binner."""
|
|
151
|
+
flat = aggregate_by_position(
|
|
152
|
+
values_flat=values_flat,
|
|
153
|
+
valid_point_index=binner.valid_point_index,
|
|
154
|
+
position=binner.cell_index,
|
|
155
|
+
n_targets=binner.nlat * binner.nlon,
|
|
156
|
+
aggregation=aggregation,
|
|
157
|
+
)
|
|
158
|
+
return flat.reshape((binner.nlat, binner.nlon))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def regrid_with_binner(
|
|
162
|
+
*,
|
|
163
|
+
binner: LatLonBinner,
|
|
164
|
+
data: np.ndarray,
|
|
165
|
+
aggregation: str = "mean",
|
|
166
|
+
) -> np.ndarray:
|
|
167
|
+
"""Regrid a 2D field onto the binner's regular grid.
|
|
168
|
+
|
|
169
|
+
Parameters
|
|
170
|
+
----------
|
|
171
|
+
binner:
|
|
172
|
+
Precomputed LatLonBinner mapping for the input geolocation grid.
|
|
173
|
+
data:
|
|
174
|
+
2D array matching the binner's input grid shape.
|
|
175
|
+
aggregation:
|
|
176
|
+
Aggregation method for multiple source pixels per target cell.
|
|
177
|
+
"""
|
|
178
|
+
values_flat = np.asarray(data).ravel()
|
|
179
|
+
return _aggregate_to_grid(values_flat=values_flat, binner=binner, aggregation=aggregation)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def grid_data_to_latlon(
|
|
183
|
+
*,
|
|
184
|
+
lat: np.ndarray,
|
|
185
|
+
lon: np.ndarray,
|
|
186
|
+
data: np.ndarray,
|
|
187
|
+
grid_spacing: float = 0.1,
|
|
188
|
+
variable_name: str = "value",
|
|
189
|
+
fill_value: float | None = None,
|
|
190
|
+
aggregation: str = "mean",
|
|
191
|
+
bounds: tuple[float, float, float, float] | None = None,
|
|
192
|
+
) -> xr.Dataset:
|
|
193
|
+
"""Grid irregular satellite data onto a regular lat/lon grid (no interpolation)."""
|
|
194
|
+
_lat_flat, _lon_flat, data_flat = _broadcast_latlon(lat, lon, data)
|
|
195
|
+
|
|
196
|
+
# Determine which data values are valid (coordinates handled by binner).
|
|
197
|
+
if fill_value is not None:
|
|
198
|
+
valid_data = ~(np.isnan(data_flat) | (data_flat == fill_value))
|
|
199
|
+
else:
|
|
200
|
+
valid_data = ~np.isnan(data_flat)
|
|
201
|
+
|
|
202
|
+
if valid_data.sum() == 0:
|
|
203
|
+
raise ValueError("No valid data points found after filtering invalid values")
|
|
204
|
+
|
|
205
|
+
binner = build_latlon_binner(
|
|
206
|
+
lat=lat, lon=lon, grid_spacing=grid_spacing, bounds=bounds, fill_value=fill_value
|
|
207
|
+
)
|
|
208
|
+
# Apply data mask on top of binner's valid coordinate mask.
|
|
209
|
+
values = np.full_like(data_flat, np.nan, dtype=np.float64)
|
|
210
|
+
values[valid_data] = _as_float_array(data_flat[valid_data])
|
|
211
|
+
|
|
212
|
+
data_gridded = _aggregate_to_grid(values_flat=values, binner=binner, aggregation=aggregation)
|
|
213
|
+
|
|
214
|
+
n_valid_cells = int(np.sum(~np.isnan(data_gridded)))
|
|
215
|
+
n_total_cells = int(data_gridded.size)
|
|
216
|
+
|
|
217
|
+
ds = xr.Dataset(
|
|
218
|
+
data_vars={variable_name: (["lat", "lon"], data_gridded)},
|
|
219
|
+
coords={
|
|
220
|
+
"lat": ("lat", binner.lat),
|
|
221
|
+
"lon": ("lon", binner.lon),
|
|
222
|
+
},
|
|
223
|
+
attrs={
|
|
224
|
+
"title": f"{variable_name} on regular lat/lon grid",
|
|
225
|
+
"description": f"Gridded from irregular data using {aggregation} aggregation (no interpolation)",
|
|
226
|
+
"grid_spacing": f"{grid_spacing}°",
|
|
227
|
+
"aggregation_method": str(aggregation),
|
|
228
|
+
"n_valid_cells": n_valid_cells,
|
|
229
|
+
"n_total_cells": n_total_cells,
|
|
230
|
+
"fill_percentage": float(n_valid_cells / n_total_cells * 100.0)
|
|
231
|
+
if n_total_cells
|
|
232
|
+
else 0.0,
|
|
233
|
+
"lat_range": f"({float(binner.lat.min()):.2f}, {float(binner.lat.max()):.2f})",
|
|
234
|
+
"lon_range": f"({float(binner.lon.min()):.2f}, {float(binner.lon.max()):.2f})",
|
|
235
|
+
},
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
return ds
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def grid_xarray_dataset(
|
|
242
|
+
*,
|
|
243
|
+
ds: xr.Dataset,
|
|
244
|
+
lat_var: str,
|
|
245
|
+
lon_var: str,
|
|
246
|
+
data_vars: list[str],
|
|
247
|
+
grid_spacing: float = 0.1,
|
|
248
|
+
fill_value: float | None = None,
|
|
249
|
+
aggregation: str = "mean",
|
|
250
|
+
bounds: tuple[float, float, float, float] | None = None,
|
|
251
|
+
) -> xr.Dataset:
|
|
252
|
+
"""Grid multiple variables from an xarray Dataset onto a regular lat/lon grid."""
|
|
253
|
+
if not data_vars:
|
|
254
|
+
raise ValueError("data_vars must contain at least one variable name")
|
|
255
|
+
|
|
256
|
+
lat = ds[lat_var].values
|
|
257
|
+
lon = ds[lon_var].values
|
|
258
|
+
binner = build_latlon_binner(
|
|
259
|
+
lat=lat, lon=lon, grid_spacing=grid_spacing, bounds=bounds, fill_value=fill_value
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
out: xr.Dataset | None = None
|
|
263
|
+
for idx, var_name in enumerate(data_vars):
|
|
264
|
+
data_flat = np.asarray(ds[var_name].values).ravel()
|
|
265
|
+
if fill_value is not None:
|
|
266
|
+
valid_data = ~(np.isnan(data_flat) | (data_flat == fill_value))
|
|
267
|
+
else:
|
|
268
|
+
valid_data = ~np.isnan(data_flat)
|
|
269
|
+
values = np.full_like(data_flat, np.nan, dtype=np.float64)
|
|
270
|
+
values[valid_data] = _as_float_array(data_flat[valid_data])
|
|
271
|
+
gridded = _aggregate_to_grid(values_flat=values, binner=binner, aggregation=aggregation)
|
|
272
|
+
|
|
273
|
+
if idx == 0:
|
|
274
|
+
out = xr.Dataset(
|
|
275
|
+
data_vars={var_name: (["lat", "lon"], gridded)},
|
|
276
|
+
coords={"lat": ("lat", binner.lat), "lon": ("lon", binner.lon)},
|
|
277
|
+
)
|
|
278
|
+
else:
|
|
279
|
+
assert out is not None
|
|
280
|
+
out[var_name] = (["lat", "lon"], gridded)
|
|
281
|
+
|
|
282
|
+
assert out is not None
|
|
283
|
+
out.attrs.update(ds.attrs)
|
|
284
|
+
out.attrs["gridding_info"] = (
|
|
285
|
+
f"Data gridded to {grid_spacing}° resolution using {aggregation} aggregation"
|
|
286
|
+
)
|
|
287
|
+
return out
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""HEALPix (DGGS) gridding utilities for satellite data.
|
|
16
|
+
|
|
17
|
+
These helpers bin irregular ``(lat, lon, value)`` samples onto a HEALPix
|
|
18
|
+
discrete global grid system (DGGS) without interpolation. They are the
|
|
19
|
+
spherical sibling of :mod:`firecube.ingestor.extensions.grid`: where that module
|
|
20
|
+
bins onto a regular lat/lon grid, this one bins onto equal-area HEALPix cells.
|
|
21
|
+
Both share the binning/aggregation core in :mod:`._binning`.
|
|
22
|
+
|
|
23
|
+
Design notes
|
|
24
|
+
------------
|
|
25
|
+
- The lat/lon -> cell mapping is delegated to :mod:`healpix_geo` (built on the
|
|
26
|
+
``cdshealpix`` Rust crate). ``healpix_geo`` is the same backend ``xdggs`` uses,
|
|
27
|
+
so cells produced here align with ``xdggs``-decoded HEALPix cubes at the same
|
|
28
|
+
``depth`` and ``ellipsoid``.
|
|
29
|
+
- ``depth`` is the HEALPix order/level; the sphere has ``12 * 4**depth`` cells.
|
|
30
|
+
- The target cell axis can be **derived** from the input (the unique cells it
|
|
31
|
+
touches) or **supplied** via ``target_cells`` -- e.g. from :func:`cells_in_bbox`
|
|
32
|
+
-- so every granule/timestep lands on one fixed axis and results can be
|
|
33
|
+
appended. The result carries the absolute cell ids as a coordinate.
|
|
34
|
+
- Aggregations mean/min/max/first/last/median mirror the lat/lon module; ``any``
|
|
35
|
+
is added for boolean presence masks (e.g. "is any source pixel flagged here").
|
|
36
|
+
|
|
37
|
+
This module is only importable when the optional ``healpix`` extra is installed
|
|
38
|
+
(``pip install firecube[healpix]``).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
from dataclasses import dataclass
|
|
44
|
+
|
|
45
|
+
import numpy as np
|
|
46
|
+
import xarray as xr
|
|
47
|
+
|
|
48
|
+
from ._binning import aggregate_by_position, as_float_array, broadcast_latlon
|
|
49
|
+
|
|
50
|
+
DEFAULT_CELL_DIM = "cell"
|
|
51
|
+
DEFAULT_CELL_COORD = "cell_ids"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _require_healpix_geo() -> None:
|
|
55
|
+
"""Raise a helpful error if the optional ``healpix-geo`` dep is missing."""
|
|
56
|
+
try:
|
|
57
|
+
import healpix_geo.nested # noqa: F401
|
|
58
|
+
except ModuleNotFoundError as exc: # pragma: no cover - exercised via extra
|
|
59
|
+
raise ModuleNotFoundError(
|
|
60
|
+
"HEALPix regridding requires the optional 'healpix' extra. "
|
|
61
|
+
"Install it with: pip install 'firecube[healpix]'"
|
|
62
|
+
) from exc
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True, slots=True)
|
|
66
|
+
class HealpixBinner:
|
|
67
|
+
"""Precomputed bin mapping from an input lat/lon grid to HEALPix cells."""
|
|
68
|
+
|
|
69
|
+
cells: np.ndarray # 1D sorted target cell ids (uint64)
|
|
70
|
+
cell_position: np.ndarray # 1D index into ``cells`` per kept input point
|
|
71
|
+
valid_point_index: np.ndarray # 1D indices into the flattened input arrays
|
|
72
|
+
depth: int
|
|
73
|
+
indexing_scheme: str
|
|
74
|
+
ellipsoid: str
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def n_cells(self) -> int:
|
|
78
|
+
"""Number of cells on the target axis."""
|
|
79
|
+
return int(self.cells.size)
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def npix(self) -> int:
|
|
83
|
+
"""Total number of HEALPix cells on the sphere at this ``depth``."""
|
|
84
|
+
return 12 * (4**self.depth)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _mesh_step_for_depth(depth: int) -> float:
|
|
88
|
+
"""Half the mean HEALPix cell side (degrees) at ``depth``.
|
|
89
|
+
|
|
90
|
+
The mean cell side is ``sqrt(sphere_area / npix)`` which works out to about
|
|
91
|
+
``58.63 / 2**depth`` degrees; sampling at half that ensures each interior
|
|
92
|
+
cell of a region receives at least one sample.
|
|
93
|
+
"""
|
|
94
|
+
return (58.632 / (2**depth)) / 2.0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def cells_in_bbox(
|
|
98
|
+
*,
|
|
99
|
+
lon_min: float,
|
|
100
|
+
lat_min: float,
|
|
101
|
+
lon_max: float,
|
|
102
|
+
lat_max: float,
|
|
103
|
+
depth: int,
|
|
104
|
+
ellipsoid: str = "WGS84",
|
|
105
|
+
step: float | None = None,
|
|
106
|
+
) -> np.ndarray:
|
|
107
|
+
"""Return the sorted HEALPix cell ids covering a lon/lat bounding box.
|
|
108
|
+
|
|
109
|
+
Useful as the ``target_cells`` axis for a regional product: enumerate the
|
|
110
|
+
footprint once, then bin every granule/timestep onto it.
|
|
111
|
+
|
|
112
|
+
The box is sampled on a regular lon/lat mesh and each sample mapped to its
|
|
113
|
+
cell; the unique set is returned. ``step`` (degrees) defaults to half the
|
|
114
|
+
cell size at ``depth`` so every interior cell is hit. This handles boxes of
|
|
115
|
+
any size (HEALPix FOV-coverage queries are limited to ~90° per side).
|
|
116
|
+
"""
|
|
117
|
+
_require_healpix_geo()
|
|
118
|
+
from healpix_geo.nested import lonlat_to_healpix
|
|
119
|
+
|
|
120
|
+
if step is None:
|
|
121
|
+
step = _mesh_step_for_depth(depth)
|
|
122
|
+
if step <= 0:
|
|
123
|
+
raise ValueError("step must be > 0")
|
|
124
|
+
|
|
125
|
+
lons = np.arange(lon_min, lon_max + step, step)
|
|
126
|
+
lats = np.arange(lat_min, lat_max + step, step)
|
|
127
|
+
lon2d, lat2d = np.meshgrid(lons, lats)
|
|
128
|
+
cells = lonlat_to_healpix(lon2d.ravel(), lat2d.ravel(), depth, ellipsoid=ellipsoid)
|
|
129
|
+
return np.unique(np.asarray(cells, dtype=np.uint64))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def build_healpix_binner(
|
|
133
|
+
*,
|
|
134
|
+
lat: np.ndarray,
|
|
135
|
+
lon: np.ndarray,
|
|
136
|
+
depth: int,
|
|
137
|
+
indexing_scheme: str = "nested",
|
|
138
|
+
ellipsoid: str = "WGS84",
|
|
139
|
+
target_cells: np.ndarray | None = None,
|
|
140
|
+
) -> HealpixBinner:
|
|
141
|
+
"""Precompute the mapping from an input lat/lon grid to HEALPix cells.
|
|
142
|
+
|
|
143
|
+
Parameters
|
|
144
|
+
----------
|
|
145
|
+
lat, lon:
|
|
146
|
+
Latitude/longitude of the input samples, in degrees. Either 1-D
|
|
147
|
+
coordinate vectors (treated as a rectilinear grid) or matching N-D
|
|
148
|
+
arrays of point coordinates.
|
|
149
|
+
depth:
|
|
150
|
+
HEALPix order/level [0, 29]. Determines the cell size
|
|
151
|
+
(``12 * 4**depth`` cells on the sphere).
|
|
152
|
+
indexing_scheme:
|
|
153
|
+
HEALPix scheme. Only ``"nested"`` is supported.
|
|
154
|
+
ellipsoid:
|
|
155
|
+
Reference ellipsoid for the lat/lon -> cell mapping (e.g. ``"WGS84"``,
|
|
156
|
+
``"sphere"``). Two grids share cell ids only when ``depth`` and
|
|
157
|
+
``ellipsoid`` match.
|
|
158
|
+
target_cells:
|
|
159
|
+
Optional fixed cell axis (sorted unique cell ids). When given, the axis
|
|
160
|
+
is exactly these cells and input points outside them are dropped; when
|
|
161
|
+
``None``, the axis is derived as the unique cells the input touches.
|
|
162
|
+
"""
|
|
163
|
+
_require_healpix_geo()
|
|
164
|
+
from healpix_geo.nested import lonlat_to_healpix
|
|
165
|
+
|
|
166
|
+
if indexing_scheme != "nested":
|
|
167
|
+
raise ValueError(
|
|
168
|
+
f"Unsupported indexing_scheme: {indexing_scheme!r}. Only 'nested' is supported."
|
|
169
|
+
)
|
|
170
|
+
if not 0 <= depth <= 29:
|
|
171
|
+
raise ValueError(f"depth must be in [0, 29], got {depth}")
|
|
172
|
+
|
|
173
|
+
lat_flat, lon_flat, _ = broadcast_latlon(lat, lon, np.zeros_like(lat, dtype=np.float32))
|
|
174
|
+
lat_flat = as_float_array(lat_flat)
|
|
175
|
+
lon_flat = as_float_array(lon_flat)
|
|
176
|
+
|
|
177
|
+
valid_mask = ~(np.isnan(lat_flat) | np.isnan(lon_flat))
|
|
178
|
+
valid_point_index = np.flatnonzero(valid_mask)
|
|
179
|
+
if valid_point_index.size == 0:
|
|
180
|
+
raise ValueError("No valid lat/lon points found after filtering NaNs")
|
|
181
|
+
|
|
182
|
+
cell_ids = np.asarray(
|
|
183
|
+
lonlat_to_healpix(lon_flat[valid_mask], lat_flat[valid_mask], depth, ellipsoid=ellipsoid),
|
|
184
|
+
dtype=np.uint64,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
if target_cells is None:
|
|
188
|
+
# Derive the axis from the cells the input actually touches.
|
|
189
|
+
cells, cell_position = np.unique(cell_ids, return_inverse=True)
|
|
190
|
+
else:
|
|
191
|
+
# Bin onto a fixed, caller-supplied axis; drop points outside it.
|
|
192
|
+
cells = np.asarray(target_cells, dtype=np.uint64)
|
|
193
|
+
pos = np.searchsorted(cells, cell_ids)
|
|
194
|
+
np.clip(pos, 0, max(cells.size - 1, 0), out=pos)
|
|
195
|
+
in_target = (cells.size > 0) & (cells[pos] == cell_ids)
|
|
196
|
+
cell_position = pos[in_target]
|
|
197
|
+
valid_point_index = valid_point_index[in_target]
|
|
198
|
+
|
|
199
|
+
return HealpixBinner(
|
|
200
|
+
cells=cells,
|
|
201
|
+
cell_position=cell_position.astype(np.int64, copy=False),
|
|
202
|
+
valid_point_index=valid_point_index.astype(np.int64, copy=False),
|
|
203
|
+
depth=int(depth),
|
|
204
|
+
indexing_scheme=indexing_scheme,
|
|
205
|
+
ellipsoid=ellipsoid,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def regrid_with_binner(
|
|
210
|
+
*,
|
|
211
|
+
binner: HealpixBinner,
|
|
212
|
+
data: np.ndarray,
|
|
213
|
+
aggregation: str = "mean",
|
|
214
|
+
) -> np.ndarray:
|
|
215
|
+
"""Regrid a field onto the binner's HEALPix cells (1-D, aligned to ``cells``)."""
|
|
216
|
+
return aggregate_by_position(
|
|
217
|
+
values_flat=np.asarray(data),
|
|
218
|
+
valid_point_index=binner.valid_point_index,
|
|
219
|
+
position=binner.cell_position,
|
|
220
|
+
n_targets=binner.n_cells,
|
|
221
|
+
aggregation=aggregation,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _cell_coord_attrs(binner: HealpixBinner) -> dict[str, object]:
|
|
226
|
+
"""xdggs-compatible attributes so the result can be ``xdggs.decode``-d."""
|
|
227
|
+
return {
|
|
228
|
+
"grid_name": "healpix",
|
|
229
|
+
"level": binner.depth,
|
|
230
|
+
"indexing_scheme": binner.indexing_scheme,
|
|
231
|
+
"ellipsoid": binner.ellipsoid,
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _to_dataset(
|
|
236
|
+
*,
|
|
237
|
+
gridded: dict[str, np.ndarray],
|
|
238
|
+
binner: HealpixBinner,
|
|
239
|
+
cell_dim: str,
|
|
240
|
+
cell_coord: str,
|
|
241
|
+
) -> xr.Dataset:
|
|
242
|
+
ds = xr.Dataset(
|
|
243
|
+
data_vars={name: ((cell_dim,), values) for name, values in gridded.items()},
|
|
244
|
+
coords={cell_coord: ((cell_dim,), binner.cells)},
|
|
245
|
+
)
|
|
246
|
+
ds[cell_coord].attrs.update(_cell_coord_attrs(binner))
|
|
247
|
+
ds.attrs["n_cells"] = binner.n_cells
|
|
248
|
+
ds.attrs["npix"] = binner.npix
|
|
249
|
+
return ds
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def grid_data_to_healpix(
|
|
253
|
+
*,
|
|
254
|
+
lat: np.ndarray,
|
|
255
|
+
lon: np.ndarray,
|
|
256
|
+
data: np.ndarray,
|
|
257
|
+
depth: int,
|
|
258
|
+
variable_name: str = "value",
|
|
259
|
+
indexing_scheme: str = "nested",
|
|
260
|
+
ellipsoid: str = "WGS84",
|
|
261
|
+
aggregation: str = "mean",
|
|
262
|
+
fill_value: float | None = None,
|
|
263
|
+
target_cells: np.ndarray | None = None,
|
|
264
|
+
) -> xr.Dataset:
|
|
265
|
+
"""Grid irregular satellite data onto HEALPix cells (no interpolation)."""
|
|
266
|
+
binner = build_healpix_binner(
|
|
267
|
+
lat=lat,
|
|
268
|
+
lon=lon,
|
|
269
|
+
depth=depth,
|
|
270
|
+
indexing_scheme=indexing_scheme,
|
|
271
|
+
ellipsoid=ellipsoid,
|
|
272
|
+
target_cells=target_cells,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
_, _, data_flat = broadcast_latlon(lat, lon, data)
|
|
276
|
+
values = as_float_array(data_flat).copy()
|
|
277
|
+
if fill_value is not None:
|
|
278
|
+
values[values == fill_value] = np.nan
|
|
279
|
+
|
|
280
|
+
gridded = regrid_with_binner(binner=binner, data=values, aggregation=aggregation)
|
|
281
|
+
return _to_dataset(
|
|
282
|
+
gridded={variable_name: gridded},
|
|
283
|
+
binner=binner,
|
|
284
|
+
cell_dim=DEFAULT_CELL_DIM,
|
|
285
|
+
cell_coord=DEFAULT_CELL_COORD,
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def grid_xarray_dataset_to_healpix(
|
|
290
|
+
*,
|
|
291
|
+
ds: xr.Dataset,
|
|
292
|
+
lat_var: str,
|
|
293
|
+
lon_var: str,
|
|
294
|
+
data_vars: list[str],
|
|
295
|
+
depth: int,
|
|
296
|
+
indexing_scheme: str = "nested",
|
|
297
|
+
ellipsoid: str = "WGS84",
|
|
298
|
+
aggregation: str = "mean",
|
|
299
|
+
fill_value: float | None = None,
|
|
300
|
+
target_cells: np.ndarray | None = None,
|
|
301
|
+
cell_dim: str = DEFAULT_CELL_DIM,
|
|
302
|
+
cell_coord: str = DEFAULT_CELL_COORD,
|
|
303
|
+
) -> xr.Dataset:
|
|
304
|
+
"""Grid multiple variables from an xarray Dataset onto HEALPix cells."""
|
|
305
|
+
if not data_vars:
|
|
306
|
+
raise ValueError("data_vars must contain at least one variable name")
|
|
307
|
+
|
|
308
|
+
binner = build_healpix_binner(
|
|
309
|
+
lat=ds[lat_var].values,
|
|
310
|
+
lon=ds[lon_var].values,
|
|
311
|
+
depth=depth,
|
|
312
|
+
indexing_scheme=indexing_scheme,
|
|
313
|
+
ellipsoid=ellipsoid,
|
|
314
|
+
target_cells=target_cells,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
gridded: dict[str, np.ndarray] = {}
|
|
318
|
+
for var_name in data_vars:
|
|
319
|
+
values = as_float_array(np.asarray(ds[var_name].values)).ravel().copy()
|
|
320
|
+
if fill_value is not None:
|
|
321
|
+
values[values == fill_value] = np.nan
|
|
322
|
+
gridded[var_name] = regrid_with_binner(binner=binner, data=values, aggregation=aggregation)
|
|
323
|
+
|
|
324
|
+
out = _to_dataset(gridded=gridded, binner=binner, cell_dim=cell_dim, cell_coord=cell_coord)
|
|
325
|
+
out.attrs.update(ds.attrs)
|
|
326
|
+
out.attrs["gridding_info"] = (
|
|
327
|
+
f"Data gridded to HEALPix depth {depth} ({indexing_scheme}, {ellipsoid}) "
|
|
328
|
+
f"using {aggregation} aggregation"
|
|
329
|
+
)
|
|
330
|
+
return out
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Copyright 2025-2026 EUMETSAT
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|