firecube 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. firecube/__init__.py +23 -0
  2. firecube/__main__.py +20 -0
  3. firecube/_version.py +46 -0
  4. firecube/cli/_audience.py +135 -0
  5. firecube/cli/_command_schemas.py +112 -0
  6. firecube/cli/_ctx.py +85 -0
  7. firecube/cli/_errors.py +115 -0
  8. firecube/cli/_formatter.py +195 -0
  9. firecube/cli/_product.py +47 -0
  10. firecube/cli/_rename_hints.py +123 -0
  11. firecube/cli/_shared_options.py +247 -0
  12. firecube/cli/_slot_env.py +102 -0
  13. firecube/cli/_slot_planning.py +95 -0
  14. firecube/cli/_typed_options.py +181 -0
  15. firecube/cli/_uri_policy.py +110 -0
  16. firecube/cli/advise.py +321 -0
  17. firecube/cli/archive.py +870 -0
  18. firecube/cli/catalog.py +196 -0
  19. firecube/cli/chunks/__init__.py +74 -0
  20. firecube/cli/chunks/_claims.py +229 -0
  21. firecube/cli/chunks/_common.py +84 -0
  22. firecube/cli/chunks/_delete.py +376 -0
  23. firecube/cli/chunks/_list.py +334 -0
  24. firecube/cli/chunks/_manager.py +161 -0
  25. firecube/cli/chunks/_runs.py +294 -0
  26. firecube/cli/chunks/_snapshots.py +165 -0
  27. firecube/cli/completion.py +73 -0
  28. firecube/cli/main.py +866 -0
  29. firecube/cli/parquet.py +289 -0
  30. firecube/cli/plugins/__init__.py +28 -0
  31. firecube/cli/plugins/commands.py +262 -0
  32. firecube/cli/plugins/introspect.py +304 -0
  33. firecube/cli/plugins/mgmt.py +198 -0
  34. firecube/cli/plugins/registry.py +132 -0
  35. firecube/cli/zarr.py +1043 -0
  36. firecube/core/__init__.py +14 -0
  37. firecube/core/api.py +88 -0
  38. firecube/core/cf/__init__.py +23 -0
  39. firecube/core/cf/check_ids.py +126 -0
  40. firecube/core/cf/report.py +57 -0
  41. firecube/core/cf/validator.py +313 -0
  42. firecube/core/config.py +307 -0
  43. firecube/core/controlplane/__init__.py +47 -0
  44. firecube/core/controlplane/_event_processor.py +136 -0
  45. firecube/core/controlplane/_helpers.py +114 -0
  46. firecube/core/controlplane/_projection.py +253 -0
  47. firecube/core/controlplane/_snapshot.py +215 -0
  48. firecube/core/controlplane/_wal_reader.py +402 -0
  49. firecube/core/controlplane/_wal_writer.py +589 -0
  50. firecube/core/controlplane/claims.py +294 -0
  51. firecube/core/controlplane/deletion.py +704 -0
  52. firecube/core/controlplane/events.py +219 -0
  53. firecube/core/controlplane/manager.py +990 -0
  54. firecube/core/controlplane/metrics.py +97 -0
  55. firecube/core/controlplane/repo.py +804 -0
  56. firecube/core/controlplane/repo_utils.py +73 -0
  57. firecube/core/controlplane/time_dim.py +111 -0
  58. firecube/core/controlplane/types.py +445 -0
  59. firecube/core/credentials.py +36 -0
  60. firecube/core/duckdb/__init__.py +27 -0
  61. firecube/core/duckdb/bridge.py +82 -0
  62. firecube/core/duckdb/io.py +30 -0
  63. firecube/core/errors.py +74 -0
  64. firecube/core/filesystem/__init__.py +60 -0
  65. firecube/core/filesystem/_obstore_compat.py +60 -0
  66. firecube/core/filesystem/fsspec_backend.py +308 -0
  67. firecube/core/filesystem/instrumentation.py +260 -0
  68. firecube/core/filesystem/obstore_backend.py +406 -0
  69. firecube/core/filesystem/ops.py +412 -0
  70. firecube/core/filesystem/protocol.py +130 -0
  71. firecube/core/filesystem/store_factory.py +149 -0
  72. firecube/core/formats/__init__.py +46 -0
  73. firecube/core/formats/_iso.py +84 -0
  74. firecube/core/formats/discovery.py +165 -0
  75. firecube/core/formats/hdf5.py +137 -0
  76. firecube/core/formats/netcdf.py +226 -0
  77. firecube/core/formats/zip.py +192 -0
  78. firecube/core/intake.py +530 -0
  79. firecube/core/observability/__init__.py +72 -0
  80. firecube/core/observability/_state.py +24 -0
  81. firecube/core/observability/logging.py +147 -0
  82. firecube/core/observability/metrics.py +230 -0
  83. firecube/core/observability/telemetry/__init__.py +87 -0
  84. firecube/core/observability/telemetry/pushgateway.py +339 -0
  85. firecube/core/observability/telemetry/sinks.py +155 -0
  86. firecube/core/observability/tracing.py +126 -0
  87. firecube/core/product/__init__.py +45 -0
  88. firecube/core/product/identity.py +76 -0
  89. firecube/core/product/resolver.py +121 -0
  90. firecube/core/product/target.py +120 -0
  91. firecube/core/runtime.py +187 -0
  92. firecube/core/slot_index.py +205 -0
  93. firecube/core/storage/__init__.py +45 -0
  94. firecube/core/storage/binding.py +39 -0
  95. firecube/core/storage/cache_key.py +40 -0
  96. firecube/core/storage/completion.py +245 -0
  97. firecube/core/storage/driver_config.py +125 -0
  98. firecube/core/storage/results.py +30 -0
  99. firecube/core/storage/session.py +431 -0
  100. firecube/core/storage/transfer.py +142 -0
  101. firecube/core/storage/uri.py +173 -0
  102. firecube/core/tensogram/__init__.py +37 -0
  103. firecube/core/tensogram/_compat.py +36 -0
  104. firecube/core/tensogram/controlplane_codec.py +230 -0
  105. firecube/core/tensogram/converter.py +390 -0
  106. firecube/core/tensogram/metadata.py +274 -0
  107. firecube/core/tensogram/restore.py +303 -0
  108. firecube/core/tensogram/schema.py +93 -0
  109. firecube/core/uris.py +182 -0
  110. firecube/core/workspaces.py +68 -0
  111. firecube/core/zarr/__init__.py +69 -0
  112. firecube/core/zarr/_reserved_attrs.py +47 -0
  113. firecube/core/zarr/_reserved_root_attrs.py +74 -0
  114. firecube/core/zarr/io.py +88 -0
  115. firecube/core/zarr/layers.py +184 -0
  116. firecube/core/zarr/multires.py +178 -0
  117. firecube/core/zarr/region_writer.py +782 -0
  118. firecube/core/zarr/scrub.py +200 -0
  119. firecube/core/zarr/state.py +258 -0
  120. firecube/core/zarr/time_decode.py +71 -0
  121. firecube/core/zarr/validation.py +622 -0
  122. firecube/ingestor/__init__.py +27 -0
  123. firecube/ingestor/api.py +215 -0
  124. firecube/ingestor/config/__init__.py +19 -0
  125. firecube/ingestor/config/coercion.py +157 -0
  126. firecube/ingestor/config/engine.py +188 -0
  127. firecube/ingestor/contracts/__init__.py +14 -0
  128. firecube/ingestor/contracts/interfaces.py +235 -0
  129. firecube/ingestor/devtools/_templates/__init__.py +14 -0
  130. firecube/ingestor/devtools/_templates/ingestor_base.py.tpl +72 -0
  131. firecube/ingestor/devtools/_templates/ingestor_direct_zarr.py.tpl +66 -0
  132. firecube/ingestor/devtools/_templates/ingestor_parquet.py.tpl +59 -0
  133. firecube/ingestor/devtools/_templates/ingestor_zarr.py.tpl +55 -0
  134. firecube/ingestor/devtools/_templates/readme.md.tpl +40 -0
  135. firecube/ingestor/devtools/scaffolding.py +229 -0
  136. firecube/ingestor/errors.py +79 -0
  137. firecube/ingestor/extensions/_binning.py +152 -0
  138. firecube/ingestor/extensions/duck.py +153 -0
  139. firecube/ingestor/extensions/grid.py +287 -0
  140. firecube/ingestor/extensions/healpix.py +330 -0
  141. firecube/ingestor/registry/__init__.py +14 -0
  142. firecube/ingestor/registry/loader.py +144 -0
  143. firecube/ingestor/registry/metadata.py +97 -0
  144. firecube/ingestor/registry/version_compat.py +95 -0
  145. firecube/ingestor/runtime/__init__.py +14 -0
  146. firecube/ingestor/runtime/aggregation.py +112 -0
  147. firecube/ingestor/runtime/base.py +836 -0
  148. firecube/ingestor/runtime/base_hooks.py +129 -0
  149. firecube/ingestor/runtime/batching.py +161 -0
  150. firecube/ingestor/runtime/configure.py +195 -0
  151. firecube/ingestor/runtime/coverage.py +177 -0
  152. firecube/ingestor/runtime/engine.py +793 -0
  153. firecube/ingestor/runtime/parallel_evidence.py +75 -0
  154. firecube/ingestor/runtime/parallel_execution_state.py +28 -0
  155. firecube/ingestor/runtime/parallel_gate.py +164 -0
  156. firecube/ingestor/runtime/parallel_run_id.py +48 -0
  157. firecube/ingestor/runtime/preflight.py +40 -0
  158. firecube/ingestor/runtime/recording.py +425 -0
  159. firecube/ingestor/runtime/resume_guard.py +556 -0
  160. firecube/ingestor/runtime/resume_types.py +65 -0
  161. firecube/ingestor/runtime/telemetry.py +171 -0
  162. firecube/ingestor/runtime/tensogram/__init__.py +14 -0
  163. firecube/ingestor/runtime/tensogram/strategy.py +164 -0
  164. firecube/ingestor/runtime/workspace.py +220 -0
  165. firecube/ingestor/runtime/zarr/__init__.py +34 -0
  166. firecube/ingestor/runtime/zarr/append.py +466 -0
  167. firecube/ingestor/runtime/zarr/append_services.py +650 -0
  168. firecube/ingestor/runtime/zarr/batch_runner.py +190 -0
  169. firecube/ingestor/runtime/zarr/contracts.py +58 -0
  170. firecube/ingestor/runtime/zarr/existing_cube_check.py +282 -0
  171. firecube/ingestor/runtime/zarr/resume_cache.py +92 -0
  172. firecube/ingestor/runtime/zarr/staged_metadata.py +249 -0
  173. firecube/ingestor/runtime/zarr/strategies/__init__.py +15 -0
  174. firecube/ingestor/runtime/zarr/strategies/append.py +172 -0
  175. firecube/ingestor/runtime/zarr/strategies/indexed_region.py +426 -0
  176. firecube/ingestor/runtime/zarr/write.py +212 -0
  177. firecube/ingestor/runtime/zarr/write_context.py +139 -0
  178. firecube/ingestor/templates/__init__.py +19 -0
  179. firecube/ingestor/templates/config.py +118 -0
  180. firecube/ingestor/templates/direct_zarr.py +790 -0
  181. firecube/ingestor/templates/generic.py +462 -0
  182. firecube/ingestor/templates/generic_tensogram.py +170 -0
  183. firecube/ingestor/types/__init__.py +19 -0
  184. firecube/ingestor/types/config.py +89 -0
  185. firecube/ingestor/types/context.py +409 -0
  186. firecube/ingestor/types/manifest.py +53 -0
  187. firecube/ingestor/types/planned_range.py +152 -0
  188. firecube/ingestor/types/result_metrics.py +386 -0
  189. firecube/ingestor/utils/__init__.py +14 -0
  190. firecube/ingestor/utils/duckdb_utils.py +235 -0
  191. firecube/ingestor/validation.py +55 -0
  192. firecube-0.1.4.dist-info/METADATA +242 -0
  193. firecube-0.1.4.dist-info/RECORD +197 -0
  194. firecube-0.1.4.dist-info/WHEEL +4 -0
  195. firecube-0.1.4.dist-info/entry_points.txt +2 -0
  196. firecube-0.1.4.dist-info/licenses/AUTHORS.md +5 -0
  197. firecube-0.1.4.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,287 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ Gridding utilities for satellite data.
17
+
18
+ These helpers bin irregular (lat, lon, value) samples onto a regular lat/lon
19
+ grid without interpolation. This is intended for curvilinear geolocation
20
+ grids (2D lat/lon) such as MSG products.
21
+
22
+ Design notes
23
+ ------------
24
+ - The target grid extent can be **derived** from the data
25
+ (``arange(floor(min), ceil(max) + spacing, spacing)``) or **supplied** via
26
+ ``bounds``. Supplying it pins one fixed grid so many granules/timesteps share
27
+ a coordinate and can be appended, instead of each input deriving its own.
28
+ - Output coordinates are *centers* aligned to full degrees. Binning uses
29
+ half-cell edges around centers, so all centers correspond to a real cell.
30
+ This fixes the common off-by-one and "extra empty last row/col" issues that
31
+ arise when treating centers as edges.
32
+ - The binning/aggregation core is shared with :mod:`.healpix` via
33
+ :mod:`._binning`. Performance: mean/min/max/first/last/any are vectorized;
34
+ median is supported but may be expensive on large grids.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ from dataclasses import dataclass
40
+
41
+ import numpy as np
42
+ import xarray as xr
43
+
44
+ from ._binning import aggregate_by_position
45
+ from ._binning import as_float_array as _as_float_array
46
+ from ._binning import broadcast_latlon as _broadcast_latlon
47
+
48
+
49
+ @dataclass(frozen=True, slots=True)
50
+ class LatLonBinner:
51
+ """Precomputed bin mapping from an input grid to a regular lat/lon grid."""
52
+
53
+ lat: np.ndarray # 1D centers
54
+ lon: np.ndarray # 1D centers
55
+ cell_index: np.ndarray # 1D flattened cell indices (len == n_valid_points)
56
+ valid_point_index: np.ndarray # 1D indices into flattened input arrays
57
+ nlat: int
58
+ nlon: int
59
+ spacing: float
60
+
61
+
62
+ def build_latlon_binner(
63
+ *,
64
+ lat: np.ndarray,
65
+ lon: np.ndarray,
66
+ grid_spacing: float,
67
+ bounds: tuple[float, float, float, float] | None = None,
68
+ fill_value: float | None = None,
69
+ ) -> LatLonBinner:
70
+ """Precompute the mapping from an input lat/lon grid to regular bins.
71
+
72
+ Parameters
73
+ ----------
74
+ lat, lon:
75
+ Input geolocation, 1-D coordinate vectors or matching N-D arrays.
76
+ grid_spacing:
77
+ Target cell size in degrees (> 0).
78
+ bounds:
79
+ Optional ``(lat_min, lat_max, lon_min, lon_max)`` fixing the output
80
+ grid extent. When given, the axis is exactly these bounds and input
81
+ points outside them are dropped -- pin one set of bounds to reuse the
82
+ same grid across granules. When ``None``, the extent is derived from
83
+ the data (``floor(min)``/``ceil(max)``).
84
+ fill_value:
85
+ Accepted for signature parity; data masking is applied by callers.
86
+ """
87
+ if grid_spacing <= 0:
88
+ raise ValueError("grid_spacing must be > 0")
89
+
90
+ lat_flat, lon_flat, _data_dummy = _broadcast_latlon(
91
+ lat, lon, np.zeros_like(lat, dtype=np.float32)
92
+ )
93
+ lat_flat = _as_float_array(lat_flat)
94
+ lon_flat = _as_float_array(lon_flat)
95
+
96
+ valid_mask = ~(np.isnan(lat_flat) | np.isnan(lon_flat))
97
+
98
+ lat_valid = lat_flat[valid_mask]
99
+ lon_valid = lon_flat[valid_mask]
100
+ valid_point_index = np.flatnonzero(valid_mask)
101
+
102
+ if lat_valid.size == 0:
103
+ raise ValueError("No valid lat/lon points found after filtering NaNs")
104
+
105
+ if bounds is not None:
106
+ lat_min, lat_max, lon_min, lon_max = (float(b) for b in bounds)
107
+ else:
108
+ lat_min = float(np.floor(np.nanmin(lat_valid)))
109
+ lat_max = float(np.ceil(np.nanmax(lat_valid)))
110
+ lon_min = float(np.floor(np.nanmin(lon_valid)))
111
+ lon_max = float(np.ceil(np.nanmax(lon_valid)))
112
+
113
+ lat_centers = np.arange(lat_min, lat_max + grid_spacing, grid_spacing, dtype=np.float64)
114
+ lon_centers = np.arange(lon_min, lon_max + grid_spacing, grid_spacing, dtype=np.float64)
115
+
116
+ # Treat centers as true cell centers; edges are half a spacing away.
117
+ lat0 = lat_centers[0] - (grid_spacing / 2.0)
118
+ lon0 = lon_centers[0] - (grid_spacing / 2.0)
119
+
120
+ lat_idx = np.floor((lat_valid - lat0) / grid_spacing).astype(np.int64, copy=False)
121
+ lon_idx = np.floor((lon_valid - lon0) / grid_spacing).astype(np.int64, copy=False)
122
+
123
+ nlat = int(lat_centers.size)
124
+ nlon = int(lon_centers.size)
125
+ in_bounds = (lat_idx >= 0) & (lat_idx < nlat) & (lon_idx >= 0) & (lon_idx < nlon)
126
+
127
+ lat_idx = lat_idx[in_bounds]
128
+ lon_idx = lon_idx[in_bounds]
129
+ valid_point_index = valid_point_index[in_bounds]
130
+
131
+ cell_index = (lat_idx * nlon + lon_idx).astype(np.int64, copy=False)
132
+
133
+ return LatLonBinner(
134
+ lat=lat_centers.astype(np.float32),
135
+ lon=lon_centers.astype(np.float32),
136
+ cell_index=cell_index,
137
+ valid_point_index=valid_point_index.astype(np.int64, copy=False),
138
+ nlat=nlat,
139
+ nlon=nlon,
140
+ spacing=float(grid_spacing),
141
+ )
142
+
143
+
144
+ def _aggregate_to_grid(
145
+ *,
146
+ values_flat: np.ndarray,
147
+ binner: LatLonBinner,
148
+ aggregation: str,
149
+ ) -> np.ndarray:
150
+ """Aggregate flattened values into a (lat, lon) grid using a precomputed binner."""
151
+ flat = aggregate_by_position(
152
+ values_flat=values_flat,
153
+ valid_point_index=binner.valid_point_index,
154
+ position=binner.cell_index,
155
+ n_targets=binner.nlat * binner.nlon,
156
+ aggregation=aggregation,
157
+ )
158
+ return flat.reshape((binner.nlat, binner.nlon))
159
+
160
+
161
+ def regrid_with_binner(
162
+ *,
163
+ binner: LatLonBinner,
164
+ data: np.ndarray,
165
+ aggregation: str = "mean",
166
+ ) -> np.ndarray:
167
+ """Regrid a 2D field onto the binner's regular grid.
168
+
169
+ Parameters
170
+ ----------
171
+ binner:
172
+ Precomputed LatLonBinner mapping for the input geolocation grid.
173
+ data:
174
+ 2D array matching the binner's input grid shape.
175
+ aggregation:
176
+ Aggregation method for multiple source pixels per target cell.
177
+ """
178
+ values_flat = np.asarray(data).ravel()
179
+ return _aggregate_to_grid(values_flat=values_flat, binner=binner, aggregation=aggregation)
180
+
181
+
182
+ def grid_data_to_latlon(
183
+ *,
184
+ lat: np.ndarray,
185
+ lon: np.ndarray,
186
+ data: np.ndarray,
187
+ grid_spacing: float = 0.1,
188
+ variable_name: str = "value",
189
+ fill_value: float | None = None,
190
+ aggregation: str = "mean",
191
+ bounds: tuple[float, float, float, float] | None = None,
192
+ ) -> xr.Dataset:
193
+ """Grid irregular satellite data onto a regular lat/lon grid (no interpolation)."""
194
+ _lat_flat, _lon_flat, data_flat = _broadcast_latlon(lat, lon, data)
195
+
196
+ # Determine which data values are valid (coordinates handled by binner).
197
+ if fill_value is not None:
198
+ valid_data = ~(np.isnan(data_flat) | (data_flat == fill_value))
199
+ else:
200
+ valid_data = ~np.isnan(data_flat)
201
+
202
+ if valid_data.sum() == 0:
203
+ raise ValueError("No valid data points found after filtering invalid values")
204
+
205
+ binner = build_latlon_binner(
206
+ lat=lat, lon=lon, grid_spacing=grid_spacing, bounds=bounds, fill_value=fill_value
207
+ )
208
+ # Apply data mask on top of binner's valid coordinate mask.
209
+ values = np.full_like(data_flat, np.nan, dtype=np.float64)
210
+ values[valid_data] = _as_float_array(data_flat[valid_data])
211
+
212
+ data_gridded = _aggregate_to_grid(values_flat=values, binner=binner, aggregation=aggregation)
213
+
214
+ n_valid_cells = int(np.sum(~np.isnan(data_gridded)))
215
+ n_total_cells = int(data_gridded.size)
216
+
217
+ ds = xr.Dataset(
218
+ data_vars={variable_name: (["lat", "lon"], data_gridded)},
219
+ coords={
220
+ "lat": ("lat", binner.lat),
221
+ "lon": ("lon", binner.lon),
222
+ },
223
+ attrs={
224
+ "title": f"{variable_name} on regular lat/lon grid",
225
+ "description": f"Gridded from irregular data using {aggregation} aggregation (no interpolation)",
226
+ "grid_spacing": f"{grid_spacing}°",
227
+ "aggregation_method": str(aggregation),
228
+ "n_valid_cells": n_valid_cells,
229
+ "n_total_cells": n_total_cells,
230
+ "fill_percentage": float(n_valid_cells / n_total_cells * 100.0)
231
+ if n_total_cells
232
+ else 0.0,
233
+ "lat_range": f"({float(binner.lat.min()):.2f}, {float(binner.lat.max()):.2f})",
234
+ "lon_range": f"({float(binner.lon.min()):.2f}, {float(binner.lon.max()):.2f})",
235
+ },
236
+ )
237
+
238
+ return ds
239
+
240
+
241
+ def grid_xarray_dataset(
242
+ *,
243
+ ds: xr.Dataset,
244
+ lat_var: str,
245
+ lon_var: str,
246
+ data_vars: list[str],
247
+ grid_spacing: float = 0.1,
248
+ fill_value: float | None = None,
249
+ aggregation: str = "mean",
250
+ bounds: tuple[float, float, float, float] | None = None,
251
+ ) -> xr.Dataset:
252
+ """Grid multiple variables from an xarray Dataset onto a regular lat/lon grid."""
253
+ if not data_vars:
254
+ raise ValueError("data_vars must contain at least one variable name")
255
+
256
+ lat = ds[lat_var].values
257
+ lon = ds[lon_var].values
258
+ binner = build_latlon_binner(
259
+ lat=lat, lon=lon, grid_spacing=grid_spacing, bounds=bounds, fill_value=fill_value
260
+ )
261
+
262
+ out: xr.Dataset | None = None
263
+ for idx, var_name in enumerate(data_vars):
264
+ data_flat = np.asarray(ds[var_name].values).ravel()
265
+ if fill_value is not None:
266
+ valid_data = ~(np.isnan(data_flat) | (data_flat == fill_value))
267
+ else:
268
+ valid_data = ~np.isnan(data_flat)
269
+ values = np.full_like(data_flat, np.nan, dtype=np.float64)
270
+ values[valid_data] = _as_float_array(data_flat[valid_data])
271
+ gridded = _aggregate_to_grid(values_flat=values, binner=binner, aggregation=aggregation)
272
+
273
+ if idx == 0:
274
+ out = xr.Dataset(
275
+ data_vars={var_name: (["lat", "lon"], gridded)},
276
+ coords={"lat": ("lat", binner.lat), "lon": ("lon", binner.lon)},
277
+ )
278
+ else:
279
+ assert out is not None
280
+ out[var_name] = (["lat", "lon"], gridded)
281
+
282
+ assert out is not None
283
+ out.attrs.update(ds.attrs)
284
+ out.attrs["gridding_info"] = (
285
+ f"Data gridded to {grid_spacing}° resolution using {aggregation} aggregation"
286
+ )
287
+ return out
@@ -0,0 +1,330 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """HEALPix (DGGS) gridding utilities for satellite data.
16
+
17
+ These helpers bin irregular ``(lat, lon, value)`` samples onto a HEALPix
18
+ discrete global grid system (DGGS) without interpolation. They are the
19
+ spherical sibling of :mod:`firecube.ingestor.extensions.grid`: where that module
20
+ bins onto a regular lat/lon grid, this one bins onto equal-area HEALPix cells.
21
+ Both share the binning/aggregation core in :mod:`._binning`.
22
+
23
+ Design notes
24
+ ------------
25
+ - The lat/lon -> cell mapping is delegated to :mod:`healpix_geo` (built on the
26
+ ``cdshealpix`` Rust crate). ``healpix_geo`` is the same backend ``xdggs`` uses,
27
+ so cells produced here align with ``xdggs``-decoded HEALPix cubes at the same
28
+ ``depth`` and ``ellipsoid``.
29
+ - ``depth`` is the HEALPix order/level; the sphere has ``12 * 4**depth`` cells.
30
+ - The target cell axis can be **derived** from the input (the unique cells it
31
+ touches) or **supplied** via ``target_cells`` -- e.g. from :func:`cells_in_bbox`
32
+ -- so every granule/timestep lands on one fixed axis and results can be
33
+ appended. The result carries the absolute cell ids as a coordinate.
34
+ - Aggregations mean/min/max/first/last/median mirror the lat/lon module; ``any``
35
+ is added for boolean presence masks (e.g. "is any source pixel flagged here").
36
+
37
+ This module is only importable when the optional ``healpix`` extra is installed
38
+ (``pip install firecube[healpix]``).
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ from dataclasses import dataclass
44
+
45
+ import numpy as np
46
+ import xarray as xr
47
+
48
+ from ._binning import aggregate_by_position, as_float_array, broadcast_latlon
49
+
50
+ DEFAULT_CELL_DIM = "cell"
51
+ DEFAULT_CELL_COORD = "cell_ids"
52
+
53
+
54
+ def _require_healpix_geo() -> None:
55
+ """Raise a helpful error if the optional ``healpix-geo`` dep is missing."""
56
+ try:
57
+ import healpix_geo.nested # noqa: F401
58
+ except ModuleNotFoundError as exc: # pragma: no cover - exercised via extra
59
+ raise ModuleNotFoundError(
60
+ "HEALPix regridding requires the optional 'healpix' extra. "
61
+ "Install it with: pip install 'firecube[healpix]'"
62
+ ) from exc
63
+
64
+
65
+ @dataclass(frozen=True, slots=True)
66
+ class HealpixBinner:
67
+ """Precomputed bin mapping from an input lat/lon grid to HEALPix cells."""
68
+
69
+ cells: np.ndarray # 1D sorted target cell ids (uint64)
70
+ cell_position: np.ndarray # 1D index into ``cells`` per kept input point
71
+ valid_point_index: np.ndarray # 1D indices into the flattened input arrays
72
+ depth: int
73
+ indexing_scheme: str
74
+ ellipsoid: str
75
+
76
+ @property
77
+ def n_cells(self) -> int:
78
+ """Number of cells on the target axis."""
79
+ return int(self.cells.size)
80
+
81
+ @property
82
+ def npix(self) -> int:
83
+ """Total number of HEALPix cells on the sphere at this ``depth``."""
84
+ return 12 * (4**self.depth)
85
+
86
+
87
+ def _mesh_step_for_depth(depth: int) -> float:
88
+ """Half the mean HEALPix cell side (degrees) at ``depth``.
89
+
90
+ The mean cell side is ``sqrt(sphere_area / npix)`` which works out to about
91
+ ``58.63 / 2**depth`` degrees; sampling at half that ensures each interior
92
+ cell of a region receives at least one sample.
93
+ """
94
+ return (58.632 / (2**depth)) / 2.0
95
+
96
+
97
+ def cells_in_bbox(
98
+ *,
99
+ lon_min: float,
100
+ lat_min: float,
101
+ lon_max: float,
102
+ lat_max: float,
103
+ depth: int,
104
+ ellipsoid: str = "WGS84",
105
+ step: float | None = None,
106
+ ) -> np.ndarray:
107
+ """Return the sorted HEALPix cell ids covering a lon/lat bounding box.
108
+
109
+ Useful as the ``target_cells`` axis for a regional product: enumerate the
110
+ footprint once, then bin every granule/timestep onto it.
111
+
112
+ The box is sampled on a regular lon/lat mesh and each sample mapped to its
113
+ cell; the unique set is returned. ``step`` (degrees) defaults to half the
114
+ cell size at ``depth`` so every interior cell is hit. This handles boxes of
115
+ any size (HEALPix FOV-coverage queries are limited to ~90° per side).
116
+ """
117
+ _require_healpix_geo()
118
+ from healpix_geo.nested import lonlat_to_healpix
119
+
120
+ if step is None:
121
+ step = _mesh_step_for_depth(depth)
122
+ if step <= 0:
123
+ raise ValueError("step must be > 0")
124
+
125
+ lons = np.arange(lon_min, lon_max + step, step)
126
+ lats = np.arange(lat_min, lat_max + step, step)
127
+ lon2d, lat2d = np.meshgrid(lons, lats)
128
+ cells = lonlat_to_healpix(lon2d.ravel(), lat2d.ravel(), depth, ellipsoid=ellipsoid)
129
+ return np.unique(np.asarray(cells, dtype=np.uint64))
130
+
131
+
132
+ def build_healpix_binner(
133
+ *,
134
+ lat: np.ndarray,
135
+ lon: np.ndarray,
136
+ depth: int,
137
+ indexing_scheme: str = "nested",
138
+ ellipsoid: str = "WGS84",
139
+ target_cells: np.ndarray | None = None,
140
+ ) -> HealpixBinner:
141
+ """Precompute the mapping from an input lat/lon grid to HEALPix cells.
142
+
143
+ Parameters
144
+ ----------
145
+ lat, lon:
146
+ Latitude/longitude of the input samples, in degrees. Either 1-D
147
+ coordinate vectors (treated as a rectilinear grid) or matching N-D
148
+ arrays of point coordinates.
149
+ depth:
150
+ HEALPix order/level [0, 29]. Determines the cell size
151
+ (``12 * 4**depth`` cells on the sphere).
152
+ indexing_scheme:
153
+ HEALPix scheme. Only ``"nested"`` is supported.
154
+ ellipsoid:
155
+ Reference ellipsoid for the lat/lon -> cell mapping (e.g. ``"WGS84"``,
156
+ ``"sphere"``). Two grids share cell ids only when ``depth`` and
157
+ ``ellipsoid`` match.
158
+ target_cells:
159
+ Optional fixed cell axis (sorted unique cell ids). When given, the axis
160
+ is exactly these cells and input points outside them are dropped; when
161
+ ``None``, the axis is derived as the unique cells the input touches.
162
+ """
163
+ _require_healpix_geo()
164
+ from healpix_geo.nested import lonlat_to_healpix
165
+
166
+ if indexing_scheme != "nested":
167
+ raise ValueError(
168
+ f"Unsupported indexing_scheme: {indexing_scheme!r}. Only 'nested' is supported."
169
+ )
170
+ if not 0 <= depth <= 29:
171
+ raise ValueError(f"depth must be in [0, 29], got {depth}")
172
+
173
+ lat_flat, lon_flat, _ = broadcast_latlon(lat, lon, np.zeros_like(lat, dtype=np.float32))
174
+ lat_flat = as_float_array(lat_flat)
175
+ lon_flat = as_float_array(lon_flat)
176
+
177
+ valid_mask = ~(np.isnan(lat_flat) | np.isnan(lon_flat))
178
+ valid_point_index = np.flatnonzero(valid_mask)
179
+ if valid_point_index.size == 0:
180
+ raise ValueError("No valid lat/lon points found after filtering NaNs")
181
+
182
+ cell_ids = np.asarray(
183
+ lonlat_to_healpix(lon_flat[valid_mask], lat_flat[valid_mask], depth, ellipsoid=ellipsoid),
184
+ dtype=np.uint64,
185
+ )
186
+
187
+ if target_cells is None:
188
+ # Derive the axis from the cells the input actually touches.
189
+ cells, cell_position = np.unique(cell_ids, return_inverse=True)
190
+ else:
191
+ # Bin onto a fixed, caller-supplied axis; drop points outside it.
192
+ cells = np.asarray(target_cells, dtype=np.uint64)
193
+ pos = np.searchsorted(cells, cell_ids)
194
+ np.clip(pos, 0, max(cells.size - 1, 0), out=pos)
195
+ in_target = (cells.size > 0) & (cells[pos] == cell_ids)
196
+ cell_position = pos[in_target]
197
+ valid_point_index = valid_point_index[in_target]
198
+
199
+ return HealpixBinner(
200
+ cells=cells,
201
+ cell_position=cell_position.astype(np.int64, copy=False),
202
+ valid_point_index=valid_point_index.astype(np.int64, copy=False),
203
+ depth=int(depth),
204
+ indexing_scheme=indexing_scheme,
205
+ ellipsoid=ellipsoid,
206
+ )
207
+
208
+
209
+ def regrid_with_binner(
210
+ *,
211
+ binner: HealpixBinner,
212
+ data: np.ndarray,
213
+ aggregation: str = "mean",
214
+ ) -> np.ndarray:
215
+ """Regrid a field onto the binner's HEALPix cells (1-D, aligned to ``cells``)."""
216
+ return aggregate_by_position(
217
+ values_flat=np.asarray(data),
218
+ valid_point_index=binner.valid_point_index,
219
+ position=binner.cell_position,
220
+ n_targets=binner.n_cells,
221
+ aggregation=aggregation,
222
+ )
223
+
224
+
225
+ def _cell_coord_attrs(binner: HealpixBinner) -> dict[str, object]:
226
+ """xdggs-compatible attributes so the result can be ``xdggs.decode``-d."""
227
+ return {
228
+ "grid_name": "healpix",
229
+ "level": binner.depth,
230
+ "indexing_scheme": binner.indexing_scheme,
231
+ "ellipsoid": binner.ellipsoid,
232
+ }
233
+
234
+
235
+ def _to_dataset(
236
+ *,
237
+ gridded: dict[str, np.ndarray],
238
+ binner: HealpixBinner,
239
+ cell_dim: str,
240
+ cell_coord: str,
241
+ ) -> xr.Dataset:
242
+ ds = xr.Dataset(
243
+ data_vars={name: ((cell_dim,), values) for name, values in gridded.items()},
244
+ coords={cell_coord: ((cell_dim,), binner.cells)},
245
+ )
246
+ ds[cell_coord].attrs.update(_cell_coord_attrs(binner))
247
+ ds.attrs["n_cells"] = binner.n_cells
248
+ ds.attrs["npix"] = binner.npix
249
+ return ds
250
+
251
+
252
+ def grid_data_to_healpix(
253
+ *,
254
+ lat: np.ndarray,
255
+ lon: np.ndarray,
256
+ data: np.ndarray,
257
+ depth: int,
258
+ variable_name: str = "value",
259
+ indexing_scheme: str = "nested",
260
+ ellipsoid: str = "WGS84",
261
+ aggregation: str = "mean",
262
+ fill_value: float | None = None,
263
+ target_cells: np.ndarray | None = None,
264
+ ) -> xr.Dataset:
265
+ """Grid irregular satellite data onto HEALPix cells (no interpolation)."""
266
+ binner = build_healpix_binner(
267
+ lat=lat,
268
+ lon=lon,
269
+ depth=depth,
270
+ indexing_scheme=indexing_scheme,
271
+ ellipsoid=ellipsoid,
272
+ target_cells=target_cells,
273
+ )
274
+
275
+ _, _, data_flat = broadcast_latlon(lat, lon, data)
276
+ values = as_float_array(data_flat).copy()
277
+ if fill_value is not None:
278
+ values[values == fill_value] = np.nan
279
+
280
+ gridded = regrid_with_binner(binner=binner, data=values, aggregation=aggregation)
281
+ return _to_dataset(
282
+ gridded={variable_name: gridded},
283
+ binner=binner,
284
+ cell_dim=DEFAULT_CELL_DIM,
285
+ cell_coord=DEFAULT_CELL_COORD,
286
+ )
287
+
288
+
289
+ def grid_xarray_dataset_to_healpix(
290
+ *,
291
+ ds: xr.Dataset,
292
+ lat_var: str,
293
+ lon_var: str,
294
+ data_vars: list[str],
295
+ depth: int,
296
+ indexing_scheme: str = "nested",
297
+ ellipsoid: str = "WGS84",
298
+ aggregation: str = "mean",
299
+ fill_value: float | None = None,
300
+ target_cells: np.ndarray | None = None,
301
+ cell_dim: str = DEFAULT_CELL_DIM,
302
+ cell_coord: str = DEFAULT_CELL_COORD,
303
+ ) -> xr.Dataset:
304
+ """Grid multiple variables from an xarray Dataset onto HEALPix cells."""
305
+ if not data_vars:
306
+ raise ValueError("data_vars must contain at least one variable name")
307
+
308
+ binner = build_healpix_binner(
309
+ lat=ds[lat_var].values,
310
+ lon=ds[lon_var].values,
311
+ depth=depth,
312
+ indexing_scheme=indexing_scheme,
313
+ ellipsoid=ellipsoid,
314
+ target_cells=target_cells,
315
+ )
316
+
317
+ gridded: dict[str, np.ndarray] = {}
318
+ for var_name in data_vars:
319
+ values = as_float_array(np.asarray(ds[var_name].values)).ravel().copy()
320
+ if fill_value is not None:
321
+ values[values == fill_value] = np.nan
322
+ gridded[var_name] = regrid_with_binner(binner=binner, data=values, aggregation=aggregation)
323
+
324
+ out = _to_dataset(gridded=gridded, binner=binner, cell_dim=cell_dim, cell_coord=cell_coord)
325
+ out.attrs.update(ds.attrs)
326
+ out.attrs["gridding_info"] = (
327
+ f"Data gridded to HEALPix depth {depth} ({indexing_scheme}, {ellipsoid}) "
328
+ f"using {aggregation} aggregation"
329
+ )
330
+ return out
@@ -0,0 +1,14 @@
1
+ # Copyright 2025-2026 EUMETSAT
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+