obpg-writel2 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,85 @@
1
+ Metadata-Version: 2.4
2
+ Name: obpg-writel2
3
+ Version: 1.0.0
4
+ Summary: Helpers for reading L2 metadata and writing derived L2 NetCDF4 products
5
+ Author-email: Gwyn Fireman <Gwyn.F.Fireman@nasa.gov>, Claude Sonnet 5 <noreply@anthropic.com>
6
+ Requires-Python: >=3.12
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: numpy>=1.26.4
9
+ Requires-Dist: h5netcdf>=1.8.1
10
+
11
+ # writel2
12
+
13
+ Helpers for writing standard-layout L2 NetCDF4 ocean color products from
14
+ Python, and for reading metadata back out of an existing L2 file.
15
+
16
+ - `writel2.L2Writer` creates the standard l2gen group layout
17
+ (`sensor_band_parameters`, `scan_line_attributes`, `geophysical_data`,
18
+ `navigation_data`, `processing_control`/`input_parameters`
19
+ [/`flag_percentages`]), preloads the static global attributes, and writes
20
+ variables into any of those groups via `write_product(name, data,
21
+ group=...)`, whose attributes (`long_name`, `units`, `valid_min`/`valid_max`,
22
+ `scale_factor`/`add_offset`, `_FillValue`, `flag_values`/`flag_meanings`,
23
+ `reference`, `comment`, ...) are resolved from `share/common/product.xml` -
24
+ the same metadata l2gen itself uses, regardless of which group a variable
25
+ belongs to.
26
+ - `writel2.L2Reader` is a read-only helper for pulling metadata back out of
27
+ an existing L2 (or L1/L3) file, mainly so `L2Writer` can seed a derived
28
+ product's global attributes from its source file.
29
+ - `writel2.ProductTable` resolves a product name (e.g. `"chlor_a"`,
30
+ `"Rrs_443"`) against `product.xml` into a `ProductInfo` record.
31
+ - `writel2.GlobalMetadata` loads the static, institution-wide global
32
+ attributes (license, naming authority, creator/publisher info, ...) from
33
+ `$OCDATAROOT/common/global_metadata.json`.
34
+
35
+ ## Install
36
+
37
+ ```bash
38
+ pip install $OCSSWROOT/src/scripts/writel2
39
+ ```
40
+
41
+ This installs as `obpg-writel2`, importable as `writel2`.
42
+
43
+ ## Usage
44
+
45
+ See [`examples/demo_write_l2.py`](examples/demo_write_l2.py) for a complete,
46
+ runnable example that writes a small file touching every group:
47
+
48
+ ```bash
49
+ python $OCSSWROOT/src/scripts/writel2/examples/demo_write_l2.py /tmp/demo_l2.nc
50
+ ```
51
+
52
+ The short version:
53
+
54
+ ```python
55
+ import numpy as np
56
+ from writel2 import L2Writer
57
+
58
+ with L2Writer("out.L2.nc", title="Demo Level-2 Data") as writer:
59
+ # geophysical_data: resolved against product.xml for long_name, units,
60
+ # valid_min/valid_max, scale_factor/add_offset, _FillValue, ...
61
+ flags = np.zeros((4, 3), dtype=np.int32)
62
+ writer.write_product("l2_flags", flags)
63
+
64
+ # processing_control/input_parameters and /flag_percentages
65
+ writer.set_input_parameters({"ifile": "input.L1B.nc", "l2prod": "l2_flags"})
66
+ writer.set_flag_percentages({"ATMFAIL": 0.0, "LAND": 0.49})
67
+ ```
68
+
69
+ Datatype and attributes for any group's variables are resolved from
70
+ `product.xml` and applied automatically by `write_product()`, including
71
+ appropriate scaling - pass `group=` to target `sensor_band_parameters`,
72
+ `scan_line_attributes`, or `navigation_data` instead of the default
73
+ `geophysical_data`. The one exception is `geophysical_data`'s own local
74
+ `wavelength` coordinate variable (used alongside 3D products), which l2gen
75
+ writes directly rather than resolving against product.xml; the calling
76
+ script must set its datatype and attributes manually. See the demo script
77
+ for the full pattern, including how to register dimensions shared across
78
+ groups.
79
+
80
+ ## Tests
81
+
82
+ ```bash
83
+ pip install $OCSSWROOT/src/scripts/writel2
84
+ python -m unittest discover -s $OCSSWROOT/src/scripts/writel2/tests
85
+ ```
@@ -0,0 +1,9 @@
1
+ writel2/__init__.py,sha256=t6pI7EVsXP0dXtceoSXcfZkvP0XZdtdAh6VsXAvQD48,312
2
+ writel2/global_metadata.py,sha256=rhFi2UfgjZAfACOMzJJV_qsGQGD_s0DCAV7QN7QCw3U,1547
3
+ writel2/product_info.py,sha256=mGp54mgGbiTgADkUio71b6y4UVl9SElPQ7tyOK0U0GI,15021
4
+ writel2/reader.py,sha256=h7pMPk9nJPyRUriSOroc4akvIj7I0MejZkZ4HnbmcDM,2429
5
+ writel2/writer.py,sha256=0Gix9QM2oreXvI30AL87REnf-m1FyOlTAa9M_rPV8LU,11606
6
+ obpg_writel2-1.0.0.dist-info/METADATA,sha256=yFlX6cCRMA0LYSDYf7Hm6_ugiwzIH5NieAYKjHBLjxg,3446
7
+ obpg_writel2-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ obpg_writel2-1.0.0.dist-info/top_level.txt,sha256=HMenOB06BMFGTLye6Pos3ArXENovGKwfkYX4NiyJ4iM,8
9
+ obpg_writel2-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ writel2
writel2/__init__.py ADDED
@@ -0,0 +1,13 @@
1
+ from .global_metadata import GlobalMetadata
2
+ from .product_info import ProductInfo, ProductNotFoundError, ProductTable
3
+ from .reader import L2Reader
4
+ from .writer import L2Writer
5
+
6
+ __all__ = [
7
+ "GlobalMetadata",
8
+ "ProductInfo",
9
+ "ProductNotFoundError",
10
+ "ProductTable",
11
+ "L2Reader",
12
+ "L2Writer",
13
+ ]
@@ -0,0 +1,48 @@
1
+ """
2
+ Loads the static, institution-wide NetCDF4 global attributes shared by every
3
+ L2 output file (license, naming_authority, creator_*/publisher_*, ...) from
4
+ $OCDATAROOT/common/global_metadata.json, mirroring how ProductTable resolves
5
+ share/common/product.xml (product_info.py). In l2gen these same values are
6
+ hardcoded #define macros in oel_hdf4/libl1/l1.h, written unconditionally by
7
+ the NetCDF4 output path in src/l2gen/l2_generic.c.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import os
14
+ from collections.abc import Mapping
15
+ from typing import Optional
16
+
17
+
18
+ class GlobalMetadata(Mapping):
19
+ """Read-only, lazily-loaded view of global_metadata.json's key/value pairs."""
20
+
21
+ def __init__(self, path: Optional[str] = None):
22
+ self._path = path
23
+ self._attrs: Optional[dict] = None
24
+
25
+ @property
26
+ def path(self) -> str:
27
+ if self._path:
28
+ return self._path
29
+ data_root = os.environ.get("OCDATAROOT")
30
+ if not data_root:
31
+ raise RuntimeError("OCDATAROOT environment variable is not defined.")
32
+ return os.path.join(data_root, "common", "global_metadata.json")
33
+
34
+ def _ensure_loaded(self) -> dict:
35
+ if self._attrs is None:
36
+ with open(self.path) as f:
37
+ data: dict = json.load(f)
38
+ self._attrs = data
39
+ return self._attrs
40
+
41
+ def __getitem__(self, name):
42
+ return self._ensure_loaded()[name]
43
+
44
+ def __iter__(self):
45
+ return iter(self._ensure_loaded())
46
+
47
+ def __len__(self):
48
+ return len(self._ensure_loaded())
@@ -0,0 +1,425 @@
1
+ """
2
+ Python port of oel_util/libpiutils/productInfo.{h,cpp} - resolves product
3
+ names (e.g. "chlor_a", "Rrs_443", "bbp_giop_443", "longitude", "year") against
4
+ share/common/product.xml into a ProductInfo record, the same metadata l2gen
5
+ uses to create variables in any standard L2 group (sensor_band_parameters,
6
+ scan_line_attributes, geophysical_data, navigation_data) via
7
+ findProductInfo().
8
+
9
+ Known gap vs. the C implementation: product_alias.json aliasing (per-sensor product
10
+ name remapping) is not ported. Add it here if a lookup that depends on it is needed.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ from dataclasses import dataclass
17
+ from typing import Optional, Sequence
18
+ from xml.etree import ElementTree as ET
19
+
20
+ NS = "{http://oceancolor.gsfc.nasa.gov}"
21
+
22
+ DEFAULT_DESCRIPTION = "no description"
23
+ DEFAULT_UNITS = ""
24
+ DEFAULT_PALETTE = "default"
25
+ DEFAULT_PARAM_DESIGNATOR = "none"
26
+ DEFAULT_PARAM_WAVE_MIN = -1
27
+ DEFAULT_PARAM_WAVE_MAX = -1
28
+ DEFAULT_STANDARD_NAME = None
29
+ DEFAULT_CATEGORY = "Miscellaneous"
30
+ DEFAULT_DATA_TYPE = "float"
31
+ DEFAULT_PREFIX = None
32
+ DEFAULT_SUFFIX = None
33
+ DEFAULT_ALGORITHM_NAME = ""
34
+ DEFAULT_PRODUCT_NAME = ""
35
+ DEFAULT_CAT_IX = -1
36
+ DEFAULT_PROD_IX = -1
37
+ DEFAULT_RANK = 2
38
+ DEFAULT_FILL_VALUE = -32767
39
+ DEFAULT_FILL_VALUE_BYTE = -1
40
+ DEFAULT_FILL_VALUE_UBYTE = 0xFF
41
+ DEFAULT_FILL_VALUE_USHORT = 0xFFFF
42
+ DEFAULT_FILL_VALUE_UINT = 0xFFFFFFFF
43
+ DEFAULT_VALID_MIN = -3.4028235e38
44
+ DEFAULT_VALID_MAX = 3.4028235e38
45
+ DEFAULT_DISPLAY_SCALE = "linear"
46
+ DEFAULT_DISPLAY_MIN = 0.0
47
+ DEFAULT_DISPLAY_MAX = 0.0
48
+ DEFAULT_SCALE_FACTOR = 1.0
49
+ DEFAULT_ADD_OFFSET = 0.0
50
+ DEFAULT_REFERENCE = None
51
+ DEFAULT_FLAG_VALUES = None
52
+ DEFAULT_FLAG_MEANINGS = None
53
+ DEFAULT_COMMENT = None
54
+ DEFAULT_TITLE_FORMAT = "no format string"
55
+ DEFAULT_USE_ALTERNATE_WAVE = False
56
+
57
+ # per-dtype _FillValue override, applied only when no explicit <fillValue> was given
58
+ FILL_VALUE_OVERRIDES = {
59
+ "byte": DEFAULT_FILL_VALUE_BYTE,
60
+ "ubyte": DEFAULT_FILL_VALUE_UBYTE,
61
+ "ushort": DEFAULT_FILL_VALUE_USHORT,
62
+ "uint": DEFAULT_FILL_VALUE_UINT,
63
+ }
64
+
65
+ # union-able wavelength ranges for <paramDesignator> values other than none/band/int
66
+ PARAM_DESIGNATOR_RANGES = {
67
+ "uv": (100, 400),
68
+ "visible": (400, 725),
69
+ "nir": (725, 1400),
70
+ "swir": (1400, 3000),
71
+ "emissive": (3000, 15000),
72
+ }
73
+
74
+
75
+ class ProductNotFoundError(KeyError):
76
+ """Raised when a product name can't be resolved against product.xml."""
77
+
78
+
79
+ @dataclass
80
+ class ProductInfo:
81
+ """Mirrors productInfo_t (oel_util/libpiutils/productInfo.h)."""
82
+
83
+ product_name: str = DEFAULT_PRODUCT_NAME
84
+ algorithm_name: str = DEFAULT_ALGORITHM_NAME
85
+ prefix: Optional[str] = DEFAULT_PREFIX
86
+ suffix: Optional[str] = DEFAULT_SUFFIX
87
+ description: str = DEFAULT_DESCRIPTION
88
+ title_format: str = DEFAULT_TITLE_FORMAT
89
+ units: str = DEFAULT_UNITS
90
+ palette: str = DEFAULT_PALETTE
91
+ param_designator: str = DEFAULT_PARAM_DESIGNATOR
92
+ param_wave_min: int = DEFAULT_PARAM_WAVE_MIN
93
+ param_wave_max: int = DEFAULT_PARAM_WAVE_MAX
94
+ standard_name: Optional[str] = DEFAULT_STANDARD_NAME
95
+ category: str = DEFAULT_CATEGORY
96
+ data_type: str = DEFAULT_DATA_TYPE
97
+ cat_ix: int = DEFAULT_CAT_IX
98
+ prod_ix: int = DEFAULT_PROD_IX
99
+ rank: int = DEFAULT_RANK
100
+ fill_value: float = DEFAULT_FILL_VALUE
101
+ valid_min: float = DEFAULT_VALID_MIN
102
+ valid_max: float = DEFAULT_VALID_MAX
103
+ display_scale: str = DEFAULT_DISPLAY_SCALE
104
+ display_min: float = DEFAULT_DISPLAY_MIN
105
+ display_max: float = DEFAULT_DISPLAY_MAX
106
+ scale_factor: float = DEFAULT_SCALE_FACTOR
107
+ add_offset: float = DEFAULT_ADD_OFFSET
108
+ flag_values: Optional[str] = DEFAULT_FLAG_VALUES
109
+ flag_meanings: Optional[str] = DEFAULT_FLAG_MEANINGS
110
+ reference: Optional[str] = DEFAULT_REFERENCE
111
+ comment: Optional[str] = DEFAULT_COMMENT
112
+ use_alternate_wave: bool = DEFAULT_USE_ALTERNATE_WAVE
113
+
114
+ @property
115
+ def full_name(self) -> str:
116
+ if self.param_designator == "none":
117
+ return f"{self.prefix}{self.suffix}"
118
+ return f"{self.prefix}{self.prod_ix}{self.suffix}"
119
+
120
+
121
+ def _text(node: Optional[ET.Element]) -> Optional[str]:
122
+ return node.text.strip() if node is not None and node.text is not None else None
123
+
124
+
125
+ # Tags copied verbatim (same name used in product.xml and on ProductInfo) from
126
+ # both a <product> and its <algorithm> element - the algorithm's value, applied
127
+ # second, overrides the product's.
128
+ _COPY_TAGS = ("flag_values", "flag_meanings", "reference", "comment")
129
+
130
+
131
+ def _apply_copy_tags(info: ProductInfo, el: ET.Element) -> None:
132
+ for tag in _COPY_TAGS:
133
+ if (text := _text(el.find(f"{NS}{tag}"))) is not None:
134
+ setattr(info, tag, text)
135
+
136
+
137
+ def _apply_param_designator(node: ET.Element, designator: str, wmin: int, wmax: int):
138
+ """Port of readParamDesignator(): with no <paramDesignator> children, (designator,
139
+ wmin, wmax) pass through unchanged; otherwise they are recomputed from scratch as
140
+ the union of this node's own <paramDesignator> values."""
141
+ children = node.findall(f"{NS}paramDesignator")
142
+ if not children:
143
+ return designator, wmin, wmax
144
+
145
+ designator, wmin, wmax = (
146
+ DEFAULT_PARAM_DESIGNATOR,
147
+ DEFAULT_PARAM_WAVE_MIN,
148
+ DEFAULT_PARAM_WAVE_MAX,
149
+ )
150
+ for child in children:
151
+ value = (child.text or "").strip()
152
+ if value in ("none", "band", "int"):
153
+ return value, DEFAULT_PARAM_WAVE_MIN, DEFAULT_PARAM_WAVE_MAX
154
+ if value not in PARAM_DESIGNATOR_RANGES:
155
+ raise ValueError(f'"paramDesignator" has illegal value "{value}"')
156
+ lo, hi = PARAM_DESIGNATOR_RANGES[value]
157
+ designator = "wave"
158
+ if wmin == DEFAULT_PARAM_WAVE_MIN or wmin > lo:
159
+ wmin = lo
160
+ if wmax == DEFAULT_PARAM_WAVE_MAX or wmax < hi:
161
+ wmax = hi
162
+ return designator, wmin, wmax
163
+
164
+
165
+ def _apply_single_range(info: ProductInfo, range_node: ET.Element) -> None:
166
+ for tag, attr in (
167
+ ("validMin", "valid_min"),
168
+ ("validMax", "valid_max"),
169
+ ("displayMin", "display_min"),
170
+ ("displayMax", "display_max"),
171
+ ("scaleFactor", "scale_factor"),
172
+ ("addOffset", "add_offset"),
173
+ ):
174
+ text = _text(range_node.find(f"{NS}{tag}"))
175
+ if text is not None:
176
+ setattr(info, attr, float(text))
177
+
178
+
179
+ def _apply_range(info: ProductInfo, node: ET.Element, param_val: int) -> None:
180
+ ranges = node.findall(f"{NS}range")
181
+ if not ranges:
182
+ return
183
+ for r in ranges:
184
+ if r.get("min") is not None or r.get("max") is not None:
185
+ continue
186
+ _apply_single_range(info, r)
187
+ for r in ranges:
188
+ lo, hi = r.get("min"), r.get("max")
189
+ if lo is None or hi is None:
190
+ continue
191
+ if int(lo) <= param_val <= int(hi):
192
+ _apply_single_range(info, r)
193
+ break
194
+
195
+
196
+ def _match_name(
197
+ name: str,
198
+ prefix: str,
199
+ suffix: str,
200
+ designator: str,
201
+ wmin: int,
202
+ wmax: int,
203
+ wavelengths: Optional[Sequence[float]],
204
+ ) -> Optional[int]:
205
+ """Port of compareAlgorithmHeaderWithWavelengths(): returns the matched prod_ix
206
+ (DEFAULT_PROD_IX for a "none"-designator match), or None if `name` doesn't match
207
+ this algorithm's prefix/suffix."""
208
+ if not name.startswith(prefix):
209
+ return None
210
+
211
+ if designator == "none":
212
+ return DEFAULT_PROD_IX if name == f"{prefix}{suffix}" else None
213
+
214
+ param_len = len(name) - len(prefix) - len(suffix)
215
+ if param_len < 1:
216
+ return None
217
+ if suffix and not name.endswith(suffix):
218
+ return None
219
+ if not suffix and len(name) != len(prefix) + param_len:
220
+ return None
221
+
222
+ param_str = name[len(prefix) : len(prefix) + param_len]
223
+ try:
224
+ value = int(param_str)
225
+ except ValueError:
226
+ return None
227
+
228
+ if designator == "wave":
229
+ if value < wmin or value > wmax:
230
+ return None
231
+ if wavelengths is not None and not any(
232
+ abs(value - w) < 0.5 for w in wavelengths
233
+ ):
234
+ return None
235
+
236
+ return value
237
+
238
+
239
+ class ProductTable:
240
+ """Loads share/common/product.xml (from $OCDATAROOT) once and resolves product
241
+ names into ProductInfo records, mirroring productInfo.cpp's getFirstProductInfo/
242
+ getNextProductInfo/findProductInfoForWavelengths."""
243
+
244
+ def __init__(self, path: Optional[str] = None):
245
+ self._path = path
246
+ self._products_node: Optional[ET.Element] = None
247
+
248
+ @property
249
+ def path(self) -> str:
250
+ if self._path:
251
+ return self._path
252
+ data_root = os.environ.get("OCDATAROOT")
253
+ if not data_root:
254
+ raise RuntimeError("OCDATAROOT environment variable is not defined.")
255
+ return os.path.join(data_root, "common", "product.xml")
256
+
257
+ def _ensure_loaded(self) -> ET.Element:
258
+ if self._products_node is None:
259
+ tree = ET.parse(self.path)
260
+ self._products_node = tree.getroot()
261
+ return self._products_node
262
+
263
+ def products(self):
264
+ """Yield every (product_element) in document order."""
265
+ yield from self._ensure_loaded().findall(f"{NS}product")
266
+
267
+ def find(
268
+ self, name: str, wavelengths: Optional[Sequence[float]] = None
269
+ ) -> ProductInfo:
270
+ """Resolve a full product name (e.g. "chlor_a", "Rrs_443") to a ProductInfo.
271
+
272
+ `wavelengths` should be the sensor's band list when resolving a wavelength-
273
+ designated product, so an out-of-band request (e.g. Rrs_999 on a sensor with
274
+ no 999nm band) is rejected the same way l2gen rejects it.
275
+ """
276
+ for product_el in self.products():
277
+ product_name = product_el.get("name", "").strip()
278
+ designator, wmin, wmax = _apply_param_designator(
279
+ product_el,
280
+ DEFAULT_PARAM_DESIGNATOR,
281
+ DEFAULT_PARAM_WAVE_MIN,
282
+ DEFAULT_PARAM_WAVE_MAX,
283
+ )
284
+
285
+ for algorithm_el in product_el.findall(f"{NS}algorithm"):
286
+ alg_designator, alg_wmin, alg_wmax = _apply_param_designator(
287
+ algorithm_el, designator, wmin, wmax
288
+ )
289
+ algorithm_name = algorithm_el.get("name", "").strip()
290
+ default_prefix = (
291
+ product_name if alg_designator == "none" else f"{product_name}_"
292
+ )
293
+ prefix = _text(algorithm_el.find(f"{NS}prefix")) or default_prefix
294
+ suffix = _text(algorithm_el.find(f"{NS}suffix"))
295
+ if suffix is None:
296
+ suffix = f"_{algorithm_name}" if algorithm_name else ""
297
+
298
+ param_val = _match_name(
299
+ name,
300
+ prefix,
301
+ suffix,
302
+ alg_designator,
303
+ alg_wmin,
304
+ alg_wmax,
305
+ wavelengths,
306
+ )
307
+ if param_val is None:
308
+ continue
309
+
310
+ info = self._build_product(
311
+ product_el, product_name, designator, wmin, wmax, param_val
312
+ )
313
+ self._apply_algorithm(
314
+ info,
315
+ algorithm_el,
316
+ algorithm_name,
317
+ prefix,
318
+ suffix,
319
+ alg_designator,
320
+ param_val,
321
+ )
322
+ return info
323
+
324
+ raise ProductNotFoundError(name)
325
+
326
+ @staticmethod
327
+ def _build_product(
328
+ product_el: ET.Element,
329
+ product_name: str,
330
+ designator: str,
331
+ wmin: int,
332
+ wmax: int,
333
+ param_val: int,
334
+ ) -> ProductInfo:
335
+ info = ProductInfo(
336
+ product_name=product_name,
337
+ param_designator=designator,
338
+ param_wave_min=wmin,
339
+ param_wave_max=wmax,
340
+ )
341
+
342
+ if (text := _text(product_el.find(f"{NS}standardName"))) is not None:
343
+ info.standard_name = text
344
+ if (text := _text(product_el.find(f"{NS}units"))) is not None:
345
+ info.units = text
346
+ if (text := _text(product_el.find(f"{NS}palette"))) is not None:
347
+ info.palette = text
348
+
349
+ category = _text(product_el.find(f"{NS}category"))
350
+ if category is None:
351
+ raise ValueError(
352
+ f'Need to define "category" in product "{product_name}" in product.xml'
353
+ )
354
+ info.category = category
355
+
356
+ if (text := _text(product_el.find(f"{NS}displayScale"))) is not None:
357
+ info.display_scale = text
358
+ if (text := _text(product_el.find(f"{NS}type"))) is not None:
359
+ info.data_type = text
360
+ if (
361
+ info.fill_value == DEFAULT_FILL_VALUE
362
+ and info.data_type in FILL_VALUE_OVERRIDES
363
+ ):
364
+ info.fill_value = FILL_VALUE_OVERRIDES[info.data_type]
365
+
366
+ _apply_copy_tags(info, product_el)
367
+
368
+ info.prod_ix = param_val if designator != "none" else DEFAULT_PROD_IX
369
+ _apply_range(info, product_el, param_val)
370
+ return info
371
+
372
+ @staticmethod
373
+ def _apply_algorithm(
374
+ info: ProductInfo,
375
+ algorithm_el: ET.Element,
376
+ algorithm_name: str,
377
+ prefix: str,
378
+ suffix: str,
379
+ designator: str,
380
+ param_val: int,
381
+ ) -> None:
382
+ info.algorithm_name = algorithm_name
383
+ info.prefix = prefix
384
+ info.suffix = suffix
385
+
386
+ cat_ix_text = _text(algorithm_el.find(f"{NS}cat_ix"))
387
+ if cat_ix_text is None:
388
+ raise ValueError(
389
+ f'Need to define "cat_ix" in product "{info.product_name}_{algorithm_name}"'
390
+ )
391
+ info.cat_ix = int(cat_ix_text)
392
+
393
+ if (text := _text(algorithm_el.find(f"{NS}rank"))) is not None:
394
+ info.rank = int(text)
395
+ if (text := _text(algorithm_el.find(f"{NS}use_alternate_wave"))) is not None:
396
+ info.use_alternate_wave = bool(int(text))
397
+ if (text := _text(algorithm_el.find(f"{NS}units"))) is not None:
398
+ info.units = text
399
+ if (text := _text(algorithm_el.find(f"{NS}fillValue"))) is not None:
400
+ info.fill_value = float(text)
401
+
402
+ description = _text(algorithm_el.find(f"{NS}description"))
403
+ if description is None:
404
+ raise ValueError(
405
+ f'Need to define "description" in product '
406
+ f'"{info.product_name}_{algorithm_name}" in product.xml'
407
+ )
408
+ info.title_format = description
409
+ if designator == "none":
410
+ if (text := _text(algorithm_el.find(f"{NS}prod_ix"))) is not None:
411
+ info.prod_ix = int(text)
412
+ info.description = description
413
+ else:
414
+ info.prod_ix = param_val
415
+ info.description = description % info.prod_ix
416
+
417
+ _apply_copy_tags(info, algorithm_el)
418
+
419
+ if (text := _text(algorithm_el.find(f"{NS}type"))) is not None:
420
+ had_explicit_fill = _text(algorithm_el.find(f"{NS}fillValue")) is not None
421
+ info.data_type = text
422
+ if not had_explicit_fill and info.data_type in FILL_VALUE_OVERRIDES:
423
+ info.fill_value = FILL_VALUE_OVERRIDES[info.data_type]
424
+
425
+ _apply_range(info, algorithm_el, param_val)
writel2/reader.py ADDED
@@ -0,0 +1,84 @@
1
+ """
2
+ Read-only helper for pulling metadata out of an existing L2 (or L1/L3) NetCDF4
3
+ file, primarily so L2Writer can seed a new output file's global attributes
4
+ from a source L2 file (see src/libl2/l2_metadata.cpp for the reference list
5
+ of attributes an L2 consumer cares about).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Iterable, Optional
11
+
12
+ import h5netcdf
13
+
14
+ # Global attributes l2gen writes that are safe/meaningful to copy forward onto
15
+ # a derived L2 product. Run-specific attributes (title, history, date_created,
16
+ # product_name, id, processing_version, ...) are deliberately excluded here -
17
+ # the writer sets its own.
18
+ COPYABLE_GLOBAL_ATTRS = (
19
+ "instrument",
20
+ "platform",
21
+ "Conventions",
22
+ "license",
23
+ "naming_authority",
24
+ "keywords_vocabulary",
25
+ "keywords",
26
+ "standard_name_vocabulary",
27
+ "institution",
28
+ "creator_name",
29
+ "creator_email",
30
+ "creator_url",
31
+ "project",
32
+ "publisher_name",
33
+ "publisher_url",
34
+ "publisher_email",
35
+ "geospatial_bounds_crs",
36
+ "processing_level",
37
+ "cdm_data_type",
38
+ "spatialResolution",
39
+ "source",
40
+ "time_coverage_start",
41
+ "time_coverage_end",
42
+ "westernmost_longitude",
43
+ "easternmost_longitude",
44
+ "southernmost_latitude",
45
+ "northernmost_latitude",
46
+ )
47
+
48
+ L2_GROUP_NAMES = ("geophysical_data", "processing_control")
49
+
50
+
51
+ class L2Reader:
52
+ def __init__(self, path):
53
+ self._file = h5netcdf.File(path, "r")
54
+
55
+ def __enter__(self) -> "L2Reader":
56
+ return self
57
+
58
+ def __exit__(self, *exc) -> None:
59
+ self.close()
60
+
61
+ def close(self) -> None:
62
+ self._file.close()
63
+
64
+ @property
65
+ def global_attrs(self) -> dict:
66
+ return dict(self._file.attrs)
67
+
68
+ def group(self, name: str) -> Optional[h5netcdf.Group]:
69
+ return self._file.groups.get(name)
70
+
71
+ def is_l2(self) -> bool:
72
+ return all(name in self._file.groups for name in L2_GROUP_NAMES)
73
+
74
+ def copy_global_attrs_to(self, dest, names: Optional[Iterable[str]] = None) -> None:
75
+ """Copy global attributes onto `dest` (an h5netcdf File or Group).
76
+
77
+ Defaults to COPYABLE_GLOBAL_ATTRS; pass `names` to copy a different set.
78
+ Attributes absent from this file are silently skipped.
79
+ """
80
+ wanted = COPYABLE_GLOBAL_ATTRS if names is None else names
81
+ attrs = self.global_attrs
82
+ for name in wanted:
83
+ if name in attrs:
84
+ dest.attrs[name] = attrs[name]
writel2/writer.py ADDED
@@ -0,0 +1,289 @@
1
+ """
2
+ L2Writer: open a NetCDF4 file and populate it with the standard L2 group
3
+ layout (sensor_band_parameters, scan_line_attributes, geophysical_data,
4
+ navigation_data, processing_control/input_parameters[/flag_percentages]) and
5
+ write variables into any of those groups via write_product(), whose
6
+ attributes (long_name, units, valid_min/max, scale_factor/add_offset,
7
+ _FillValue, reference, comment, ...) come from share/common/product.xml -
8
+ mirroring createDS()/createDS2() in oel_hdf4/libdfutils/wrapper.c, which both
9
+ resolve against product.xml (via findProductInfo()) regardless of which
10
+ group they're writing into. h5netcdf has no auto scale/mask support (unlike
11
+ netCDF4-python/xarray), so packing physical-units data into storage units on
12
+ write is done here explicitly. The file is opened with format="NETCDF4_CLASSIC"
13
+ so plain Python str attributes are stored as NC_CHAR text, matching
14
+ nc_put_att_text()/SetChrGA() in wrapper.c - h5netcdf's default ("NETCDF4")
15
+ would instead store them as NC_STRING, which real l2gen output never uses.
16
+
17
+ One documented exception: geophysical_data's own local "wavelength"
18
+ coordinate variable (used alongside 3D products like Rrs) is written with a
19
+ hardcoded call in src/l2gen/l2_generic.c, bypassing product.xml entirely -
20
+ write_product() can't be used for that one variable.
21
+
22
+ flag_values/flag_meanings and the _FillValue skip for flag_*/l2_flags-named
23
+ variables aren't part of ScaledNcVar (which has no notion of flag products at
24
+ all) - that behavior is carried over from the older oel_hdf4/libdfutils/
25
+ wrapper.c, since it's the only concrete spec available for flag products.
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import warnings
31
+ from datetime import datetime, timezone
32
+ from typing import Optional, Sequence
33
+
34
+ import h5netcdf
35
+ import numpy as np
36
+
37
+ from .global_metadata import GlobalMetadata
38
+ from .product_info import (
39
+ DEFAULT_VALID_MAX,
40
+ DEFAULT_VALID_MIN,
41
+ ProductInfo,
42
+ ProductTable,
43
+ )
44
+ from .reader import L2Reader
45
+
46
+ # Level-specific, not part of global_metadata.json: every L2 writer sets these
47
+ # itself (e.g. oel_hdf4/libl1/l1_generic_write.c uses "L1B"/"swath", l3mapgen's
48
+ # OutFileNetcdf4.cpp uses "L3 Mapped"/"grid").
49
+ PROCESSING_LEVEL = "L2"
50
+ CDM_DATA_TYPE = "swath"
51
+
52
+ DTYPE_MAP = {
53
+ "byte": np.int8,
54
+ "ubyte": np.uint8,
55
+ "short": np.int16,
56
+ "ushort": np.uint16,
57
+ "int": np.int32,
58
+ "uint": np.uint32,
59
+ "float": np.float32,
60
+ "double": np.float64,
61
+ }
62
+
63
+
64
+ def _is_flag_product(name: str) -> bool:
65
+ return name.startswith("flag_") or name == "l2_flags"
66
+
67
+
68
+ def _scale_to_storage(value: float, info: ProductInfo, np_dtype) -> "np.generic":
69
+ """Scale a *physical-units* value (e.g. valid_min/valid_max) into storage
70
+ units: ROUND((value - add_offset) / scale_factor). Note: scaled_nc_var.cpp's
71
+ populateScaleFactors() writes valid_min/valid_max unscaled (cast straight to
72
+ the storage NcType) - that looks like a bug (e.g. bbp's validMin=5e-06 would
73
+ cast to int16 as 0), so this keeps the scaled, CF-correct behavior instead."""
74
+ if info.scale_factor == 1.0 and info.add_offset == 0.0:
75
+ return _raw_cast(value, np_dtype)
76
+ if np.issubdtype(np_dtype, np.floating):
77
+ return np_dtype(value)
78
+ return np_dtype(round((value - info.add_offset) / info.scale_factor))
79
+
80
+
81
+ def _raw_cast(value: float, np_dtype) -> "np.generic":
82
+ """Cast a value already expressed in storage units (fillValue, flag_values)
83
+ directly to the storage dtype - no scale_factor/add_offset applied."""
84
+ return (
85
+ np_dtype(round(value))
86
+ if not np.issubdtype(np_dtype, np.floating)
87
+ else np_dtype(value)
88
+ )
89
+
90
+
91
+ def _pack(data: np.ndarray, info: ProductInfo, np_dtype) -> np.ndarray:
92
+ """Pack physical-units `data` into storage units. Rounds to nearest (unlike
93
+ scaled_nc_var.cpp's compress(), which truncates) and, like compress(), clamps
94
+ any value outside the storage dtype's representable range to the fill value
95
+ instead of letting it silently wrap on cast."""
96
+ if np.issubdtype(np_dtype, np.floating):
97
+ return data.astype(np_dtype)
98
+
99
+ nonfinite = (
100
+ ~np.isfinite(data)
101
+ if np.issubdtype(data.dtype, np.floating)
102
+ else np.zeros(data.shape, dtype=bool)
103
+ )
104
+ if info.scale_factor == 1.0 and info.add_offset == 0.0:
105
+ packed = np.round(data.astype(np.float64))
106
+ else:
107
+ packed = np.round(
108
+ (data.astype(np.float64) - info.add_offset) / info.scale_factor
109
+ )
110
+
111
+ bounds = np.iinfo(np_dtype)
112
+ bad = nonfinite | (packed < bounds.min) | (packed > bounds.max)
113
+ fill = float(_raw_cast(info.fill_value, np_dtype))
114
+ return np.where(bad, fill, packed).astype(np_dtype)
115
+
116
+
117
+ class L2Writer:
118
+ GROUP_NAMES = (
119
+ "sensor_band_parameters",
120
+ "scan_line_attributes",
121
+ "geophysical_data",
122
+ "navigation_data",
123
+ "processing_control",
124
+ )
125
+
126
+ def __init__(
127
+ self,
128
+ path,
129
+ source: Optional[L2Reader] = None,
130
+ wavelengths: Optional[Sequence[float]] = None,
131
+ title: Optional[str] = None,
132
+ global_attrs: Optional[dict] = None,
133
+ product_table: Optional[ProductTable] = None,
134
+ global_metadata: Optional[GlobalMetadata] = None,
135
+ ):
136
+ self.file = h5netcdf.File(path, "w", format="NETCDF4_CLASSIC")
137
+ self.groups = {name: self.file.create_group(name) for name in self.GROUP_NAMES}
138
+ self.groups["input_parameters"] = self.groups[
139
+ "processing_control"
140
+ ].create_group("input_parameters")
141
+
142
+ self._product_table = product_table or ProductTable()
143
+ self._global_metadata = global_metadata or GlobalMetadata()
144
+ self._wavelengths = wavelengths
145
+
146
+ if source is not None:
147
+ if source.is_l2():
148
+ self.copy_global_attrs_from(source)
149
+ else:
150
+ warnings.warn(
151
+ "source file is not L2-shaped; skipping global attribute copy"
152
+ )
153
+
154
+ self._set_standard_global_attrs(title=title)
155
+ if global_attrs:
156
+ for name, value in global_attrs.items():
157
+ self.file.attrs[name] = value
158
+
159
+ def __enter__(self) -> "L2Writer":
160
+ return self
161
+
162
+ def __exit__(self, *exc) -> None:
163
+ self.close()
164
+
165
+ def close(self) -> None:
166
+ self.file.close()
167
+
168
+ def _set_standard_global_attrs(self, title: Optional[str] = None) -> None:
169
+ for name, value in self._global_metadata.items():
170
+ self.file.attrs[name] = value
171
+ self.file.attrs["processing_level"] = PROCESSING_LEVEL
172
+ self.file.attrs["cdm_data_type"] = CDM_DATA_TYPE
173
+ self.file.attrs["date_created"] = datetime.now(timezone.utc).strftime(
174
+ "%Y-%m-%dT%H:%M:%S.000Z"
175
+ )
176
+ if title:
177
+ self.file.attrs["title"] = title
178
+
179
+ def copy_global_attrs_from(
180
+ self, source: L2Reader, names=None, overrides: Optional[dict] = None
181
+ ) -> None:
182
+ source.copy_global_attrs_to(self.file, names=names)
183
+ if overrides:
184
+ for name, value in overrides.items():
185
+ self.file.attrs[name] = value
186
+
187
+ def set_input_parameters(self, params: dict) -> None:
188
+ for name, value in params.items():
189
+ self.groups["input_parameters"].attrs[name] = value
190
+
191
+ def flag_percentages_group(self, name: str = "flag_percentages") -> h5netcdf.Group:
192
+ if name not in self.groups["processing_control"].groups:
193
+ self.groups["processing_control"].create_group(name)
194
+ return self.groups["processing_control"].groups[name]
195
+
196
+ def set_flag_percentages(
197
+ self, percentages: dict, group_name: str = "flag_percentages"
198
+ ) -> None:
199
+ grp = self.flag_percentages_group(group_name)
200
+ for flag_name, percentage in percentages.items():
201
+ grp.attrs[flag_name] = np.float32(percentage)
202
+
203
+ def _ensure_dim(self, name: str, size: int) -> None:
204
+ if name not in self.file.dimensions:
205
+ self.file.dimensions[name] = size
206
+
207
+ def write_product(
208
+ self,
209
+ name: str,
210
+ data,
211
+ dim_names: Sequence[str] = ("number_of_lines", "pixels_per_line"),
212
+ group: str = "geophysical_data",
213
+ ) -> h5netcdf.Variable:
214
+ """Resolve `name` against product.xml and write it into `group`.
215
+
216
+ `group` defaults to "geophysical_data" but can be any of
217
+ sensor_band_parameters/scan_line_attributes/navigation_data too -
218
+ l2gen resolves all of them against product.xml the same way (see
219
+ module docstring), so the same lookup/packing logic applies regardless
220
+ of which group a product belongs to.
221
+
222
+ `data` is expected in physical units; it is packed into storage units
223
+ using the resolved scale_factor/add_offset when the product's dtype is
224
+ not float/double. NaN/inf entries are written as the product's fill value.
225
+ The written scale_factor/add_offset attributes match `data`'s own
226
+ float precision (float32 or float64).
227
+ """
228
+ info = self._product_table.find(name, wavelengths=self._wavelengths)
229
+ np_dtype = DTYPE_MAP[info.data_type]
230
+ data = np.asarray(data)
231
+ dims = tuple(dim_names[: data.ndim])
232
+ for dim_name, size in zip(dims, data.shape):
233
+ self._ensure_dim(dim_name, size)
234
+
235
+ kwargs = {}
236
+ if not _is_flag_product(name):
237
+ kwargs["fillvalue"] = _raw_cast(info.fill_value, np_dtype)
238
+
239
+ grp = self.groups[group]
240
+ var = grp.create_variable(
241
+ name,
242
+ dimensions=dims,
243
+ dtype=np_dtype,
244
+ data=_pack(data, info, np_dtype),
245
+ **kwargs,
246
+ )
247
+
248
+ var.attrs["long_name"] = info.description
249
+ if not np.issubdtype(np_dtype, np.floating) and (
250
+ info.scale_factor != 1.0 or info.add_offset != 0.0
251
+ ):
252
+ # scale_factor/add_offset are stored in whatever float precision
253
+ # the caller's physical-units `data` was given in (float32 ->
254
+ # NC_FLOAT, float64 -> NC_DOUBLE), rather than a fixed precision -
255
+ # product.xml has no type for these attributes themselves, only
256
+ # for the variable's own storage type (np_dtype, above).
257
+ scale_dtype = (
258
+ data.dtype
259
+ if np.issubdtype(data.dtype, np.floating)
260
+ else np.dtype(np.float64)
261
+ )
262
+ var.attrs["scale_factor"] = scale_dtype.type(info.scale_factor)
263
+ var.attrs["add_offset"] = scale_dtype.type(info.add_offset)
264
+ if info.units and info.units.lower() not in (
265
+ "",
266
+ "dimensionless",
267
+ "unitless",
268
+ "none",
269
+ ):
270
+ var.attrs["units"] = info.units
271
+ if data.ndim > 1 and name not in ("longitude", "latitude"):
272
+ var.attrs["coordinates"] = "longitude latitude"
273
+ if info.standard_name:
274
+ var.attrs["standard_name"] = info.standard_name
275
+ if info.valid_min != DEFAULT_VALID_MIN or info.valid_max != DEFAULT_VALID_MAX:
276
+ var.attrs["valid_min"] = _scale_to_storage(info.valid_min, info, np_dtype)
277
+ var.attrs["valid_max"] = _scale_to_storage(info.valid_max, info, np_dtype)
278
+ if info.flag_values and info.flag_meanings:
279
+ values = np.array(
280
+ [int(v) for v in info.flag_values.split(",")], dtype=np_dtype
281
+ )
282
+ var.attrs["flag_values"] = values
283
+ var.attrs["flag_meanings"] = info.flag_meanings
284
+ if info.reference:
285
+ var.attrs["reference"] = info.reference
286
+ if info.comment:
287
+ var.attrs["comment"] = info.comment
288
+
289
+ return var