obpg-writel2 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- obpg_writel2-1.0.0.dist-info/METADATA +85 -0
- obpg_writel2-1.0.0.dist-info/RECORD +9 -0
- obpg_writel2-1.0.0.dist-info/WHEEL +5 -0
- obpg_writel2-1.0.0.dist-info/top_level.txt +1 -0
- writel2/__init__.py +13 -0
- writel2/global_metadata.py +48 -0
- writel2/product_info.py +425 -0
- writel2/reader.py +84 -0
- writel2/writer.py +289 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: obpg-writel2
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Helpers for reading L2 metadata and writing derived L2 NetCDF4 products
|
|
5
|
+
Author-email: Gwyn Fireman <Gwyn.F.Fireman@nasa.gov>, Claude Sonnet 5 <noreply@anthropic.com>
|
|
6
|
+
Requires-Python: >=3.12
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: numpy>=1.26.4
|
|
9
|
+
Requires-Dist: h5netcdf>=1.8.1
|
|
10
|
+
|
|
11
|
+
# writel2
|
|
12
|
+
|
|
13
|
+
Helpers for writing standard-layout L2 NetCDF4 ocean color products from
|
|
14
|
+
Python, and for reading metadata back out of an existing L2 file.
|
|
15
|
+
|
|
16
|
+
- `writel2.L2Writer` creates the standard l2gen group layout
|
|
17
|
+
(`sensor_band_parameters`, `scan_line_attributes`, `geophysical_data`,
|
|
18
|
+
`navigation_data`, `processing_control`/`input_parameters`
|
|
19
|
+
[/`flag_percentages`]), preloads the static global attributes, and writes
|
|
20
|
+
variables into any of those groups via `write_product(name, data,
|
|
21
|
+
group=...)`, whose attributes (`long_name`, `units`, `valid_min`/`valid_max`,
|
|
22
|
+
`scale_factor`/`add_offset`, `_FillValue`, `flag_values`/`flag_meanings`,
|
|
23
|
+
`reference`, `comment`, ...) are resolved from `share/common/product.xml` -
|
|
24
|
+
the same metadata l2gen itself uses, regardless of which group a variable
|
|
25
|
+
belongs to.
|
|
26
|
+
- `writel2.L2Reader` is a read-only helper for pulling metadata back out of
|
|
27
|
+
an existing L2 (or L1/L3) file, mainly so `L2Writer` can seed a derived
|
|
28
|
+
product's global attributes from its source file.
|
|
29
|
+
- `writel2.ProductTable` resolves a product name (e.g. `"chlor_a"`,
|
|
30
|
+
`"Rrs_443"`) against `product.xml` into a `ProductInfo` record.
|
|
31
|
+
- `writel2.GlobalMetadata` loads the static, institution-wide global
|
|
32
|
+
attributes (license, naming authority, creator/publisher info, ...) from
|
|
33
|
+
`$OCDATAROOT/common/global_metadata.json`.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install $OCSSWROOT/src/scripts/writel2
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
This installs as `obpg-writel2`, importable as `writel2`.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
See [`examples/demo_write_l2.py`](examples/demo_write_l2.py) for a complete,
|
|
46
|
+
runnable example that writes a small file touching every group:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
python $OCSSWROOT/src/scripts/writel2/examples/demo_write_l2.py /tmp/demo_l2.nc
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The short version:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import numpy as np
|
|
56
|
+
from writel2 import L2Writer
|
|
57
|
+
|
|
58
|
+
with L2Writer("out.L2.nc", title="Demo Level-2 Data") as writer:
|
|
59
|
+
# geophysical_data: resolved against product.xml for long_name, units,
|
|
60
|
+
# valid_min/valid_max, scale_factor/add_offset, _FillValue, ...
|
|
61
|
+
flags = np.zeros((4, 3), dtype=np.int32)
|
|
62
|
+
writer.write_product("l2_flags", flags)
|
|
63
|
+
|
|
64
|
+
# processing_control/input_parameters and /flag_percentages
|
|
65
|
+
writer.set_input_parameters({"ifile": "input.L1B.nc", "l2prod": "l2_flags"})
|
|
66
|
+
writer.set_flag_percentages({"ATMFAIL": 0.0, "LAND": 0.49})
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Datatype and attributes for any group's variables are resolved from
|
|
70
|
+
`product.xml` and applied automatically by `write_product()`, including
|
|
71
|
+
appropriate scaling - pass `group=` to target `sensor_band_parameters`,
|
|
72
|
+
`scan_line_attributes`, or `navigation_data` instead of the default
|
|
73
|
+
`geophysical_data`. The one exception is `geophysical_data`'s own local
|
|
74
|
+
`wavelength` coordinate variable (used alongside 3D products), which l2gen
|
|
75
|
+
writes directly rather than resolving against product.xml; the calling
|
|
76
|
+
script must set its datatype and attributes manually. See the demo script
|
|
77
|
+
for the full pattern, including how to register dimensions shared across
|
|
78
|
+
groups.
|
|
79
|
+
|
|
80
|
+
## Tests
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install $OCSSWROOT/src/scripts/writel2
|
|
84
|
+
python -m unittest discover -s $OCSSWROOT/src/scripts/writel2/tests
|
|
85
|
+
```
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
writel2/__init__.py,sha256=t6pI7EVsXP0dXtceoSXcfZkvP0XZdtdAh6VsXAvQD48,312
|
|
2
|
+
writel2/global_metadata.py,sha256=rhFi2UfgjZAfACOMzJJV_qsGQGD_s0DCAV7QN7QCw3U,1547
|
|
3
|
+
writel2/product_info.py,sha256=mGp54mgGbiTgADkUio71b6y4UVl9SElPQ7tyOK0U0GI,15021
|
|
4
|
+
writel2/reader.py,sha256=h7pMPk9nJPyRUriSOroc4akvIj7I0MejZkZ4HnbmcDM,2429
|
|
5
|
+
writel2/writer.py,sha256=0Gix9QM2oreXvI30AL87REnf-m1FyOlTAa9M_rPV8LU,11606
|
|
6
|
+
obpg_writel2-1.0.0.dist-info/METADATA,sha256=yFlX6cCRMA0LYSDYf7Hm6_ugiwzIH5NieAYKjHBLjxg,3446
|
|
7
|
+
obpg_writel2-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
8
|
+
obpg_writel2-1.0.0.dist-info/top_level.txt,sha256=HMenOB06BMFGTLye6Pos3ArXENovGKwfkYX4NiyJ4iM,8
|
|
9
|
+
obpg_writel2-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
writel2
|
writel2/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from .global_metadata import GlobalMetadata
|
|
2
|
+
from .product_info import ProductInfo, ProductNotFoundError, ProductTable
|
|
3
|
+
from .reader import L2Reader
|
|
4
|
+
from .writer import L2Writer
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"GlobalMetadata",
|
|
8
|
+
"ProductInfo",
|
|
9
|
+
"ProductNotFoundError",
|
|
10
|
+
"ProductTable",
|
|
11
|
+
"L2Reader",
|
|
12
|
+
"L2Writer",
|
|
13
|
+
]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Loads the static, institution-wide NetCDF4 global attributes shared by every
|
|
3
|
+
L2 output file (license, naming_authority, creator_*/publisher_*, ...) from
|
|
4
|
+
$OCDATAROOT/common/global_metadata.json, mirroring how ProductTable resolves
|
|
5
|
+
share/common/product.xml (product_info.py). In l2gen these same values are
|
|
6
|
+
hardcoded #define macros in oel_hdf4/libl1/l1.h, written unconditionally by
|
|
7
|
+
the NetCDF4 output path in src/l2gen/l2_generic.c.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class GlobalMetadata(Mapping):
|
|
19
|
+
"""Read-only, lazily-loaded view of global_metadata.json's key/value pairs."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, path: Optional[str] = None):
|
|
22
|
+
self._path = path
|
|
23
|
+
self._attrs: Optional[dict] = None
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def path(self) -> str:
|
|
27
|
+
if self._path:
|
|
28
|
+
return self._path
|
|
29
|
+
data_root = os.environ.get("OCDATAROOT")
|
|
30
|
+
if not data_root:
|
|
31
|
+
raise RuntimeError("OCDATAROOT environment variable is not defined.")
|
|
32
|
+
return os.path.join(data_root, "common", "global_metadata.json")
|
|
33
|
+
|
|
34
|
+
def _ensure_loaded(self) -> dict:
|
|
35
|
+
if self._attrs is None:
|
|
36
|
+
with open(self.path) as f:
|
|
37
|
+
data: dict = json.load(f)
|
|
38
|
+
self._attrs = data
|
|
39
|
+
return self._attrs
|
|
40
|
+
|
|
41
|
+
def __getitem__(self, name):
|
|
42
|
+
return self._ensure_loaded()[name]
|
|
43
|
+
|
|
44
|
+
def __iter__(self):
|
|
45
|
+
return iter(self._ensure_loaded())
|
|
46
|
+
|
|
47
|
+
def __len__(self):
|
|
48
|
+
return len(self._ensure_loaded())
|
writel2/product_info.py
ADDED
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Python port of oel_util/libpiutils/productInfo.{h,cpp} - resolves product
|
|
3
|
+
names (e.g. "chlor_a", "Rrs_443", "bbp_giop_443", "longitude", "year") against
|
|
4
|
+
share/common/product.xml into a ProductInfo record, the same metadata l2gen
|
|
5
|
+
uses to create variables in any standard L2 group (sensor_band_parameters,
|
|
6
|
+
scan_line_attributes, geophysical_data, navigation_data) via
|
|
7
|
+
findProductInfo().
|
|
8
|
+
|
|
9
|
+
Known gap vs. the C implementation: product_alias.json aliasing (per-sensor product
|
|
10
|
+
name remapping) is not ported. Add it here if a lookup that depends on it is needed.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Optional, Sequence
|
|
18
|
+
from xml.etree import ElementTree as ET
|
|
19
|
+
|
|
20
|
+
NS = "{http://oceancolor.gsfc.nasa.gov}"
|
|
21
|
+
|
|
22
|
+
DEFAULT_DESCRIPTION = "no description"
|
|
23
|
+
DEFAULT_UNITS = ""
|
|
24
|
+
DEFAULT_PALETTE = "default"
|
|
25
|
+
DEFAULT_PARAM_DESIGNATOR = "none"
|
|
26
|
+
DEFAULT_PARAM_WAVE_MIN = -1
|
|
27
|
+
DEFAULT_PARAM_WAVE_MAX = -1
|
|
28
|
+
DEFAULT_STANDARD_NAME = None
|
|
29
|
+
DEFAULT_CATEGORY = "Miscellaneous"
|
|
30
|
+
DEFAULT_DATA_TYPE = "float"
|
|
31
|
+
DEFAULT_PREFIX = None
|
|
32
|
+
DEFAULT_SUFFIX = None
|
|
33
|
+
DEFAULT_ALGORITHM_NAME = ""
|
|
34
|
+
DEFAULT_PRODUCT_NAME = ""
|
|
35
|
+
DEFAULT_CAT_IX = -1
|
|
36
|
+
DEFAULT_PROD_IX = -1
|
|
37
|
+
DEFAULT_RANK = 2
|
|
38
|
+
DEFAULT_FILL_VALUE = -32767
|
|
39
|
+
DEFAULT_FILL_VALUE_BYTE = -1
|
|
40
|
+
DEFAULT_FILL_VALUE_UBYTE = 0xFF
|
|
41
|
+
DEFAULT_FILL_VALUE_USHORT = 0xFFFF
|
|
42
|
+
DEFAULT_FILL_VALUE_UINT = 0xFFFFFFFF
|
|
43
|
+
DEFAULT_VALID_MIN = -3.4028235e38
|
|
44
|
+
DEFAULT_VALID_MAX = 3.4028235e38
|
|
45
|
+
DEFAULT_DISPLAY_SCALE = "linear"
|
|
46
|
+
DEFAULT_DISPLAY_MIN = 0.0
|
|
47
|
+
DEFAULT_DISPLAY_MAX = 0.0
|
|
48
|
+
DEFAULT_SCALE_FACTOR = 1.0
|
|
49
|
+
DEFAULT_ADD_OFFSET = 0.0
|
|
50
|
+
DEFAULT_REFERENCE = None
|
|
51
|
+
DEFAULT_FLAG_VALUES = None
|
|
52
|
+
DEFAULT_FLAG_MEANINGS = None
|
|
53
|
+
DEFAULT_COMMENT = None
|
|
54
|
+
DEFAULT_TITLE_FORMAT = "no format string"
|
|
55
|
+
DEFAULT_USE_ALTERNATE_WAVE = False
|
|
56
|
+
|
|
57
|
+
# per-dtype _FillValue override, applied only when no explicit <fillValue> was given
|
|
58
|
+
FILL_VALUE_OVERRIDES = {
|
|
59
|
+
"byte": DEFAULT_FILL_VALUE_BYTE,
|
|
60
|
+
"ubyte": DEFAULT_FILL_VALUE_UBYTE,
|
|
61
|
+
"ushort": DEFAULT_FILL_VALUE_USHORT,
|
|
62
|
+
"uint": DEFAULT_FILL_VALUE_UINT,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# union-able wavelength ranges for <paramDesignator> values other than none/band/int
|
|
66
|
+
PARAM_DESIGNATOR_RANGES = {
|
|
67
|
+
"uv": (100, 400),
|
|
68
|
+
"visible": (400, 725),
|
|
69
|
+
"nir": (725, 1400),
|
|
70
|
+
"swir": (1400, 3000),
|
|
71
|
+
"emissive": (3000, 15000),
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ProductNotFoundError(KeyError):
|
|
76
|
+
"""Raised when a product name can't be resolved against product.xml."""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class ProductInfo:
|
|
81
|
+
"""Mirrors productInfo_t (oel_util/libpiutils/productInfo.h)."""
|
|
82
|
+
|
|
83
|
+
product_name: str = DEFAULT_PRODUCT_NAME
|
|
84
|
+
algorithm_name: str = DEFAULT_ALGORITHM_NAME
|
|
85
|
+
prefix: Optional[str] = DEFAULT_PREFIX
|
|
86
|
+
suffix: Optional[str] = DEFAULT_SUFFIX
|
|
87
|
+
description: str = DEFAULT_DESCRIPTION
|
|
88
|
+
title_format: str = DEFAULT_TITLE_FORMAT
|
|
89
|
+
units: str = DEFAULT_UNITS
|
|
90
|
+
palette: str = DEFAULT_PALETTE
|
|
91
|
+
param_designator: str = DEFAULT_PARAM_DESIGNATOR
|
|
92
|
+
param_wave_min: int = DEFAULT_PARAM_WAVE_MIN
|
|
93
|
+
param_wave_max: int = DEFAULT_PARAM_WAVE_MAX
|
|
94
|
+
standard_name: Optional[str] = DEFAULT_STANDARD_NAME
|
|
95
|
+
category: str = DEFAULT_CATEGORY
|
|
96
|
+
data_type: str = DEFAULT_DATA_TYPE
|
|
97
|
+
cat_ix: int = DEFAULT_CAT_IX
|
|
98
|
+
prod_ix: int = DEFAULT_PROD_IX
|
|
99
|
+
rank: int = DEFAULT_RANK
|
|
100
|
+
fill_value: float = DEFAULT_FILL_VALUE
|
|
101
|
+
valid_min: float = DEFAULT_VALID_MIN
|
|
102
|
+
valid_max: float = DEFAULT_VALID_MAX
|
|
103
|
+
display_scale: str = DEFAULT_DISPLAY_SCALE
|
|
104
|
+
display_min: float = DEFAULT_DISPLAY_MIN
|
|
105
|
+
display_max: float = DEFAULT_DISPLAY_MAX
|
|
106
|
+
scale_factor: float = DEFAULT_SCALE_FACTOR
|
|
107
|
+
add_offset: float = DEFAULT_ADD_OFFSET
|
|
108
|
+
flag_values: Optional[str] = DEFAULT_FLAG_VALUES
|
|
109
|
+
flag_meanings: Optional[str] = DEFAULT_FLAG_MEANINGS
|
|
110
|
+
reference: Optional[str] = DEFAULT_REFERENCE
|
|
111
|
+
comment: Optional[str] = DEFAULT_COMMENT
|
|
112
|
+
use_alternate_wave: bool = DEFAULT_USE_ALTERNATE_WAVE
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def full_name(self) -> str:
|
|
116
|
+
if self.param_designator == "none":
|
|
117
|
+
return f"{self.prefix}{self.suffix}"
|
|
118
|
+
return f"{self.prefix}{self.prod_ix}{self.suffix}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _text(node: Optional[ET.Element]) -> Optional[str]:
|
|
122
|
+
return node.text.strip() if node is not None and node.text is not None else None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# Tags copied verbatim (same name used in product.xml and on ProductInfo) from
|
|
126
|
+
# both a <product> and its <algorithm> element - the algorithm's value, applied
|
|
127
|
+
# second, overrides the product's.
|
|
128
|
+
_COPY_TAGS = ("flag_values", "flag_meanings", "reference", "comment")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _apply_copy_tags(info: ProductInfo, el: ET.Element) -> None:
|
|
132
|
+
for tag in _COPY_TAGS:
|
|
133
|
+
if (text := _text(el.find(f"{NS}{tag}"))) is not None:
|
|
134
|
+
setattr(info, tag, text)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _apply_param_designator(node: ET.Element, designator: str, wmin: int, wmax: int):
|
|
138
|
+
"""Port of readParamDesignator(): with no <paramDesignator> children, (designator,
|
|
139
|
+
wmin, wmax) pass through unchanged; otherwise they are recomputed from scratch as
|
|
140
|
+
the union of this node's own <paramDesignator> values."""
|
|
141
|
+
children = node.findall(f"{NS}paramDesignator")
|
|
142
|
+
if not children:
|
|
143
|
+
return designator, wmin, wmax
|
|
144
|
+
|
|
145
|
+
designator, wmin, wmax = (
|
|
146
|
+
DEFAULT_PARAM_DESIGNATOR,
|
|
147
|
+
DEFAULT_PARAM_WAVE_MIN,
|
|
148
|
+
DEFAULT_PARAM_WAVE_MAX,
|
|
149
|
+
)
|
|
150
|
+
for child in children:
|
|
151
|
+
value = (child.text or "").strip()
|
|
152
|
+
if value in ("none", "band", "int"):
|
|
153
|
+
return value, DEFAULT_PARAM_WAVE_MIN, DEFAULT_PARAM_WAVE_MAX
|
|
154
|
+
if value not in PARAM_DESIGNATOR_RANGES:
|
|
155
|
+
raise ValueError(f'"paramDesignator" has illegal value "{value}"')
|
|
156
|
+
lo, hi = PARAM_DESIGNATOR_RANGES[value]
|
|
157
|
+
designator = "wave"
|
|
158
|
+
if wmin == DEFAULT_PARAM_WAVE_MIN or wmin > lo:
|
|
159
|
+
wmin = lo
|
|
160
|
+
if wmax == DEFAULT_PARAM_WAVE_MAX or wmax < hi:
|
|
161
|
+
wmax = hi
|
|
162
|
+
return designator, wmin, wmax
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _apply_single_range(info: ProductInfo, range_node: ET.Element) -> None:
|
|
166
|
+
for tag, attr in (
|
|
167
|
+
("validMin", "valid_min"),
|
|
168
|
+
("validMax", "valid_max"),
|
|
169
|
+
("displayMin", "display_min"),
|
|
170
|
+
("displayMax", "display_max"),
|
|
171
|
+
("scaleFactor", "scale_factor"),
|
|
172
|
+
("addOffset", "add_offset"),
|
|
173
|
+
):
|
|
174
|
+
text = _text(range_node.find(f"{NS}{tag}"))
|
|
175
|
+
if text is not None:
|
|
176
|
+
setattr(info, attr, float(text))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _apply_range(info: ProductInfo, node: ET.Element, param_val: int) -> None:
|
|
180
|
+
ranges = node.findall(f"{NS}range")
|
|
181
|
+
if not ranges:
|
|
182
|
+
return
|
|
183
|
+
for r in ranges:
|
|
184
|
+
if r.get("min") is not None or r.get("max") is not None:
|
|
185
|
+
continue
|
|
186
|
+
_apply_single_range(info, r)
|
|
187
|
+
for r in ranges:
|
|
188
|
+
lo, hi = r.get("min"), r.get("max")
|
|
189
|
+
if lo is None or hi is None:
|
|
190
|
+
continue
|
|
191
|
+
if int(lo) <= param_val <= int(hi):
|
|
192
|
+
_apply_single_range(info, r)
|
|
193
|
+
break
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _match_name(
|
|
197
|
+
name: str,
|
|
198
|
+
prefix: str,
|
|
199
|
+
suffix: str,
|
|
200
|
+
designator: str,
|
|
201
|
+
wmin: int,
|
|
202
|
+
wmax: int,
|
|
203
|
+
wavelengths: Optional[Sequence[float]],
|
|
204
|
+
) -> Optional[int]:
|
|
205
|
+
"""Port of compareAlgorithmHeaderWithWavelengths(): returns the matched prod_ix
|
|
206
|
+
(DEFAULT_PROD_IX for a "none"-designator match), or None if `name` doesn't match
|
|
207
|
+
this algorithm's prefix/suffix."""
|
|
208
|
+
if not name.startswith(prefix):
|
|
209
|
+
return None
|
|
210
|
+
|
|
211
|
+
if designator == "none":
|
|
212
|
+
return DEFAULT_PROD_IX if name == f"{prefix}{suffix}" else None
|
|
213
|
+
|
|
214
|
+
param_len = len(name) - len(prefix) - len(suffix)
|
|
215
|
+
if param_len < 1:
|
|
216
|
+
return None
|
|
217
|
+
if suffix and not name.endswith(suffix):
|
|
218
|
+
return None
|
|
219
|
+
if not suffix and len(name) != len(prefix) + param_len:
|
|
220
|
+
return None
|
|
221
|
+
|
|
222
|
+
param_str = name[len(prefix) : len(prefix) + param_len]
|
|
223
|
+
try:
|
|
224
|
+
value = int(param_str)
|
|
225
|
+
except ValueError:
|
|
226
|
+
return None
|
|
227
|
+
|
|
228
|
+
if designator == "wave":
|
|
229
|
+
if value < wmin or value > wmax:
|
|
230
|
+
return None
|
|
231
|
+
if wavelengths is not None and not any(
|
|
232
|
+
abs(value - w) < 0.5 for w in wavelengths
|
|
233
|
+
):
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
return value
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class ProductTable:
|
|
240
|
+
"""Loads share/common/product.xml (from $OCDATAROOT) once and resolves product
|
|
241
|
+
names into ProductInfo records, mirroring productInfo.cpp's getFirstProductInfo/
|
|
242
|
+
getNextProductInfo/findProductInfoForWavelengths."""
|
|
243
|
+
|
|
244
|
+
def __init__(self, path: Optional[str] = None):
|
|
245
|
+
self._path = path
|
|
246
|
+
self._products_node: Optional[ET.Element] = None
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def path(self) -> str:
|
|
250
|
+
if self._path:
|
|
251
|
+
return self._path
|
|
252
|
+
data_root = os.environ.get("OCDATAROOT")
|
|
253
|
+
if not data_root:
|
|
254
|
+
raise RuntimeError("OCDATAROOT environment variable is not defined.")
|
|
255
|
+
return os.path.join(data_root, "common", "product.xml")
|
|
256
|
+
|
|
257
|
+
def _ensure_loaded(self) -> ET.Element:
|
|
258
|
+
if self._products_node is None:
|
|
259
|
+
tree = ET.parse(self.path)
|
|
260
|
+
self._products_node = tree.getroot()
|
|
261
|
+
return self._products_node
|
|
262
|
+
|
|
263
|
+
def products(self):
|
|
264
|
+
"""Yield every (product_element) in document order."""
|
|
265
|
+
yield from self._ensure_loaded().findall(f"{NS}product")
|
|
266
|
+
|
|
267
|
+
def find(
|
|
268
|
+
self, name: str, wavelengths: Optional[Sequence[float]] = None
|
|
269
|
+
) -> ProductInfo:
|
|
270
|
+
"""Resolve a full product name (e.g. "chlor_a", "Rrs_443") to a ProductInfo.
|
|
271
|
+
|
|
272
|
+
`wavelengths` should be the sensor's band list when resolving a wavelength-
|
|
273
|
+
designated product, so an out-of-band request (e.g. Rrs_999 on a sensor with
|
|
274
|
+
no 999nm band) is rejected the same way l2gen rejects it.
|
|
275
|
+
"""
|
|
276
|
+
for product_el in self.products():
|
|
277
|
+
product_name = product_el.get("name", "").strip()
|
|
278
|
+
designator, wmin, wmax = _apply_param_designator(
|
|
279
|
+
product_el,
|
|
280
|
+
DEFAULT_PARAM_DESIGNATOR,
|
|
281
|
+
DEFAULT_PARAM_WAVE_MIN,
|
|
282
|
+
DEFAULT_PARAM_WAVE_MAX,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
for algorithm_el in product_el.findall(f"{NS}algorithm"):
|
|
286
|
+
alg_designator, alg_wmin, alg_wmax = _apply_param_designator(
|
|
287
|
+
algorithm_el, designator, wmin, wmax
|
|
288
|
+
)
|
|
289
|
+
algorithm_name = algorithm_el.get("name", "").strip()
|
|
290
|
+
default_prefix = (
|
|
291
|
+
product_name if alg_designator == "none" else f"{product_name}_"
|
|
292
|
+
)
|
|
293
|
+
prefix = _text(algorithm_el.find(f"{NS}prefix")) or default_prefix
|
|
294
|
+
suffix = _text(algorithm_el.find(f"{NS}suffix"))
|
|
295
|
+
if suffix is None:
|
|
296
|
+
suffix = f"_{algorithm_name}" if algorithm_name else ""
|
|
297
|
+
|
|
298
|
+
param_val = _match_name(
|
|
299
|
+
name,
|
|
300
|
+
prefix,
|
|
301
|
+
suffix,
|
|
302
|
+
alg_designator,
|
|
303
|
+
alg_wmin,
|
|
304
|
+
alg_wmax,
|
|
305
|
+
wavelengths,
|
|
306
|
+
)
|
|
307
|
+
if param_val is None:
|
|
308
|
+
continue
|
|
309
|
+
|
|
310
|
+
info = self._build_product(
|
|
311
|
+
product_el, product_name, designator, wmin, wmax, param_val
|
|
312
|
+
)
|
|
313
|
+
self._apply_algorithm(
|
|
314
|
+
info,
|
|
315
|
+
algorithm_el,
|
|
316
|
+
algorithm_name,
|
|
317
|
+
prefix,
|
|
318
|
+
suffix,
|
|
319
|
+
alg_designator,
|
|
320
|
+
param_val,
|
|
321
|
+
)
|
|
322
|
+
return info
|
|
323
|
+
|
|
324
|
+
raise ProductNotFoundError(name)
|
|
325
|
+
|
|
326
|
+
@staticmethod
|
|
327
|
+
def _build_product(
|
|
328
|
+
product_el: ET.Element,
|
|
329
|
+
product_name: str,
|
|
330
|
+
designator: str,
|
|
331
|
+
wmin: int,
|
|
332
|
+
wmax: int,
|
|
333
|
+
param_val: int,
|
|
334
|
+
) -> ProductInfo:
|
|
335
|
+
info = ProductInfo(
|
|
336
|
+
product_name=product_name,
|
|
337
|
+
param_designator=designator,
|
|
338
|
+
param_wave_min=wmin,
|
|
339
|
+
param_wave_max=wmax,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
if (text := _text(product_el.find(f"{NS}standardName"))) is not None:
|
|
343
|
+
info.standard_name = text
|
|
344
|
+
if (text := _text(product_el.find(f"{NS}units"))) is not None:
|
|
345
|
+
info.units = text
|
|
346
|
+
if (text := _text(product_el.find(f"{NS}palette"))) is not None:
|
|
347
|
+
info.palette = text
|
|
348
|
+
|
|
349
|
+
category = _text(product_el.find(f"{NS}category"))
|
|
350
|
+
if category is None:
|
|
351
|
+
raise ValueError(
|
|
352
|
+
f'Need to define "category" in product "{product_name}" in product.xml'
|
|
353
|
+
)
|
|
354
|
+
info.category = category
|
|
355
|
+
|
|
356
|
+
if (text := _text(product_el.find(f"{NS}displayScale"))) is not None:
|
|
357
|
+
info.display_scale = text
|
|
358
|
+
if (text := _text(product_el.find(f"{NS}type"))) is not None:
|
|
359
|
+
info.data_type = text
|
|
360
|
+
if (
|
|
361
|
+
info.fill_value == DEFAULT_FILL_VALUE
|
|
362
|
+
and info.data_type in FILL_VALUE_OVERRIDES
|
|
363
|
+
):
|
|
364
|
+
info.fill_value = FILL_VALUE_OVERRIDES[info.data_type]
|
|
365
|
+
|
|
366
|
+
_apply_copy_tags(info, product_el)
|
|
367
|
+
|
|
368
|
+
info.prod_ix = param_val if designator != "none" else DEFAULT_PROD_IX
|
|
369
|
+
_apply_range(info, product_el, param_val)
|
|
370
|
+
return info
|
|
371
|
+
|
|
372
|
+
@staticmethod
|
|
373
|
+
def _apply_algorithm(
|
|
374
|
+
info: ProductInfo,
|
|
375
|
+
algorithm_el: ET.Element,
|
|
376
|
+
algorithm_name: str,
|
|
377
|
+
prefix: str,
|
|
378
|
+
suffix: str,
|
|
379
|
+
designator: str,
|
|
380
|
+
param_val: int,
|
|
381
|
+
) -> None:
|
|
382
|
+
info.algorithm_name = algorithm_name
|
|
383
|
+
info.prefix = prefix
|
|
384
|
+
info.suffix = suffix
|
|
385
|
+
|
|
386
|
+
cat_ix_text = _text(algorithm_el.find(f"{NS}cat_ix"))
|
|
387
|
+
if cat_ix_text is None:
|
|
388
|
+
raise ValueError(
|
|
389
|
+
f'Need to define "cat_ix" in product "{info.product_name}_{algorithm_name}"'
|
|
390
|
+
)
|
|
391
|
+
info.cat_ix = int(cat_ix_text)
|
|
392
|
+
|
|
393
|
+
if (text := _text(algorithm_el.find(f"{NS}rank"))) is not None:
|
|
394
|
+
info.rank = int(text)
|
|
395
|
+
if (text := _text(algorithm_el.find(f"{NS}use_alternate_wave"))) is not None:
|
|
396
|
+
info.use_alternate_wave = bool(int(text))
|
|
397
|
+
if (text := _text(algorithm_el.find(f"{NS}units"))) is not None:
|
|
398
|
+
info.units = text
|
|
399
|
+
if (text := _text(algorithm_el.find(f"{NS}fillValue"))) is not None:
|
|
400
|
+
info.fill_value = float(text)
|
|
401
|
+
|
|
402
|
+
description = _text(algorithm_el.find(f"{NS}description"))
|
|
403
|
+
if description is None:
|
|
404
|
+
raise ValueError(
|
|
405
|
+
f'Need to define "description" in product '
|
|
406
|
+
f'"{info.product_name}_{algorithm_name}" in product.xml'
|
|
407
|
+
)
|
|
408
|
+
info.title_format = description
|
|
409
|
+
if designator == "none":
|
|
410
|
+
if (text := _text(algorithm_el.find(f"{NS}prod_ix"))) is not None:
|
|
411
|
+
info.prod_ix = int(text)
|
|
412
|
+
info.description = description
|
|
413
|
+
else:
|
|
414
|
+
info.prod_ix = param_val
|
|
415
|
+
info.description = description % info.prod_ix
|
|
416
|
+
|
|
417
|
+
_apply_copy_tags(info, algorithm_el)
|
|
418
|
+
|
|
419
|
+
if (text := _text(algorithm_el.find(f"{NS}type"))) is not None:
|
|
420
|
+
had_explicit_fill = _text(algorithm_el.find(f"{NS}fillValue")) is not None
|
|
421
|
+
info.data_type = text
|
|
422
|
+
if not had_explicit_fill and info.data_type in FILL_VALUE_OVERRIDES:
|
|
423
|
+
info.fill_value = FILL_VALUE_OVERRIDES[info.data_type]
|
|
424
|
+
|
|
425
|
+
_apply_range(info, algorithm_el, param_val)
|
writel2/reader.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Read-only helper for pulling metadata out of an existing L2 (or L1/L3) NetCDF4
|
|
3
|
+
file, primarily so L2Writer can seed a new output file's global attributes
|
|
4
|
+
from a source L2 file (see src/libl2/l2_metadata.cpp for the reference list
|
|
5
|
+
of attributes an L2 consumer cares about).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Iterable, Optional
|
|
11
|
+
|
|
12
|
+
import h5netcdf
|
|
13
|
+
|
|
14
|
+
# Global attributes l2gen writes that are safe/meaningful to copy forward onto
|
|
15
|
+
# a derived L2 product. Run-specific attributes (title, history, date_created,
|
|
16
|
+
# product_name, id, processing_version, ...) are deliberately excluded here -
|
|
17
|
+
# the writer sets its own.
|
|
18
|
+
COPYABLE_GLOBAL_ATTRS = (
|
|
19
|
+
"instrument",
|
|
20
|
+
"platform",
|
|
21
|
+
"Conventions",
|
|
22
|
+
"license",
|
|
23
|
+
"naming_authority",
|
|
24
|
+
"keywords_vocabulary",
|
|
25
|
+
"keywords",
|
|
26
|
+
"standard_name_vocabulary",
|
|
27
|
+
"institution",
|
|
28
|
+
"creator_name",
|
|
29
|
+
"creator_email",
|
|
30
|
+
"creator_url",
|
|
31
|
+
"project",
|
|
32
|
+
"publisher_name",
|
|
33
|
+
"publisher_url",
|
|
34
|
+
"publisher_email",
|
|
35
|
+
"geospatial_bounds_crs",
|
|
36
|
+
"processing_level",
|
|
37
|
+
"cdm_data_type",
|
|
38
|
+
"spatialResolution",
|
|
39
|
+
"source",
|
|
40
|
+
"time_coverage_start",
|
|
41
|
+
"time_coverage_end",
|
|
42
|
+
"westernmost_longitude",
|
|
43
|
+
"easternmost_longitude",
|
|
44
|
+
"southernmost_latitude",
|
|
45
|
+
"northernmost_latitude",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
L2_GROUP_NAMES = ("geophysical_data", "processing_control")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class L2Reader:
|
|
52
|
+
def __init__(self, path):
|
|
53
|
+
self._file = h5netcdf.File(path, "r")
|
|
54
|
+
|
|
55
|
+
def __enter__(self) -> "L2Reader":
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
def __exit__(self, *exc) -> None:
|
|
59
|
+
self.close()
|
|
60
|
+
|
|
61
|
+
def close(self) -> None:
|
|
62
|
+
self._file.close()
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def global_attrs(self) -> dict:
|
|
66
|
+
return dict(self._file.attrs)
|
|
67
|
+
|
|
68
|
+
def group(self, name: str) -> Optional[h5netcdf.Group]:
|
|
69
|
+
return self._file.groups.get(name)
|
|
70
|
+
|
|
71
|
+
def is_l2(self) -> bool:
|
|
72
|
+
return all(name in self._file.groups for name in L2_GROUP_NAMES)
|
|
73
|
+
|
|
74
|
+
def copy_global_attrs_to(self, dest, names: Optional[Iterable[str]] = None) -> None:
|
|
75
|
+
"""Copy global attributes onto `dest` (an h5netcdf File or Group).
|
|
76
|
+
|
|
77
|
+
Defaults to COPYABLE_GLOBAL_ATTRS; pass `names` to copy a different set.
|
|
78
|
+
Attributes absent from this file are silently skipped.
|
|
79
|
+
"""
|
|
80
|
+
wanted = COPYABLE_GLOBAL_ATTRS if names is None else names
|
|
81
|
+
attrs = self.global_attrs
|
|
82
|
+
for name in wanted:
|
|
83
|
+
if name in attrs:
|
|
84
|
+
dest.attrs[name] = attrs[name]
|
writel2/writer.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""
|
|
2
|
+
L2Writer: open a NetCDF4 file and populate it with the standard L2 group
|
|
3
|
+
layout (sensor_band_parameters, scan_line_attributes, geophysical_data,
|
|
4
|
+
navigation_data, processing_control/input_parameters[/flag_percentages]) and
|
|
5
|
+
write variables into any of those groups via write_product(), whose
|
|
6
|
+
attributes (long_name, units, valid_min/max, scale_factor/add_offset,
|
|
7
|
+
_FillValue, reference, comment, ...) come from share/common/product.xml -
|
|
8
|
+
mirroring createDS()/createDS2() in oel_hdf4/libdfutils/wrapper.c, which both
|
|
9
|
+
resolve against product.xml (via findProductInfo()) regardless of which
|
|
10
|
+
group they're writing into. h5netcdf has no auto scale/mask support (unlike
|
|
11
|
+
netCDF4-python/xarray), so packing physical-units data into storage units on
|
|
12
|
+
write is done here explicitly. The file is opened with format="NETCDF4_CLASSIC"
|
|
13
|
+
so plain Python str attributes are stored as NC_CHAR text, matching
|
|
14
|
+
nc_put_att_text()/SetChrGA() in wrapper.c - h5netcdf's default ("NETCDF4")
|
|
15
|
+
would instead store them as NC_STRING, which real l2gen output never uses.
|
|
16
|
+
|
|
17
|
+
One documented exception: geophysical_data's own local "wavelength"
|
|
18
|
+
coordinate variable (used alongside 3D products like Rrs) is written with a
|
|
19
|
+
hardcoded call in src/l2gen/l2_generic.c, bypassing product.xml entirely -
|
|
20
|
+
write_product() can't be used for that one variable.
|
|
21
|
+
|
|
22
|
+
flag_values/flag_meanings and the _FillValue skip for flag_*/l2_flags-named
|
|
23
|
+
variables aren't part of ScaledNcVar (which has no notion of flag products at
|
|
24
|
+
all) - that behavior is carried over from the older oel_hdf4/libdfutils/
|
|
25
|
+
wrapper.c, since it's the only concrete spec available for flag products.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import warnings
|
|
31
|
+
from datetime import datetime, timezone
|
|
32
|
+
from typing import Optional, Sequence
|
|
33
|
+
|
|
34
|
+
import h5netcdf
|
|
35
|
+
import numpy as np
|
|
36
|
+
|
|
37
|
+
from .global_metadata import GlobalMetadata
|
|
38
|
+
from .product_info import (
|
|
39
|
+
DEFAULT_VALID_MAX,
|
|
40
|
+
DEFAULT_VALID_MIN,
|
|
41
|
+
ProductInfo,
|
|
42
|
+
ProductTable,
|
|
43
|
+
)
|
|
44
|
+
from .reader import L2Reader
|
|
45
|
+
|
|
46
|
+
# Level-specific, not part of global_metadata.json: every L2 writer sets these
|
|
47
|
+
# itself (e.g. oel_hdf4/libl1/l1_generic_write.c uses "L1B"/"swath", l3mapgen's
|
|
48
|
+
# OutFileNetcdf4.cpp uses "L3 Mapped"/"grid").
|
|
49
|
+
PROCESSING_LEVEL = "L2"
|
|
50
|
+
CDM_DATA_TYPE = "swath"
|
|
51
|
+
|
|
52
|
+
DTYPE_MAP = {
|
|
53
|
+
"byte": np.int8,
|
|
54
|
+
"ubyte": np.uint8,
|
|
55
|
+
"short": np.int16,
|
|
56
|
+
"ushort": np.uint16,
|
|
57
|
+
"int": np.int32,
|
|
58
|
+
"uint": np.uint32,
|
|
59
|
+
"float": np.float32,
|
|
60
|
+
"double": np.float64,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _is_flag_product(name: str) -> bool:
|
|
65
|
+
return name.startswith("flag_") or name == "l2_flags"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _scale_to_storage(value: float, info: ProductInfo, np_dtype) -> "np.generic":
|
|
69
|
+
"""Scale a *physical-units* value (e.g. valid_min/valid_max) into storage
|
|
70
|
+
units: ROUND((value - add_offset) / scale_factor). Note: scaled_nc_var.cpp's
|
|
71
|
+
populateScaleFactors() writes valid_min/valid_max unscaled (cast straight to
|
|
72
|
+
the storage NcType) - that looks like a bug (e.g. bbp's validMin=5e-06 would
|
|
73
|
+
cast to int16 as 0), so this keeps the scaled, CF-correct behavior instead."""
|
|
74
|
+
if info.scale_factor == 1.0 and info.add_offset == 0.0:
|
|
75
|
+
return _raw_cast(value, np_dtype)
|
|
76
|
+
if np.issubdtype(np_dtype, np.floating):
|
|
77
|
+
return np_dtype(value)
|
|
78
|
+
return np_dtype(round((value - info.add_offset) / info.scale_factor))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _raw_cast(value: float, np_dtype) -> "np.generic":
|
|
82
|
+
"""Cast a value already expressed in storage units (fillValue, flag_values)
|
|
83
|
+
directly to the storage dtype - no scale_factor/add_offset applied."""
|
|
84
|
+
return (
|
|
85
|
+
np_dtype(round(value))
|
|
86
|
+
if not np.issubdtype(np_dtype, np.floating)
|
|
87
|
+
else np_dtype(value)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _pack(data: np.ndarray, info: ProductInfo, np_dtype) -> np.ndarray:
|
|
92
|
+
"""Pack physical-units `data` into storage units. Rounds to nearest (unlike
|
|
93
|
+
scaled_nc_var.cpp's compress(), which truncates) and, like compress(), clamps
|
|
94
|
+
any value outside the storage dtype's representable range to the fill value
|
|
95
|
+
instead of letting it silently wrap on cast."""
|
|
96
|
+
if np.issubdtype(np_dtype, np.floating):
|
|
97
|
+
return data.astype(np_dtype)
|
|
98
|
+
|
|
99
|
+
nonfinite = (
|
|
100
|
+
~np.isfinite(data)
|
|
101
|
+
if np.issubdtype(data.dtype, np.floating)
|
|
102
|
+
else np.zeros(data.shape, dtype=bool)
|
|
103
|
+
)
|
|
104
|
+
if info.scale_factor == 1.0 and info.add_offset == 0.0:
|
|
105
|
+
packed = np.round(data.astype(np.float64))
|
|
106
|
+
else:
|
|
107
|
+
packed = np.round(
|
|
108
|
+
(data.astype(np.float64) - info.add_offset) / info.scale_factor
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
bounds = np.iinfo(np_dtype)
|
|
112
|
+
bad = nonfinite | (packed < bounds.min) | (packed > bounds.max)
|
|
113
|
+
fill = float(_raw_cast(info.fill_value, np_dtype))
|
|
114
|
+
return np.where(bad, fill, packed).astype(np_dtype)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class L2Writer:
|
|
118
|
+
GROUP_NAMES = (
|
|
119
|
+
"sensor_band_parameters",
|
|
120
|
+
"scan_line_attributes",
|
|
121
|
+
"geophysical_data",
|
|
122
|
+
"navigation_data",
|
|
123
|
+
"processing_control",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
path,
|
|
129
|
+
source: Optional[L2Reader] = None,
|
|
130
|
+
wavelengths: Optional[Sequence[float]] = None,
|
|
131
|
+
title: Optional[str] = None,
|
|
132
|
+
global_attrs: Optional[dict] = None,
|
|
133
|
+
product_table: Optional[ProductTable] = None,
|
|
134
|
+
global_metadata: Optional[GlobalMetadata] = None,
|
|
135
|
+
):
|
|
136
|
+
self.file = h5netcdf.File(path, "w", format="NETCDF4_CLASSIC")
|
|
137
|
+
self.groups = {name: self.file.create_group(name) for name in self.GROUP_NAMES}
|
|
138
|
+
self.groups["input_parameters"] = self.groups[
|
|
139
|
+
"processing_control"
|
|
140
|
+
].create_group("input_parameters")
|
|
141
|
+
|
|
142
|
+
self._product_table = product_table or ProductTable()
|
|
143
|
+
self._global_metadata = global_metadata or GlobalMetadata()
|
|
144
|
+
self._wavelengths = wavelengths
|
|
145
|
+
|
|
146
|
+
if source is not None:
|
|
147
|
+
if source.is_l2():
|
|
148
|
+
self.copy_global_attrs_from(source)
|
|
149
|
+
else:
|
|
150
|
+
warnings.warn(
|
|
151
|
+
"source file is not L2-shaped; skipping global attribute copy"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
self._set_standard_global_attrs(title=title)
|
|
155
|
+
if global_attrs:
|
|
156
|
+
for name, value in global_attrs.items():
|
|
157
|
+
self.file.attrs[name] = value
|
|
158
|
+
|
|
159
|
+
def __enter__(self) -> "L2Writer":
|
|
160
|
+
return self
|
|
161
|
+
|
|
162
|
+
def __exit__(self, *exc) -> None:
|
|
163
|
+
self.close()
|
|
164
|
+
|
|
165
|
+
def close(self) -> None:
|
|
166
|
+
self.file.close()
|
|
167
|
+
|
|
168
|
+
def _set_standard_global_attrs(self, title: Optional[str] = None) -> None:
|
|
169
|
+
for name, value in self._global_metadata.items():
|
|
170
|
+
self.file.attrs[name] = value
|
|
171
|
+
self.file.attrs["processing_level"] = PROCESSING_LEVEL
|
|
172
|
+
self.file.attrs["cdm_data_type"] = CDM_DATA_TYPE
|
|
173
|
+
self.file.attrs["date_created"] = datetime.now(timezone.utc).strftime(
|
|
174
|
+
"%Y-%m-%dT%H:%M:%S.000Z"
|
|
175
|
+
)
|
|
176
|
+
if title:
|
|
177
|
+
self.file.attrs["title"] = title
|
|
178
|
+
|
|
179
|
+
def copy_global_attrs_from(
|
|
180
|
+
self, source: L2Reader, names=None, overrides: Optional[dict] = None
|
|
181
|
+
) -> None:
|
|
182
|
+
source.copy_global_attrs_to(self.file, names=names)
|
|
183
|
+
if overrides:
|
|
184
|
+
for name, value in overrides.items():
|
|
185
|
+
self.file.attrs[name] = value
|
|
186
|
+
|
|
187
|
+
def set_input_parameters(self, params: dict) -> None:
|
|
188
|
+
for name, value in params.items():
|
|
189
|
+
self.groups["input_parameters"].attrs[name] = value
|
|
190
|
+
|
|
191
|
+
def flag_percentages_group(self, name: str = "flag_percentages") -> h5netcdf.Group:
|
|
192
|
+
if name not in self.groups["processing_control"].groups:
|
|
193
|
+
self.groups["processing_control"].create_group(name)
|
|
194
|
+
return self.groups["processing_control"].groups[name]
|
|
195
|
+
|
|
196
|
+
def set_flag_percentages(
|
|
197
|
+
self, percentages: dict, group_name: str = "flag_percentages"
|
|
198
|
+
) -> None:
|
|
199
|
+
grp = self.flag_percentages_group(group_name)
|
|
200
|
+
for flag_name, percentage in percentages.items():
|
|
201
|
+
grp.attrs[flag_name] = np.float32(percentage)
|
|
202
|
+
|
|
203
|
+
def _ensure_dim(self, name: str, size: int) -> None:
|
|
204
|
+
if name not in self.file.dimensions:
|
|
205
|
+
self.file.dimensions[name] = size
|
|
206
|
+
|
|
207
|
+
def write_product(
|
|
208
|
+
self,
|
|
209
|
+
name: str,
|
|
210
|
+
data,
|
|
211
|
+
dim_names: Sequence[str] = ("number_of_lines", "pixels_per_line"),
|
|
212
|
+
group: str = "geophysical_data",
|
|
213
|
+
) -> h5netcdf.Variable:
|
|
214
|
+
"""Resolve `name` against product.xml and write it into `group`.
|
|
215
|
+
|
|
216
|
+
`group` defaults to "geophysical_data" but can be any of
|
|
217
|
+
sensor_band_parameters/scan_line_attributes/navigation_data too -
|
|
218
|
+
l2gen resolves all of them against product.xml the same way (see
|
|
219
|
+
module docstring), so the same lookup/packing logic applies regardless
|
|
220
|
+
of which group a product belongs to.
|
|
221
|
+
|
|
222
|
+
`data` is expected in physical units; it is packed into storage units
|
|
223
|
+
using the resolved scale_factor/add_offset when the product's dtype is
|
|
224
|
+
not float/double. NaN/inf entries are written as the product's fill value.
|
|
225
|
+
The written scale_factor/add_offset attributes match `data`'s own
|
|
226
|
+
float precision (float32 or float64).
|
|
227
|
+
"""
|
|
228
|
+
info = self._product_table.find(name, wavelengths=self._wavelengths)
|
|
229
|
+
np_dtype = DTYPE_MAP[info.data_type]
|
|
230
|
+
data = np.asarray(data)
|
|
231
|
+
dims = tuple(dim_names[: data.ndim])
|
|
232
|
+
for dim_name, size in zip(dims, data.shape):
|
|
233
|
+
self._ensure_dim(dim_name, size)
|
|
234
|
+
|
|
235
|
+
kwargs = {}
|
|
236
|
+
if not _is_flag_product(name):
|
|
237
|
+
kwargs["fillvalue"] = _raw_cast(info.fill_value, np_dtype)
|
|
238
|
+
|
|
239
|
+
grp = self.groups[group]
|
|
240
|
+
var = grp.create_variable(
|
|
241
|
+
name,
|
|
242
|
+
dimensions=dims,
|
|
243
|
+
dtype=np_dtype,
|
|
244
|
+
data=_pack(data, info, np_dtype),
|
|
245
|
+
**kwargs,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
var.attrs["long_name"] = info.description
|
|
249
|
+
if not np.issubdtype(np_dtype, np.floating) and (
|
|
250
|
+
info.scale_factor != 1.0 or info.add_offset != 0.0
|
|
251
|
+
):
|
|
252
|
+
# scale_factor/add_offset are stored in whatever float precision
|
|
253
|
+
# the caller's physical-units `data` was given in (float32 ->
|
|
254
|
+
# NC_FLOAT, float64 -> NC_DOUBLE), rather than a fixed precision -
|
|
255
|
+
# product.xml has no type for these attributes themselves, only
|
|
256
|
+
# for the variable's own storage type (np_dtype, above).
|
|
257
|
+
scale_dtype = (
|
|
258
|
+
data.dtype
|
|
259
|
+
if np.issubdtype(data.dtype, np.floating)
|
|
260
|
+
else np.dtype(np.float64)
|
|
261
|
+
)
|
|
262
|
+
var.attrs["scale_factor"] = scale_dtype.type(info.scale_factor)
|
|
263
|
+
var.attrs["add_offset"] = scale_dtype.type(info.add_offset)
|
|
264
|
+
if info.units and info.units.lower() not in (
|
|
265
|
+
"",
|
|
266
|
+
"dimensionless",
|
|
267
|
+
"unitless",
|
|
268
|
+
"none",
|
|
269
|
+
):
|
|
270
|
+
var.attrs["units"] = info.units
|
|
271
|
+
if data.ndim > 1 and name not in ("longitude", "latitude"):
|
|
272
|
+
var.attrs["coordinates"] = "longitude latitude"
|
|
273
|
+
if info.standard_name:
|
|
274
|
+
var.attrs["standard_name"] = info.standard_name
|
|
275
|
+
if info.valid_min != DEFAULT_VALID_MIN or info.valid_max != DEFAULT_VALID_MAX:
|
|
276
|
+
var.attrs["valid_min"] = _scale_to_storage(info.valid_min, info, np_dtype)
|
|
277
|
+
var.attrs["valid_max"] = _scale_to_storage(info.valid_max, info, np_dtype)
|
|
278
|
+
if info.flag_values and info.flag_meanings:
|
|
279
|
+
values = np.array(
|
|
280
|
+
[int(v) for v in info.flag_values.split(",")], dtype=np_dtype
|
|
281
|
+
)
|
|
282
|
+
var.attrs["flag_values"] = values
|
|
283
|
+
var.attrs["flag_meanings"] = info.flag_meanings
|
|
284
|
+
if info.reference:
|
|
285
|
+
var.attrs["reference"] = info.reference
|
|
286
|
+
if info.comment:
|
|
287
|
+
var.attrs["comment"] = info.comment
|
|
288
|
+
|
|
289
|
+
return var
|