cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -2
- cloudnetpy/categorize/atmos_utils.py +297 -67
- cloudnetpy/categorize/attenuation.py +31 -0
- cloudnetpy/categorize/attenuations/__init__.py +37 -0
- cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
- cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
- cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
- cloudnetpy/categorize/categorize.py +332 -156
- cloudnetpy/categorize/classify.py +127 -125
- cloudnetpy/categorize/containers.py +107 -76
- cloudnetpy/categorize/disdrometer.py +40 -0
- cloudnetpy/categorize/droplet.py +23 -21
- cloudnetpy/categorize/falling.py +53 -24
- cloudnetpy/categorize/freezing.py +25 -12
- cloudnetpy/categorize/insects.py +35 -23
- cloudnetpy/categorize/itu.py +243 -0
- cloudnetpy/categorize/lidar.py +36 -41
- cloudnetpy/categorize/melting.py +34 -26
- cloudnetpy/categorize/model.py +84 -37
- cloudnetpy/categorize/mwr.py +18 -14
- cloudnetpy/categorize/radar.py +215 -102
- cloudnetpy/cli.py +578 -0
- cloudnetpy/cloudnetarray.py +43 -89
- cloudnetpy/concat_lib.py +218 -78
- cloudnetpy/constants.py +28 -10
- cloudnetpy/datasource.py +61 -86
- cloudnetpy/exceptions.py +49 -20
- cloudnetpy/instruments/__init__.py +5 -0
- cloudnetpy/instruments/basta.py +29 -12
- cloudnetpy/instruments/bowtie.py +135 -0
- cloudnetpy/instruments/ceilo.py +138 -115
- cloudnetpy/instruments/ceilometer.py +164 -80
- cloudnetpy/instruments/cl61d.py +21 -5
- cloudnetpy/instruments/cloudnet_instrument.py +74 -36
- cloudnetpy/instruments/copernicus.py +108 -30
- cloudnetpy/instruments/da10.py +54 -0
- cloudnetpy/instruments/disdrometer/common.py +126 -223
- cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
- cloudnetpy/instruments/disdrometer/thies.py +254 -87
- cloudnetpy/instruments/fd12p.py +201 -0
- cloudnetpy/instruments/galileo.py +65 -23
- cloudnetpy/instruments/hatpro.py +123 -49
- cloudnetpy/instruments/instruments.py +113 -1
- cloudnetpy/instruments/lufft.py +39 -17
- cloudnetpy/instruments/mira.py +268 -61
- cloudnetpy/instruments/mrr.py +187 -0
- cloudnetpy/instruments/nc_lidar.py +19 -8
- cloudnetpy/instruments/nc_radar.py +109 -55
- cloudnetpy/instruments/pollyxt.py +135 -51
- cloudnetpy/instruments/radiometrics.py +313 -59
- cloudnetpy/instruments/rain_e_h3.py +171 -0
- cloudnetpy/instruments/rpg.py +321 -189
- cloudnetpy/instruments/rpg_reader.py +74 -40
- cloudnetpy/instruments/toa5.py +49 -0
- cloudnetpy/instruments/vaisala.py +95 -343
- cloudnetpy/instruments/weather_station.py +774 -105
- cloudnetpy/metadata.py +90 -19
- cloudnetpy/model_evaluation/file_handler.py +55 -52
- cloudnetpy/model_evaluation/metadata.py +46 -20
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
- cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
- cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
- cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
- cloudnetpy/model_evaluation/products/model_products.py +43 -35
- cloudnetpy/model_evaluation/products/observation_products.py +41 -35
- cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
- cloudnetpy/model_evaluation/products/tools.py +29 -20
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
- cloudnetpy/model_evaluation/utils.py +2 -1
- cloudnetpy/output.py +170 -111
- cloudnetpy/plotting/__init__.py +2 -1
- cloudnetpy/plotting/plot_meta.py +562 -822
- cloudnetpy/plotting/plotting.py +1142 -704
- cloudnetpy/products/__init__.py +1 -0
- cloudnetpy/products/classification.py +370 -88
- cloudnetpy/products/der.py +85 -55
- cloudnetpy/products/drizzle.py +77 -34
- cloudnetpy/products/drizzle_error.py +15 -11
- cloudnetpy/products/drizzle_tools.py +79 -59
- cloudnetpy/products/epsilon.py +211 -0
- cloudnetpy/products/ier.py +27 -50
- cloudnetpy/products/iwc.py +55 -48
- cloudnetpy/products/lwc.py +96 -70
- cloudnetpy/products/mwr_tools.py +186 -0
- cloudnetpy/products/product_tools.py +170 -128
- cloudnetpy/utils.py +455 -240
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
- cloudnetpy-1.87.3.dist-info/RECORD +127 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
- cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
- docs/source/conf.py +2 -2
- cloudnetpy/categorize/atmos.py +0 -361
- cloudnetpy/products/mwr_multi.py +0 -68
- cloudnetpy/products/mwr_single.py +0 -75
- cloudnetpy-1.49.9.dist-info/RECORD +0 -112
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
cloudnetpy/instruments/rpg.py
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
import datetime
|
|
2
2
|
import logging
|
|
3
3
|
import math
|
|
4
|
-
from
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from os import PathLike
|
|
6
|
+
from uuid import UUID
|
|
5
7
|
|
|
6
8
|
import numpy as np
|
|
9
|
+
import numpy.typing as npt
|
|
7
10
|
from numpy import ma
|
|
11
|
+
from rpgpy import RPGFileError
|
|
8
12
|
|
|
9
13
|
from cloudnetpy import output, utils
|
|
10
|
-
from cloudnetpy.categorize.atmos_utils import mmh2ms
|
|
11
14
|
from cloudnetpy.cloudnetarray import CloudnetArray
|
|
12
|
-
from cloudnetpy.
|
|
15
|
+
from cloudnetpy.constants import G_TO_KG, HPA_TO_PA, KM_H_TO_M_S, MM_H_TO_M_S
|
|
16
|
+
from cloudnetpy.exceptions import ValidTimeStampError
|
|
13
17
|
from cloudnetpy.instruments import instruments
|
|
14
18
|
from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
|
|
15
19
|
from cloudnetpy.instruments.instruments import Instrument
|
|
@@ -18,12 +22,12 @@ from cloudnetpy.metadata import MetaData
|
|
|
18
22
|
|
|
19
23
|
|
|
20
24
|
def rpg2nc(
|
|
21
|
-
path_to_l1_files: str,
|
|
22
|
-
output_file: str,
|
|
25
|
+
path_to_l1_files: str | PathLike,
|
|
26
|
+
output_file: str | PathLike,
|
|
23
27
|
site_meta: dict,
|
|
24
|
-
uuid: str | None = None,
|
|
25
|
-
date: str | None = None,
|
|
26
|
-
) -> tuple[
|
|
28
|
+
uuid: str | UUID | None = None,
|
|
29
|
+
date: str | datetime.date | None = None,
|
|
30
|
+
) -> tuple[UUID, list[str]]:
|
|
27
31
|
"""Converts RPG-FMCW-94 cloud radar data into Cloudnet Level 1b netCDF file.
|
|
28
32
|
|
|
29
33
|
This function reads one day of RPG Level 1 cloud radar binary files,
|
|
@@ -56,12 +60,14 @@ def rpg2nc(
|
|
|
56
60
|
>>> rpg2nc('/path/to/files/', 'test.nc', site_meta)
|
|
57
61
|
|
|
58
62
|
"""
|
|
63
|
+
if isinstance(date, str):
|
|
64
|
+
date = datetime.date.fromisoformat(date)
|
|
65
|
+
uuid = utils.get_uuid(uuid)
|
|
59
66
|
l1_files = utils.get_sorted_filenames(path_to_l1_files, ".LV1")
|
|
60
67
|
fmcw94_objects, valid_files = _get_fmcw94_objects(l1_files, date)
|
|
61
68
|
one_day_of_data = create_one_day_data_record(fmcw94_objects)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
print_info(one_day_of_data)
|
|
69
|
+
one_day_of_data["nyquist_velocity"] = _expand_nyquist(one_day_of_data)
|
|
70
|
+
_print_info(one_day_of_data)
|
|
65
71
|
fmcw = Fmcw(one_day_of_data, site_meta)
|
|
66
72
|
fmcw.convert_time_to_fraction_hour()
|
|
67
73
|
fmcw.mask_invalid_ldr()
|
|
@@ -71,15 +77,20 @@ def rpg2nc(
|
|
|
71
77
|
fmcw.linear_to_db(("Zh", "antenna_gain"))
|
|
72
78
|
fmcw.convert_units()
|
|
73
79
|
fmcw.add_site_geolocation()
|
|
74
|
-
fmcw.add_zenith_angle()
|
|
80
|
+
valid_ind = fmcw.add_zenith_angle()
|
|
81
|
+
fmcw.screen_time_indices(valid_ind)
|
|
75
82
|
fmcw.add_height()
|
|
83
|
+
if len(np.unique(fmcw.data["time"][:].astype("f4"))) != len(fmcw.data["time"][:]):
|
|
84
|
+
msg = "Convert time to f8 to keep values unique in netCDF"
|
|
85
|
+
logging.info(msg)
|
|
86
|
+
fmcw.data["time"].data_type = "f8"
|
|
76
87
|
attributes = output.add_time_attribute(RPG_ATTRIBUTES, fmcw.date)
|
|
77
88
|
output.update_attributes(fmcw.data, attributes)
|
|
78
|
-
|
|
89
|
+
output.save_level1b(fmcw, output_file, uuid)
|
|
79
90
|
return uuid, valid_files
|
|
80
91
|
|
|
81
92
|
|
|
82
|
-
def
|
|
93
|
+
def _print_info(data: dict) -> None:
|
|
83
94
|
dual_pol = data["dual_polarization"]
|
|
84
95
|
if dual_pol == 0:
|
|
85
96
|
mode = "single polarisation"
|
|
@@ -87,65 +98,88 @@ def print_info(data: dict) -> None:
|
|
|
87
98
|
mode = "LDR"
|
|
88
99
|
else:
|
|
89
100
|
mode = "STSR"
|
|
90
|
-
logging.info(
|
|
101
|
+
logging.info("RPG cloud radar in %s mode", mode)
|
|
91
102
|
|
|
92
103
|
|
|
93
104
|
RpgObjects = Sequence[Fmcw94Bin] | Sequence[HatproBinCombined]
|
|
94
105
|
|
|
95
106
|
|
|
96
107
|
def create_one_day_data_record(rpg_objects: RpgObjects) -> dict:
|
|
97
|
-
"""Concatenates all RPG data from one day."""
|
|
108
|
+
"""Concatenates all RPG FMCW / HATPRO data from one day."""
|
|
98
109
|
rpg_raw_data, rpg_header = _stack_rpg_data(rpg_objects)
|
|
99
|
-
if
|
|
100
|
-
rpg_header =
|
|
110
|
+
if "range" in rpg_header:
|
|
111
|
+
rpg_header["range"] = rpg_objects[0].header["range"]
|
|
112
|
+
should_be_constant = [
|
|
113
|
+
"customer_name",
|
|
114
|
+
"model_number",
|
|
115
|
+
"dual_polarization",
|
|
116
|
+
"antenna_separation",
|
|
117
|
+
"antenna_diameter",
|
|
118
|
+
"antenna_gain",
|
|
119
|
+
"half_power_beam_width",
|
|
120
|
+
"radar_frequency",
|
|
121
|
+
]
|
|
122
|
+
to_be_removed = ["customer_name"]
|
|
123
|
+
for key in should_be_constant:
|
|
124
|
+
if key not in rpg_header:
|
|
125
|
+
continue
|
|
126
|
+
unique_values = np.unique(rpg_header[key])
|
|
127
|
+
if len(unique_values) > 1:
|
|
128
|
+
msg = f"More than one value for {key} found: {unique_values}"
|
|
129
|
+
raise ValueError(msg)
|
|
130
|
+
if key in to_be_removed:
|
|
131
|
+
del rpg_header[key]
|
|
132
|
+
else:
|
|
133
|
+
rpg_header[key] = unique_values[0]
|
|
134
|
+
|
|
101
135
|
rpg_raw_data = _mask_invalid_data(rpg_raw_data)
|
|
102
136
|
return {**rpg_header, **rpg_raw_data}
|
|
103
137
|
|
|
104
138
|
|
|
105
|
-
def
|
|
106
|
-
"""
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
139
|
+
def _expand_nyquist(data: dict) -> npt.NDArray:
|
|
140
|
+
"""Expands Nyquist velocity from time X chirp => time X range."""
|
|
141
|
+
nyquist_velocity = ma.array(data["nyquist_velocity"])
|
|
142
|
+
chirp_start_indices = ma.array(data["chirp_start_indices"])
|
|
143
|
+
n_time = chirp_start_indices.shape[0]
|
|
144
|
+
n_range = len(data["range"])
|
|
145
|
+
expanded_nyquist = np.empty((n_time, n_range))
|
|
146
|
+
for t in range(n_time):
|
|
147
|
+
starts = chirp_start_indices[t].compressed()
|
|
148
|
+
v_nyq = nyquist_velocity[t].compressed()
|
|
149
|
+
ends = np.r_[starts[1:], n_range]
|
|
150
|
+
seg_lengths = ends - starts
|
|
151
|
+
expanded_nyquist[t, :] = np.repeat(v_nyq, seg_lengths)
|
|
152
|
+
return expanded_nyquist
|
|
112
153
|
|
|
113
|
-
def _stack(source, target, fun):
|
|
114
|
-
for name, value in source.items():
|
|
115
|
-
if not name.startswith("_"):
|
|
116
|
-
target[name] = fun((target[name], value)) if name in target else value
|
|
117
154
|
|
|
155
|
+
def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
|
|
118
156
|
data: dict = {}
|
|
119
157
|
header: dict = {}
|
|
120
158
|
for rpg in rpg_objects:
|
|
121
|
-
|
|
122
|
-
|
|
159
|
+
for src, dst in ((rpg.data, data), (rpg.header, header)):
|
|
160
|
+
for name, value in src.items():
|
|
161
|
+
if name.startswith("_"):
|
|
162
|
+
continue
|
|
163
|
+
arr = dst.get(name)
|
|
164
|
+
fun = (
|
|
165
|
+
ma.concatenate
|
|
166
|
+
if any(isinstance(x, ma.MaskedArray) for x in (value, arr))
|
|
167
|
+
else np.concatenate
|
|
168
|
+
)
|
|
169
|
+
dst[name] = fun((arr, value)) if arr is not None else value
|
|
123
170
|
return data, header
|
|
124
171
|
|
|
125
172
|
|
|
126
|
-
def _reduce_header(header: dict) -> dict:
|
|
127
|
-
"""Removes duplicate header data. Otherwise, we would need n_files dimension."""
|
|
128
|
-
reduced_header = {}
|
|
129
|
-
for key, data in header.items():
|
|
130
|
-
first_profile_value = data[0]
|
|
131
|
-
is_identical_value = bool(
|
|
132
|
-
np.isclose(data, first_profile_value, rtol=1e-2).all()
|
|
133
|
-
)
|
|
134
|
-
if is_identical_value is False:
|
|
135
|
-
msg = f"Inconsistent header: {key}"
|
|
136
|
-
if key in ("latitude", "longitude", "sample_duration"):
|
|
137
|
-
logging.warning(f"{msg}: {data}")
|
|
138
|
-
else:
|
|
139
|
-
raise InconsistentDataError(msg)
|
|
140
|
-
reduced_header[key] = first_profile_value
|
|
141
|
-
return reduced_header
|
|
142
|
-
|
|
143
|
-
|
|
144
173
|
def _mask_invalid_data(data_in: dict) -> dict:
|
|
145
174
|
"""Masks zeros and other fill values from data."""
|
|
146
175
|
data = data_in.copy()
|
|
147
176
|
fill_values = (-999, 1e-10)
|
|
177
|
+
extra_keys = ("air_temperature", "air_pressure")
|
|
148
178
|
for name in data:
|
|
179
|
+
if np.issubdtype(data[name].dtype, np.integer) or (
|
|
180
|
+
data[name].ndim < 2 and name not in extra_keys
|
|
181
|
+
):
|
|
182
|
+
continue
|
|
149
183
|
data[name] = ma.masked_equal(data[name], 0)
|
|
150
184
|
for value in fill_values:
|
|
151
185
|
data[name][data[name] == value] = ma.masked
|
|
@@ -154,7 +188,9 @@ def _mask_invalid_data(data_in: dict) -> dict:
|
|
|
154
188
|
return data
|
|
155
189
|
|
|
156
190
|
|
|
157
|
-
def _get_fmcw94_objects(
|
|
191
|
+
def _get_fmcw94_objects(
|
|
192
|
+
files: list[str], expected_date: datetime.date | None
|
|
193
|
+
) -> tuple[list[Fmcw94Bin], list[str]]:
|
|
158
194
|
"""Creates a list of Rpg() objects from the file names."""
|
|
159
195
|
objects = []
|
|
160
196
|
valid_files = []
|
|
@@ -163,46 +199,98 @@ def _get_fmcw94_objects(files: list, expected_date: str | None) -> tuple[list, l
|
|
|
163
199
|
obj = Fmcw94Bin(file)
|
|
164
200
|
if expected_date is not None:
|
|
165
201
|
_validate_date(obj, expected_date)
|
|
166
|
-
except (TypeError, ValueError, IndexError) as err:
|
|
202
|
+
except (RPGFileError, TypeError, ValueError, IndexError) as err:
|
|
167
203
|
logging.warning(err)
|
|
168
204
|
continue
|
|
169
205
|
objects.append(obj)
|
|
170
206
|
valid_files.append(file)
|
|
171
|
-
if objects:
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
207
|
+
if not objects:
|
|
208
|
+
msg = "No valid files found"
|
|
209
|
+
raise ValidTimeStampError(msg)
|
|
210
|
+
objects = _interpolate_to_common_height(objects)
|
|
211
|
+
objects = _pad_chirp_related_fields(objects)
|
|
212
|
+
objects = _expand_time_related_fields(objects)
|
|
175
213
|
return objects, valid_files
|
|
176
214
|
|
|
177
215
|
|
|
178
|
-
def
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
216
|
+
def _interpolate_to_common_height(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
|
|
217
|
+
range_arrays = [obj.header["range"] for obj in objects]
|
|
218
|
+
if all(np.array_equal(range_arrays[0], r) for r in range_arrays[1:]):
|
|
219
|
+
return objects
|
|
220
|
+
# Use range with the highest range gate for interpolation
|
|
221
|
+
target_range = max(range_arrays, key=lambda r: r[-1])
|
|
222
|
+
for obj in objects:
|
|
223
|
+
src_range = obj.header["range"]
|
|
224
|
+
if np.array_equal(src_range, target_range):
|
|
225
|
+
continue
|
|
226
|
+
for key, arr in obj.data.items():
|
|
227
|
+
if arr.ndim == 2 and arr.shape[1] == src_range.size:
|
|
228
|
+
obj.data[key] = utils.interpolate_2D_along_y(
|
|
229
|
+
src_range, arr, target_range
|
|
230
|
+
)
|
|
231
|
+
_interpolate_chirp_start_indices(obj, target_range)
|
|
232
|
+
obj.header["range"] = target_range
|
|
233
|
+
return objects
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _interpolate_chirp_start_indices(obj: Fmcw94Bin, target_range: np.ndarray) -> None:
|
|
237
|
+
range_orig = obj.header["range"]
|
|
238
|
+
vals = range_orig[obj.header["chirp_start_indices"]]
|
|
239
|
+
indices = np.abs(target_range[:, None] - vals).argmin(axis=0)
|
|
240
|
+
# Chirp start indices should always start from 0:
|
|
241
|
+
if indices[0] != 0:
|
|
242
|
+
indices[0] = 0
|
|
243
|
+
obj.header["chirp_start_indices"] = indices
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _pad_chirp_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
|
|
247
|
+
"""Pads chirp-related header fields with masked values to have the same length."""
|
|
248
|
+
chirp_lens = [len(obj.header["chirp_start_indices"]) for obj in objects]
|
|
249
|
+
if all(chirp_lens[0] == length for length in chirp_lens[1:]):
|
|
250
|
+
return objects
|
|
251
|
+
max_chirp_len = max(chirp_lens)
|
|
252
|
+
for obj in objects:
|
|
253
|
+
n_chirps = len(obj.header["chirp_start_indices"])
|
|
254
|
+
if n_chirps == max_chirp_len:
|
|
255
|
+
continue
|
|
256
|
+
for key, arr in obj.header.items():
|
|
257
|
+
if not isinstance(arr, str) and arr.ndim == 1 and arr.size == n_chirps:
|
|
258
|
+
pad_len = max_chirp_len - n_chirps
|
|
259
|
+
masked_arr = ma.array(arr, dtype=arr.dtype)
|
|
260
|
+
pad = ma.masked_all(pad_len, dtype=arr.dtype)
|
|
261
|
+
obj.header[key] = ma.concatenate([masked_arr, pad])
|
|
262
|
+
return objects
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _expand_time_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
|
|
266
|
+
for obj in objects:
|
|
267
|
+
n_time = obj.data["time"].size
|
|
268
|
+
for key in obj.header:
|
|
269
|
+
if key in ("range", "time") or key.startswith("_"):
|
|
270
|
+
continue
|
|
271
|
+
arr = obj.header[key]
|
|
272
|
+
# Handle outliers in latitude and longitude (e.g. Galati 2024-02-11):
|
|
273
|
+
if key in ("latitude", "longitude"):
|
|
274
|
+
arr = ma.median(arr)
|
|
275
|
+
if utils.isscalar(arr):
|
|
276
|
+
obj.header[key] = np.repeat(arr, n_time)
|
|
277
|
+
else:
|
|
278
|
+
obj.header[key] = np.tile(arr, (n_time, 1))
|
|
279
|
+
return objects
|
|
193
280
|
|
|
194
281
|
|
|
195
|
-
def _validate_date(obj, expected_date:
|
|
282
|
+
def _validate_date(obj: Fmcw94Bin, expected_date: datetime.date) -> None:
|
|
196
283
|
for t in obj.data["time"][:]:
|
|
197
|
-
|
|
198
|
-
if
|
|
199
|
-
|
|
284
|
+
date = utils.seconds2date(t).date()
|
|
285
|
+
if date != expected_date:
|
|
286
|
+
msg = "Ignoring a file (time stamps not what expected)"
|
|
287
|
+
raise ValueError(msg)
|
|
200
288
|
|
|
201
289
|
|
|
202
290
|
class Rpg(CloudnetInstrument):
|
|
203
291
|
"""Base class for RPG FMCW-94 cloud radar and HATPRO mwr."""
|
|
204
292
|
|
|
205
|
-
def __init__(self, raw_data: dict, site_meta: dict):
|
|
293
|
+
def __init__(self, raw_data: dict, site_meta: dict) -> None:
|
|
206
294
|
super().__init__()
|
|
207
295
|
self.raw_data = raw_data
|
|
208
296
|
self.site_meta = site_meta
|
|
@@ -212,17 +300,21 @@ class Rpg(CloudnetInstrument):
|
|
|
212
300
|
|
|
213
301
|
def convert_time_to_fraction_hour(self, data_type: str | None = None) -> None:
|
|
214
302
|
"""Converts time to fraction hour."""
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
303
|
+
ms2s = 1e-3
|
|
304
|
+
total_time_sec = self.raw_data["time"] + self.raw_data.get("time_ms", 0) * ms2s
|
|
305
|
+
fraction_hour = utils.seconds2hours(total_time_sec)
|
|
306
|
+
|
|
307
|
+
self.data["time"] = CloudnetArray(
|
|
308
|
+
np.array(fraction_hour),
|
|
309
|
+
"time",
|
|
310
|
+
data_type=data_type,
|
|
219
311
|
)
|
|
220
312
|
|
|
221
|
-
def _get_date(self) ->
|
|
313
|
+
def _get_date(self) -> datetime.date:
|
|
222
314
|
time_first = self.raw_data["time"][0]
|
|
223
315
|
time_last = self.raw_data["time"][-1]
|
|
224
|
-
date_first = utils.seconds2date(time_first)
|
|
225
|
-
date_last = utils.seconds2date(time_last)
|
|
316
|
+
date_first = utils.seconds2date(time_first).date()
|
|
317
|
+
date_last = utils.seconds2date(time_last).date()
|
|
226
318
|
if date_first != date_last:
|
|
227
319
|
logging.warning("Measurements from different days")
|
|
228
320
|
return date_first
|
|
@@ -237,7 +329,7 @@ class Rpg(CloudnetInstrument):
|
|
|
237
329
|
class Fmcw(Rpg):
|
|
238
330
|
"""Class for RPG cloud radars."""
|
|
239
331
|
|
|
240
|
-
def __init__(self, raw_data: dict, site_properties: dict):
|
|
332
|
+
def __init__(self, raw_data: dict, site_properties: dict) -> None:
|
|
241
333
|
super().__init__(raw_data, site_properties)
|
|
242
334
|
self.instrument = self._get_instrument(raw_data)
|
|
243
335
|
|
|
@@ -246,7 +338,8 @@ class Fmcw(Rpg):
|
|
|
246
338
|
threshold = -35
|
|
247
339
|
if "ldr" in self.data:
|
|
248
340
|
self.data["ldr"].data = ma.masked_less_equal(
|
|
249
|
-
self.data["ldr"].data,
|
|
341
|
+
self.data["ldr"].data,
|
|
342
|
+
threshold,
|
|
250
343
|
)
|
|
251
344
|
|
|
252
345
|
def mask_invalid_width(self) -> None:
|
|
@@ -262,211 +355,250 @@ class Fmcw(Rpg):
|
|
|
262
355
|
self.data[key].data[ind] = ma.masked
|
|
263
356
|
|
|
264
357
|
def add_zenith_angle(self) -> list:
|
|
265
|
-
"""Adds zenith angle and returns time indices
|
|
358
|
+
"""Adds zenith angle and returns time indices with valid zenith angle."""
|
|
266
359
|
elevation = self.data["elevation"].data
|
|
267
360
|
zenith = 90 - elevation
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
361
|
+
is_valid_zenith = _filter_zenith_angle(zenith)
|
|
362
|
+
n_removed = len(is_valid_zenith) - np.count_nonzero(is_valid_zenith)
|
|
363
|
+
if n_removed == len(zenith):
|
|
364
|
+
msg = "No profiles with valid zenith angle"
|
|
365
|
+
raise ValidTimeStampError(msg)
|
|
273
366
|
if n_removed > 0:
|
|
274
367
|
logging.warning(
|
|
275
|
-
|
|
368
|
+
"Filtering %s profiles due to invalid zenith angle",
|
|
369
|
+
n_removed,
|
|
276
370
|
)
|
|
277
371
|
self.data["zenith_angle"] = CloudnetArray(zenith, "zenith_angle")
|
|
278
372
|
del self.data["elevation"]
|
|
279
|
-
return list(
|
|
373
|
+
return list(is_valid_zenith)
|
|
280
374
|
|
|
281
|
-
def convert_units(self):
|
|
375
|
+
def convert_units(self) -> None:
|
|
282
376
|
"""Converts units."""
|
|
283
|
-
self.data["rainfall_rate"].data =
|
|
284
|
-
self.data["lwp"].data *=
|
|
377
|
+
self.data["rainfall_rate"].data = self.data["rainfall_rate"].data * MM_H_TO_M_S
|
|
378
|
+
self.data["lwp"].data *= G_TO_KG
|
|
379
|
+
self.data["relative_humidity"].data /= 100
|
|
380
|
+
self.data["air_pressure"].data *= HPA_TO_PA
|
|
381
|
+
self.data["wind_speed"].data *= KM_H_TO_M_S
|
|
285
382
|
|
|
286
383
|
@staticmethod
|
|
287
|
-
def _get_instrument(data: dict):
|
|
384
|
+
def _get_instrument(data: dict) -> Instrument:
|
|
288
385
|
frequency = data["radar_frequency"]
|
|
289
386
|
if math.isclose(frequency, 35, abs_tol=0.1):
|
|
290
387
|
return instruments.FMCW35
|
|
291
388
|
if math.isclose(frequency, 94, abs_tol=0.1):
|
|
292
389
|
return instruments.FMCW94
|
|
293
|
-
|
|
390
|
+
msg = f"Unknown RPG cloud radar frequency: {frequency}"
|
|
391
|
+
raise RuntimeError(msg)
|
|
294
392
|
|
|
295
393
|
|
|
296
394
|
class Hatpro(Rpg):
|
|
297
395
|
"""Class for RPG HATPRO mwr."""
|
|
298
396
|
|
|
299
|
-
def __init__(
|
|
397
|
+
def __init__(
|
|
398
|
+
self, raw_data: dict, site_properties: dict, instrument: Instrument
|
|
399
|
+
) -> None:
|
|
300
400
|
super().__init__(raw_data, site_properties)
|
|
301
|
-
self.instrument =
|
|
401
|
+
self.instrument = instrument
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _filter_zenith_angle(zenith: ma.MaskedArray) -> npt.NDArray:
|
|
405
|
+
"""Returns indices of profiles with stable zenith angle close to 0 deg."""
|
|
406
|
+
zenith = ma.array(zenith)
|
|
407
|
+
if zenith.mask.all():
|
|
408
|
+
return np.zeros(zenith.shape, dtype=bool)
|
|
409
|
+
limits = [-5, 15]
|
|
410
|
+
ind_close_to_zenith = np.where(
|
|
411
|
+
np.logical_and(zenith > limits[0], zenith < limits[1]),
|
|
412
|
+
)
|
|
413
|
+
if not ind_close_to_zenith[0].size:
|
|
414
|
+
return np.zeros_like(zenith, dtype=bool)
|
|
415
|
+
valid_range_median = ma.median(zenith[ind_close_to_zenith])
|
|
416
|
+
is_stable_zenith = np.isclose(zenith, valid_range_median, atol=0.1)
|
|
417
|
+
is_stable_zenith[zenith.mask] = False
|
|
418
|
+
return np.array(is_stable_zenith)
|
|
302
419
|
|
|
303
420
|
|
|
304
421
|
DEFINITIONS = {
|
|
305
|
-
"model_number":
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
422
|
+
"model_number": utils.status_field_definition(
|
|
423
|
+
{
|
|
424
|
+
0: "Single polarisation radar.",
|
|
425
|
+
1: "Dual polarisation radar.",
|
|
426
|
+
}
|
|
427
|
+
),
|
|
428
|
+
"dual_polarization": utils.status_field_definition(
|
|
429
|
+
{
|
|
430
|
+
0: """Single polarisation radar.""",
|
|
431
|
+
1: """Dual polarisation radar in linear depolarisation ratio (LDR)
|
|
432
|
+
mode.""",
|
|
433
|
+
2: """Dual polarisation radar in simultaneous transmission
|
|
434
|
+
simultaneous reception (STSR) mode.""",
|
|
435
|
+
}
|
|
436
|
+
),
|
|
437
|
+
"FFT_window": utils.status_field_definition(
|
|
438
|
+
{
|
|
439
|
+
0: "Square",
|
|
440
|
+
1: "Parzen",
|
|
441
|
+
2: "Blackman",
|
|
442
|
+
3: "Welch",
|
|
443
|
+
4: "Slepian2",
|
|
444
|
+
5: "Slepian3",
|
|
445
|
+
}
|
|
446
|
+
),
|
|
447
|
+
"quality_flag": utils.bit_field_definition(
|
|
448
|
+
{
|
|
449
|
+
0: "ADC saturation.",
|
|
450
|
+
1: "Spectral width too high.",
|
|
451
|
+
2: "No transmission power levelling.",
|
|
452
|
+
}
|
|
329
453
|
),
|
|
330
454
|
}
|
|
331
455
|
|
|
332
456
|
RPG_ATTRIBUTES = {
|
|
333
457
|
# LDR-mode radars:
|
|
334
|
-
"ldr": MetaData(
|
|
335
|
-
|
|
336
|
-
|
|
458
|
+
"ldr": MetaData(
|
|
459
|
+
long_name="Linear depolarisation ratio",
|
|
460
|
+
units="dB",
|
|
461
|
+
dimensions=("time", "range"),
|
|
462
|
+
),
|
|
463
|
+
"rho_cx": MetaData(
|
|
464
|
+
long_name="Co-cross-channel correlation coefficient",
|
|
465
|
+
units="1",
|
|
466
|
+
dimensions=("time", "range"),
|
|
467
|
+
),
|
|
468
|
+
"phi_cx": MetaData(
|
|
469
|
+
long_name="Co-cross-channel differential phase",
|
|
470
|
+
units="rad",
|
|
471
|
+
dimensions=("time", "range"),
|
|
472
|
+
),
|
|
337
473
|
# STSR-mode radars
|
|
338
|
-
"zdr": MetaData(
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
"
|
|
342
|
-
|
|
474
|
+
"zdr": MetaData(
|
|
475
|
+
long_name="Differential reflectivity", units="dB", dimensions=("time", "range")
|
|
476
|
+
),
|
|
477
|
+
"rho_hv": MetaData(
|
|
478
|
+
long_name="Correlation coefficient", units="1", dimensions=("time", "range")
|
|
479
|
+
),
|
|
480
|
+
"phi_dp": MetaData(
|
|
481
|
+
long_name="Differential phase", units="rad", dimensions=("time", "range")
|
|
482
|
+
),
|
|
483
|
+
"srho_hv": MetaData(
|
|
484
|
+
long_name="Slanted correlation coefficient",
|
|
485
|
+
units="1",
|
|
486
|
+
dimensions=("time", "range"),
|
|
487
|
+
),
|
|
488
|
+
"kdp": MetaData(
|
|
489
|
+
long_name="Specific differential phase shift",
|
|
490
|
+
units="rad km-1",
|
|
491
|
+
dimensions=("time", "range"),
|
|
492
|
+
),
|
|
343
493
|
"differential_attenuation": MetaData(
|
|
344
|
-
long_name="Differential attenuation",
|
|
494
|
+
long_name="Differential attenuation",
|
|
495
|
+
units="dB km-1",
|
|
496
|
+
dimensions=("time", "range"),
|
|
345
497
|
),
|
|
346
498
|
# All radars
|
|
347
499
|
"file_code": MetaData(
|
|
348
500
|
long_name="File code",
|
|
349
501
|
units="1",
|
|
350
502
|
comment="Indicates the RPG software version.",
|
|
503
|
+
dimensions=("time",),
|
|
504
|
+
),
|
|
505
|
+
"program_number": MetaData(
|
|
506
|
+
long_name="Program number", units="1", dimensions=("time",)
|
|
351
507
|
),
|
|
352
|
-
"program_number": MetaData(long_name="Program number", units="1"),
|
|
353
508
|
"model_number": MetaData(
|
|
354
509
|
long_name="Model number",
|
|
355
510
|
units="1",
|
|
356
511
|
definition=DEFINITIONS["model_number"],
|
|
512
|
+
dimensions=None,
|
|
357
513
|
),
|
|
358
514
|
"antenna_separation": MetaData(
|
|
359
|
-
long_name="Antenna separation",
|
|
360
|
-
units="m",
|
|
515
|
+
long_name="Antenna separation", units="m", dimensions=None
|
|
361
516
|
),
|
|
362
517
|
"antenna_diameter": MetaData(
|
|
363
|
-
long_name="Antenna diameter",
|
|
364
|
-
units="m",
|
|
365
|
-
),
|
|
366
|
-
"antenna_gain": MetaData(
|
|
367
|
-
long_name="Antenna gain",
|
|
368
|
-
units="dB",
|
|
518
|
+
long_name="Antenna diameter", units="m", dimensions=None
|
|
369
519
|
),
|
|
520
|
+
"antenna_gain": MetaData(long_name="Antenna gain", units="dB", dimensions=None),
|
|
370
521
|
"half_power_beam_width": MetaData(
|
|
371
|
-
long_name="Half power beam width",
|
|
372
|
-
units="degree",
|
|
522
|
+
long_name="Half power beam width", units="degree", dimensions=None
|
|
373
523
|
),
|
|
374
524
|
"dual_polarization": MetaData(
|
|
375
525
|
long_name="Dual polarisation type",
|
|
376
526
|
units="1",
|
|
377
527
|
definition=DEFINITIONS["dual_polarization"],
|
|
528
|
+
dimensions=None,
|
|
529
|
+
),
|
|
530
|
+
"sample_duration": MetaData(
|
|
531
|
+
long_name="Sample duration", units="s", dimensions=("time",)
|
|
378
532
|
),
|
|
379
|
-
"sample_duration": MetaData(long_name="Sample duration", units="s"),
|
|
380
533
|
"calibration_interval": MetaData(
|
|
381
|
-
long_name="Calibration interval in samples",
|
|
382
|
-
units="1",
|
|
534
|
+
long_name="Calibration interval in samples", units="1", dimensions=("time",)
|
|
383
535
|
),
|
|
384
536
|
"number_of_spectral_samples": MetaData(
|
|
385
537
|
long_name="Number of spectral samples in each chirp sequence",
|
|
386
538
|
units="1",
|
|
387
|
-
|
|
388
|
-
"chirp_start_indices": MetaData(
|
|
389
|
-
long_name="Chirp sequences start indices",
|
|
390
|
-
units="1",
|
|
539
|
+
dimensions=("time", "chirp_sequence"),
|
|
391
540
|
),
|
|
392
541
|
"number_of_averaged_chirps": MetaData(
|
|
393
542
|
long_name="Number of averaged chirps in sequence",
|
|
394
543
|
units="1",
|
|
544
|
+
dimensions=("time", "chirp_sequence"),
|
|
545
|
+
),
|
|
546
|
+
"chirp_start_indices": MetaData(
|
|
547
|
+
long_name="Chirp sequences start indices",
|
|
548
|
+
units="1",
|
|
549
|
+
dimensions=("time", "chirp_sequence"),
|
|
395
550
|
),
|
|
396
551
|
"integration_time": MetaData(
|
|
397
552
|
long_name="Integration time",
|
|
398
553
|
units="s",
|
|
399
554
|
comment="Effective integration time of chirp sequence",
|
|
555
|
+
dimensions=("time", "chirp_sequence"),
|
|
400
556
|
),
|
|
401
557
|
"range_resolution": MetaData(
|
|
402
558
|
long_name="Vertical resolution of range",
|
|
403
559
|
units="m",
|
|
560
|
+
dimensions=("time", "chirp_sequence"),
|
|
404
561
|
),
|
|
405
562
|
"FFT_window": MetaData(
|
|
406
563
|
long_name="FFT window type",
|
|
407
564
|
units="1",
|
|
408
565
|
definition=DEFINITIONS["FFT_window"],
|
|
566
|
+
dimensions=("time",),
|
|
409
567
|
),
|
|
410
568
|
"input_voltage_range": MetaData(
|
|
411
|
-
long_name="ADC input voltage range (+/-)",
|
|
412
|
-
units="mV",
|
|
569
|
+
long_name="ADC input voltage range (+/-)", units="mV", dimensions=("time",)
|
|
413
570
|
),
|
|
414
571
|
"noise_threshold": MetaData(
|
|
415
572
|
long_name="Noise filter threshold factor",
|
|
416
573
|
units="1",
|
|
417
574
|
comment="Multiple of the standard deviation of Doppler spectra.",
|
|
575
|
+
dimensions=("time",),
|
|
418
576
|
),
|
|
419
|
-
"time_ms": MetaData(
|
|
420
|
-
long_name="Time ms",
|
|
421
|
-
units="ms",
|
|
422
|
-
),
|
|
577
|
+
"time_ms": MetaData(long_name="Time ms", units="ms", dimensions=("time",)),
|
|
423
578
|
"quality_flag": MetaData(
|
|
424
579
|
long_name="Quality flag",
|
|
425
580
|
definition=DEFINITIONS["quality_flag"],
|
|
426
581
|
units="1",
|
|
582
|
+
dimensions=("time",),
|
|
427
583
|
),
|
|
428
|
-
"voltage": MetaData(
|
|
429
|
-
long_name="Voltage",
|
|
430
|
-
units="V",
|
|
431
|
-
),
|
|
584
|
+
"voltage": MetaData(long_name="Voltage", units="V", dimensions=("time",)),
|
|
432
585
|
"brightness_temperature": MetaData(
|
|
433
|
-
long_name="Brightness temperature",
|
|
434
|
-
units="K",
|
|
435
|
-
),
|
|
436
|
-
"if_power": MetaData(
|
|
437
|
-
long_name="IF power at ACD",
|
|
438
|
-
units="uW",
|
|
586
|
+
long_name="Brightness temperature", units="K", dimensions=("time",)
|
|
439
587
|
),
|
|
588
|
+
"if_power": MetaData(long_name="IF power at ACD", units="uW", dimensions=("time",)),
|
|
440
589
|
"status_flag": MetaData(
|
|
441
|
-
long_name="Status flag for heater and blower",
|
|
442
|
-
units="1",
|
|
590
|
+
long_name="Status flag for heater and blower", units="1", dimensions=("time",)
|
|
443
591
|
),
|
|
444
592
|
"transmitted_power": MetaData(
|
|
445
|
-
long_name="Transmitted power",
|
|
446
|
-
units="W",
|
|
593
|
+
long_name="Transmitted power", units="W", dimensions=("time",)
|
|
447
594
|
),
|
|
448
595
|
"transmitter_temperature": MetaData(
|
|
449
|
-
long_name="Transmitter temperature",
|
|
450
|
-
units="K",
|
|
596
|
+
long_name="Transmitter temperature", units="K", dimensions=("time",)
|
|
451
597
|
),
|
|
452
598
|
"receiver_temperature": MetaData(
|
|
453
|
-
long_name="Receiver temperature",
|
|
454
|
-
units="K",
|
|
599
|
+
long_name="Receiver temperature", units="K", dimensions=("time",)
|
|
455
600
|
),
|
|
456
601
|
"pc_temperature": MetaData(
|
|
457
|
-
long_name="PC temperature",
|
|
458
|
-
units="K",
|
|
459
|
-
),
|
|
460
|
-
"skewness": MetaData(
|
|
461
|
-
long_name="Skewness of spectra",
|
|
462
|
-
units="1",
|
|
463
|
-
),
|
|
464
|
-
"kurtosis": MetaData(
|
|
465
|
-
long_name="Kurtosis of spectra",
|
|
466
|
-
units="1",
|
|
467
|
-
),
|
|
468
|
-
"correlation_coefficient": MetaData(
|
|
469
|
-
long_name="Correlation coefficient",
|
|
470
|
-
units="1",
|
|
602
|
+
long_name="PC temperature", units="K", dimensions=("time",)
|
|
471
603
|
),
|
|
472
604
|
}
|