cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -2
- cloudnetpy/categorize/atmos_utils.py +297 -67
- cloudnetpy/categorize/attenuation.py +31 -0
- cloudnetpy/categorize/attenuations/__init__.py +37 -0
- cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
- cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
- cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
- cloudnetpy/categorize/categorize.py +332 -156
- cloudnetpy/categorize/classify.py +127 -125
- cloudnetpy/categorize/containers.py +107 -76
- cloudnetpy/categorize/disdrometer.py +40 -0
- cloudnetpy/categorize/droplet.py +23 -21
- cloudnetpy/categorize/falling.py +53 -24
- cloudnetpy/categorize/freezing.py +25 -12
- cloudnetpy/categorize/insects.py +35 -23
- cloudnetpy/categorize/itu.py +243 -0
- cloudnetpy/categorize/lidar.py +36 -41
- cloudnetpy/categorize/melting.py +34 -26
- cloudnetpy/categorize/model.py +84 -37
- cloudnetpy/categorize/mwr.py +18 -14
- cloudnetpy/categorize/radar.py +215 -102
- cloudnetpy/cli.py +578 -0
- cloudnetpy/cloudnetarray.py +43 -89
- cloudnetpy/concat_lib.py +218 -78
- cloudnetpy/constants.py +28 -10
- cloudnetpy/datasource.py +61 -86
- cloudnetpy/exceptions.py +49 -20
- cloudnetpy/instruments/__init__.py +5 -0
- cloudnetpy/instruments/basta.py +29 -12
- cloudnetpy/instruments/bowtie.py +135 -0
- cloudnetpy/instruments/ceilo.py +138 -115
- cloudnetpy/instruments/ceilometer.py +164 -80
- cloudnetpy/instruments/cl61d.py +21 -5
- cloudnetpy/instruments/cloudnet_instrument.py +74 -36
- cloudnetpy/instruments/copernicus.py +108 -30
- cloudnetpy/instruments/da10.py +54 -0
- cloudnetpy/instruments/disdrometer/common.py +126 -223
- cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
- cloudnetpy/instruments/disdrometer/thies.py +254 -87
- cloudnetpy/instruments/fd12p.py +201 -0
- cloudnetpy/instruments/galileo.py +65 -23
- cloudnetpy/instruments/hatpro.py +123 -49
- cloudnetpy/instruments/instruments.py +113 -1
- cloudnetpy/instruments/lufft.py +39 -17
- cloudnetpy/instruments/mira.py +268 -61
- cloudnetpy/instruments/mrr.py +187 -0
- cloudnetpy/instruments/nc_lidar.py +19 -8
- cloudnetpy/instruments/nc_radar.py +109 -55
- cloudnetpy/instruments/pollyxt.py +135 -51
- cloudnetpy/instruments/radiometrics.py +313 -59
- cloudnetpy/instruments/rain_e_h3.py +171 -0
- cloudnetpy/instruments/rpg.py +321 -189
- cloudnetpy/instruments/rpg_reader.py +74 -40
- cloudnetpy/instruments/toa5.py +49 -0
- cloudnetpy/instruments/vaisala.py +95 -343
- cloudnetpy/instruments/weather_station.py +774 -105
- cloudnetpy/metadata.py +90 -19
- cloudnetpy/model_evaluation/file_handler.py +55 -52
- cloudnetpy/model_evaluation/metadata.py +46 -20
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
- cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
- cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
- cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
- cloudnetpy/model_evaluation/products/model_products.py +43 -35
- cloudnetpy/model_evaluation/products/observation_products.py +41 -35
- cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
- cloudnetpy/model_evaluation/products/tools.py +29 -20
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
- cloudnetpy/model_evaluation/utils.py +2 -1
- cloudnetpy/output.py +170 -111
- cloudnetpy/plotting/__init__.py +2 -1
- cloudnetpy/plotting/plot_meta.py +562 -822
- cloudnetpy/plotting/plotting.py +1142 -704
- cloudnetpy/products/__init__.py +1 -0
- cloudnetpy/products/classification.py +370 -88
- cloudnetpy/products/der.py +85 -55
- cloudnetpy/products/drizzle.py +77 -34
- cloudnetpy/products/drizzle_error.py +15 -11
- cloudnetpy/products/drizzle_tools.py +79 -59
- cloudnetpy/products/epsilon.py +211 -0
- cloudnetpy/products/ier.py +27 -50
- cloudnetpy/products/iwc.py +55 -48
- cloudnetpy/products/lwc.py +96 -70
- cloudnetpy/products/mwr_tools.py +186 -0
- cloudnetpy/products/product_tools.py +170 -128
- cloudnetpy/utils.py +455 -240
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
- cloudnetpy-1.87.3.dist-info/RECORD +127 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
- cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
- docs/source/conf.py +2 -2
- cloudnetpy/categorize/atmos.py +0 -361
- cloudnetpy/products/mwr_multi.py +0 -68
- cloudnetpy/products/mwr_single.py +0 -75
- cloudnetpy-1.49.9.dist-info/RECORD +0 -112
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
|
@@ -1,75 +1,115 @@
|
|
|
1
1
|
"""Module that generates Cloudnet categorize file."""
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import fields
|
|
7
|
+
from os import PathLike
|
|
8
|
+
from typing import TypedDict
|
|
9
|
+
from uuid import UUID
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from numpy.typing import NDArray
|
|
13
|
+
from typing_extensions import NotRequired
|
|
14
|
+
|
|
2
15
|
from cloudnetpy import output, utils
|
|
3
|
-
from cloudnetpy.categorize import
|
|
16
|
+
from cloudnetpy.categorize import attenuation, classify
|
|
17
|
+
from cloudnetpy.categorize.containers import Observations
|
|
18
|
+
from cloudnetpy.categorize.disdrometer import Disdrometer
|
|
4
19
|
from cloudnetpy.categorize.lidar import Lidar
|
|
5
20
|
from cloudnetpy.categorize.model import Model
|
|
6
21
|
from cloudnetpy.categorize.mwr import Mwr
|
|
7
22
|
from cloudnetpy.categorize.radar import Radar
|
|
8
|
-
from cloudnetpy.
|
|
9
|
-
from cloudnetpy.
|
|
23
|
+
from cloudnetpy.datasource import DataSource
|
|
24
|
+
from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
|
|
25
|
+
from cloudnetpy.instruments.rpg import RPG_ATTRIBUTES
|
|
26
|
+
from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
|
|
27
|
+
from cloudnetpy.products.product_tools import CategoryBits, QualityBits
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CategorizeInput(TypedDict):
|
|
31
|
+
radar: str | PathLike
|
|
32
|
+
lidar: str | PathLike
|
|
33
|
+
model: str | PathLike
|
|
34
|
+
disdrometer: NotRequired[str | PathLike]
|
|
35
|
+
mwr: NotRequired[str | PathLike]
|
|
36
|
+
lv0_files: NotRequired[Sequence[str | PathLike]]
|
|
10
37
|
|
|
11
38
|
|
|
12
39
|
def generate_categorize(
|
|
13
|
-
input_files:
|
|
14
|
-
|
|
15
|
-
|
|
40
|
+
input_files: CategorizeInput,
|
|
41
|
+
output_file: str | PathLike,
|
|
42
|
+
uuid: str | UUID | None = None,
|
|
43
|
+
options: dict | None = None,
|
|
44
|
+
) -> UUID:
|
|
45
|
+
"""Generates a Cloudnet Level 1c categorize file.
|
|
16
46
|
|
|
17
|
-
|
|
18
|
-
and
|
|
19
|
-
insects, etc.
|
|
20
|
-
attenuation, and error estimates are computed.
|
|
21
|
-
|
|
47
|
+
This function rebins measurements into a common height/time grid
|
|
48
|
+
and classifies them into different scatterer types, such as ice,
|
|
49
|
+
liquid, insects, etc. The radar signal is corrected for atmospheric
|
|
50
|
+
attenuation, and error estimates are computed. The results are saved in
|
|
51
|
+
*output_file*, a compressed netCDF4 file.
|
|
22
52
|
|
|
23
53
|
Args:
|
|
24
|
-
input_files
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
54
|
+
input_files (dict): Contains filenames for calibrated `radar`, `lidar`,
|
|
55
|
+
and `model` files. Optionally, it can also include `disdrometer`,
|
|
56
|
+
`mwr` (containing the LWP variable), and `lv0_files` (a list of RPG
|
|
57
|
+
Level 0 files).
|
|
58
|
+
output_file (str): The full path of the output file.
|
|
59
|
+
uuid (str): Specific UUID to assign to the generated file.
|
|
60
|
+
options (dict): Dictionary containing optional parameters.
|
|
29
61
|
|
|
30
62
|
Returns:
|
|
31
|
-
UUID of the generated file.
|
|
63
|
+
str: UUID of the generated file.
|
|
32
64
|
|
|
33
65
|
Raises:
|
|
34
|
-
RuntimeError:
|
|
66
|
+
RuntimeError: Raised if the categorize file creation fails.
|
|
35
67
|
|
|
36
68
|
Notes:
|
|
37
|
-
|
|
38
|
-
measures liquid water path.
|
|
39
|
-
|
|
69
|
+
A separate MWR file is not required when using an RPG cloud radar that
|
|
70
|
+
measures liquid water path (LWP). In this case, the radar file can also
|
|
71
|
+
serve as the MWR file (e.g., {'mwr': 'radar.nc'}). If no MWR file
|
|
72
|
+
is provided, liquid attenuation correction cannot be performed.
|
|
40
73
|
|
|
41
|
-
If RPG L0 files are
|
|
42
|
-
to detect liquid droplets.
|
|
74
|
+
If RPG L0 files are included as additional input, the Voodoo method
|
|
75
|
+
is used to detect liquid droplets.
|
|
43
76
|
|
|
44
77
|
Examples:
|
|
45
78
|
>>> from cloudnetpy.categorize import generate_categorize
|
|
46
|
-
>>> input_files = {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
79
|
+
>>> input_files = {
|
|
80
|
+
... 'radar': 'radar.nc',
|
|
81
|
+
... 'lidar': 'lidar.nc',
|
|
82
|
+
... 'model': 'model.nc',
|
|
83
|
+
... 'mwr': 'mwr.nc'
|
|
84
|
+
... }
|
|
50
85
|
>>> generate_categorize(input_files, 'output.nc')
|
|
51
86
|
|
|
52
|
-
>>> input_files[
|
|
53
|
-
>>> generate_categorize(input_files, 'output.nc') # Use Voodoo method
|
|
54
|
-
|
|
87
|
+
>>> input_files['lv0_files'] = ['file1.LV0', 'file2.LV0'] # Add RPG LV0 files
|
|
88
|
+
>>> generate_categorize(input_files, 'output.nc') # Use the Voodoo method
|
|
55
89
|
"""
|
|
90
|
+
uuid = utils.get_uuid(uuid)
|
|
56
91
|
|
|
57
|
-
def _interpolate_to_cloudnet_grid() -> list:
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
data
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
92
|
+
def _interpolate_to_cloudnet_grid() -> list[int]:
|
|
93
|
+
if data.disdrometer is not None:
|
|
94
|
+
data.disdrometer.interpolate_to_grid(time)
|
|
95
|
+
if data.mwr is not None:
|
|
96
|
+
data.mwr.interpolate_to_grid(time)
|
|
97
|
+
data.model.calc_attenuations(data.radar.radar_frequency)
|
|
98
|
+
data.model.interpolate_to_common_height()
|
|
99
|
+
model_gap_ind = data.model.interpolate_to_grid(time, height)
|
|
100
|
+
radar_gap_ind = data.radar.rebin_to_grid(time)
|
|
101
|
+
lidar_gap_ind = data.lidar.interpolate_to_grid(time, height)
|
|
102
|
+
gap_indices = set(radar_gap_ind + lidar_gap_ind + model_gap_ind)
|
|
103
|
+
return [ind for ind in range(len(time)) if ind not in gap_indices]
|
|
67
104
|
|
|
68
105
|
def _screen_bad_time_indices(valid_indices: list) -> None:
|
|
69
106
|
n_time_full = len(time)
|
|
70
|
-
data
|
|
71
|
-
for
|
|
72
|
-
|
|
107
|
+
data.radar.time = time[valid_indices]
|
|
108
|
+
for field in fields(data):
|
|
109
|
+
obj = getattr(data, field.name)
|
|
110
|
+
if not hasattr(obj, "data"):
|
|
111
|
+
continue
|
|
112
|
+
for key, item in obj.data.items():
|
|
73
113
|
if utils.isscalar(item.data):
|
|
74
114
|
continue
|
|
75
115
|
array = item[:]
|
|
@@ -80,106 +120,155 @@ def generate_categorize(
|
|
|
80
120
|
array = array[valid_indices, :]
|
|
81
121
|
else:
|
|
82
122
|
continue
|
|
83
|
-
|
|
84
|
-
for key, item in data
|
|
85
|
-
data
|
|
123
|
+
obj.data[key].data = array
|
|
124
|
+
for key, item in data.model.data_dense.items():
|
|
125
|
+
data.model.data_dense[key] = item[valid_indices, :]
|
|
86
126
|
|
|
87
127
|
def _prepare_output() -> dict:
|
|
88
|
-
data
|
|
89
|
-
data
|
|
90
|
-
|
|
91
|
-
|
|
128
|
+
data.radar.add_meta()
|
|
129
|
+
data.model.screen_sparse_fields()
|
|
130
|
+
|
|
131
|
+
if data.disdrometer is not None:
|
|
132
|
+
data.radar.data.pop("rainfall_rate", None)
|
|
133
|
+
data.disdrometer.data.pop("n_particles", None)
|
|
134
|
+
|
|
135
|
+
data.radar.append_data(attenuations.gas.amount, "radar_gas_atten")
|
|
136
|
+
data.radar.append_data(attenuations.liquid.amount, "radar_liquid_atten")
|
|
137
|
+
data.radar.append_data(attenuations.rain.amount, "radar_rain_atten")
|
|
138
|
+
data.radar.append_data(attenuations.melting.amount, "radar_melting_atten")
|
|
139
|
+
|
|
140
|
+
data.radar.append_data(_classes_to_bits(quality), "quality_bits")
|
|
141
|
+
|
|
142
|
+
data.radar.append_data(classification.insect_prob, "insect_prob")
|
|
143
|
+
data.radar.append_data(classification.is_rain, "rain_detected")
|
|
144
|
+
data.radar.append_data(
|
|
145
|
+
_classes_to_bits(classification.category_bits), "category_bits"
|
|
146
|
+
)
|
|
147
|
+
|
|
92
148
|
if classification.liquid_prob is not None:
|
|
93
|
-
data
|
|
94
|
-
|
|
95
|
-
data["radar"].append_data(attenuations[key], key)
|
|
96
|
-
data["radar"].append_data(quality["quality_bits"], "quality_bits")
|
|
149
|
+
data.radar.append_data(classification.liquid_prob, "liquid_prob")
|
|
150
|
+
|
|
97
151
|
return {
|
|
98
|
-
**data
|
|
99
|
-
**data
|
|
100
|
-
**data
|
|
101
|
-
**data
|
|
102
|
-
**data
|
|
152
|
+
**data.radar.data,
|
|
153
|
+
**data.lidar.data,
|
|
154
|
+
**data.model.data,
|
|
155
|
+
**data.model.data_sparse,
|
|
156
|
+
**(data.mwr.data if data.mwr is not None else {}),
|
|
157
|
+
**(data.disdrometer.data if data.disdrometer is not None else {}),
|
|
103
158
|
}
|
|
104
159
|
|
|
105
|
-
def _define_dense_grid():
|
|
106
|
-
return utils.time_grid(), data
|
|
160
|
+
def _define_dense_grid() -> tuple:
|
|
161
|
+
return utils.time_grid(), data.radar.height
|
|
107
162
|
|
|
108
|
-
def _close_all():
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
obj.
|
|
163
|
+
def _close_all() -> None:
|
|
164
|
+
if "data" in locals():
|
|
165
|
+
for field in fields(data):
|
|
166
|
+
obj = getattr(data, field.name)
|
|
167
|
+
if isinstance(obj, DataSource):
|
|
168
|
+
obj.close()
|
|
112
169
|
|
|
113
170
|
try:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
"
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
171
|
+
radar = Radar(input_files["radar"])
|
|
172
|
+
data = Observations(
|
|
173
|
+
radar=radar,
|
|
174
|
+
lidar=Lidar(input_files["lidar"]),
|
|
175
|
+
model=Model(input_files["model"], radar.altitude, options),
|
|
176
|
+
lv0_files=input_files.get("lv0_files"),
|
|
177
|
+
)
|
|
178
|
+
if "mwr" in input_files:
|
|
179
|
+
data.mwr = Mwr(input_files["mwr"])
|
|
180
|
+
if "disdrometer" in input_files:
|
|
181
|
+
try:
|
|
182
|
+
data.disdrometer = Disdrometer(input_files["disdrometer"])
|
|
183
|
+
except DisdrometerDataError as err:
|
|
184
|
+
logging.warning("Unable to use disdrometer: %s", err)
|
|
122
185
|
time, height = _define_dense_grid()
|
|
186
|
+
data.radar.add_location(time)
|
|
123
187
|
valid_ind = _interpolate_to_cloudnet_grid()
|
|
124
|
-
if
|
|
125
|
-
|
|
188
|
+
if len(valid_ind) < 2:
|
|
189
|
+
msg = "Less than 2 overlapping radar, lidar and model timestamps found"
|
|
190
|
+
raise ValidTimeStampError(msg)
|
|
126
191
|
_screen_bad_time_indices(valid_ind)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
data
|
|
192
|
+
|
|
193
|
+
if any(source in data.radar.source_type.lower() for source in ("rpg", "basta")):
|
|
194
|
+
data.radar.filter_speckle_noise()
|
|
195
|
+
data.radar.filter_1st_gate_artifact()
|
|
130
196
|
for variable in ("v", "v_sigma", "ldr"):
|
|
131
|
-
data
|
|
132
|
-
data
|
|
133
|
-
data
|
|
197
|
+
data.radar.filter_stripes(variable)
|
|
198
|
+
data.radar.remove_incomplete_pixels()
|
|
199
|
+
data.model.calc_wet_bulb()
|
|
200
|
+
|
|
134
201
|
classification = classify.classify_measurements(data)
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
202
|
+
|
|
203
|
+
attenuations = attenuation.get_attenuations(data, classification)
|
|
204
|
+
|
|
205
|
+
data.radar.correct_atten(attenuations)
|
|
206
|
+
data.radar.calc_errors(attenuations, classification.is_clutter)
|
|
207
|
+
|
|
138
208
|
quality = classify.fetch_quality(data, classification, attenuations)
|
|
209
|
+
|
|
139
210
|
cloudnet_arrays = _prepare_output()
|
|
140
|
-
|
|
211
|
+
|
|
212
|
+
date = data.radar.get_date()
|
|
141
213
|
attributes = output.add_time_attribute(CATEGORIZE_ATTRIBUTES, date)
|
|
142
214
|
attributes = output.add_time_attribute(attributes, date, "model_time")
|
|
143
215
|
attributes = output.add_source_attribute(attributes, data)
|
|
144
216
|
output.update_attributes(cloudnet_arrays, attributes)
|
|
145
|
-
|
|
217
|
+
_save_cat(output_file, data, cloudnet_arrays, uuid)
|
|
146
218
|
return uuid
|
|
147
219
|
finally:
|
|
148
220
|
_close_all()
|
|
149
221
|
|
|
150
222
|
|
|
151
223
|
def _save_cat(
|
|
152
|
-
full_path: str
|
|
153
|
-
|
|
224
|
+
full_path: str | PathLike,
|
|
225
|
+
data_obs: Observations,
|
|
226
|
+
cloudnet_arrays: dict,
|
|
227
|
+
uuid: UUID,
|
|
228
|
+
) -> None:
|
|
154
229
|
"""Creates a categorize netCDF4 file and saves all data into it."""
|
|
155
|
-
|
|
156
230
|
dims = {
|
|
157
|
-
"time": len(data_obs
|
|
158
|
-
"height": len(data_obs
|
|
159
|
-
"model_time": len(data_obs
|
|
160
|
-
"model_height": len(data_obs
|
|
231
|
+
"time": len(data_obs.radar.time),
|
|
232
|
+
"height": len(data_obs.radar.height),
|
|
233
|
+
"model_time": len(data_obs.model.time),
|
|
234
|
+
"model_height": len(data_obs.model.mean_height),
|
|
161
235
|
}
|
|
162
236
|
|
|
163
237
|
file_type = "categorize"
|
|
238
|
+
if "liquid_prob" in cloudnet_arrays:
|
|
239
|
+
file_type += "-voodoo"
|
|
240
|
+
|
|
164
241
|
with output.init_file(full_path, dims, cloudnet_arrays, uuid) as nc:
|
|
165
|
-
uuid_out = nc.file_uuid
|
|
166
242
|
nc.cloudnet_file_type = file_type
|
|
167
243
|
output.copy_global(
|
|
168
|
-
data_obs
|
|
244
|
+
data_obs.radar.dataset,
|
|
245
|
+
nc,
|
|
246
|
+
("year", "month", "day", "location"),
|
|
169
247
|
)
|
|
170
|
-
nc.title = f"Cloud categorization products from {data_obs
|
|
171
|
-
nc.source_file_uuids = output.get_source_uuids(
|
|
248
|
+
nc.title = f"Cloud categorization products from {data_obs.radar.location}"
|
|
249
|
+
nc.source_file_uuids = output.get_source_uuids(data_obs)
|
|
250
|
+
is_voodoo = "liquid_prob" in cloudnet_arrays
|
|
172
251
|
extra_references = (
|
|
173
|
-
["https://doi.org/10.5194/amt-15-5343-2022"]
|
|
174
|
-
if "liquid_prob" in cloudnet_arrays
|
|
175
|
-
else None
|
|
252
|
+
["https://doi.org/10.5194/amt-15-5343-2022"] if is_voodoo else None
|
|
176
253
|
)
|
|
177
254
|
nc.references = output.get_references(
|
|
178
|
-
identifier=file_type,
|
|
255
|
+
identifier=file_type,
|
|
256
|
+
extra=extra_references,
|
|
179
257
|
)
|
|
258
|
+
if is_voodoo:
|
|
259
|
+
import voodoonet.version # noqa: PLC0415
|
|
260
|
+
|
|
261
|
+
nc.voodoonet_version = voodoonet.version.__version__
|
|
180
262
|
output.add_source_instruments(nc, data_obs)
|
|
181
263
|
output.merge_history(nc, file_type, data_obs)
|
|
182
|
-
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _classes_to_bits(data: QualityBits | CategoryBits) -> NDArray[np.int_]:
|
|
267
|
+
shape = data.radar.shape if hasattr(data, "radar") else data.droplet.shape
|
|
268
|
+
quality = np.zeros(shape, dtype=np.int64)
|
|
269
|
+
for i, field in enumerate(dataclasses.fields(data)):
|
|
270
|
+
quality |= (1 << i) * getattr(data, field.name)
|
|
271
|
+
return quality
|
|
183
272
|
|
|
184
273
|
|
|
185
274
|
COMMENTS = {
|
|
@@ -216,14 +305,17 @@ COMMENTS = {
|
|
|
216
305
|
"humidity, but forcing pixels containing liquid cloud to saturation with\n"
|
|
217
306
|
"respect to liquid water. It has been used to correct Z."
|
|
218
307
|
),
|
|
219
|
-
"
|
|
220
|
-
"This variable was
|
|
221
|
-
"humidity."
|
|
308
|
+
"radar_rain_atten": (
|
|
309
|
+
"This variable was calculated from the disdrometer rainfall rate."
|
|
222
310
|
),
|
|
311
|
+
"radar_melting_atten": (
|
|
312
|
+
"This variable was calculated from the disdrometer rainfall rate."
|
|
313
|
+
),
|
|
314
|
+
"Tw": "This variable was derived from model temperature, pressure and humidity.",
|
|
223
315
|
"Z_sensitivity": (
|
|
224
316
|
"This variable is an estimate of the radar sensitivity, i.e. the minimum\n"
|
|
225
317
|
"detectable radar reflectivity, as a function of height. It includes the\n"
|
|
226
|
-
"effect of ground clutter and gas attenuation but not
|
|
318
|
+
"effect of ground clutter and gas attenuation but not other attenuations."
|
|
227
319
|
),
|
|
228
320
|
"Z_error": (
|
|
229
321
|
"This variable is an estimate of the one-standard-deviation random error\n"
|
|
@@ -233,22 +325,22 @@ COMMENTS = {
|
|
|
233
325
|
" finite number of pulses\n"
|
|
234
326
|
"2) 10% uncertainty in gaseous attenuation correction (mainly due to error\n"
|
|
235
327
|
" in model humidity field)\n"
|
|
236
|
-
"3)
|
|
328
|
+
"3) 20% uncertainty in rain attenuation correction (mainly due to error\n"
|
|
329
|
+
" in disdrometer rainfall rate)\n"
|
|
330
|
+
"4) 10%-20% uncertainty in melting layer attenuation correction (mainly due\n"
|
|
331
|
+
" to error in disdrometer rainfall rate)\n"
|
|
332
|
+
"5) Error in liquid water path (given by the variable lwp_error) and its\n"
|
|
237
333
|
" partitioning with height)."
|
|
238
334
|
),
|
|
239
335
|
"Z": (
|
|
240
|
-
"This variable has been corrected for attenuation by gaseous attenuation
|
|
241
|
-
"
|
|
242
|
-
"
|
|
243
|
-
"
|
|
244
|
-
"
|
|
245
|
-
"Calibration convention: in the absence of attenuation, a cloud at 273 K\n"
|
|
246
|
-
"containing one million 100-micron droplets per cubic metre will have\n"
|
|
247
|
-
"a reflectivity of 0 dBZ at all frequencies."
|
|
336
|
+
"This variable has been corrected for attenuation by gaseous attenuation,\n"
|
|
337
|
+
"and possibly liquid water, rain and melting layer (see quality_bits\n"
|
|
338
|
+
"variable). Calibration convention: in the absence of attenuation, a cloud\n"
|
|
339
|
+
"at 273 K containing one million 100-micron droplets per cubic metre will\n"
|
|
340
|
+
"have\n a reflectivity of 0 dBZ at all frequencies."
|
|
248
341
|
),
|
|
249
342
|
"bias": (
|
|
250
|
-
"This variable is an estimate of the one-standard-deviation\
|
|
251
|
-
"calibration error."
|
|
343
|
+
"This variable is an estimate of the one-standard-deviation\ncalibration error."
|
|
252
344
|
),
|
|
253
345
|
"insect_prob": (
|
|
254
346
|
"Ad-hoc estimation of the probability that the pixel contains insects."
|
|
@@ -259,61 +351,94 @@ COMMENTS = {
|
|
|
259
351
|
}
|
|
260
352
|
|
|
261
353
|
DEFINITIONS = {
|
|
262
|
-
"category_bits": (
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
354
|
+
"category_bits": utils.bit_field_definition(
|
|
355
|
+
{
|
|
356
|
+
0: """Small liquid droplets are present.""",
|
|
357
|
+
1: """Falling hydrometeors are present; if Bit 2 is set then these
|
|
358
|
+
are most likely ice particles, otherwise they are drizzle or
|
|
359
|
+
rain drops.""",
|
|
360
|
+
2: """Wet-bulb temperature is less than 0 degrees C, implying the
|
|
361
|
+
phase of Bit-1 particles.""",
|
|
362
|
+
3: """Melting ice particles are present.""",
|
|
363
|
+
4: """Aerosol particles are present and visible to the lidar.""",
|
|
364
|
+
5: """Insects are present and visible to the radar.""",
|
|
365
|
+
}
|
|
272
366
|
),
|
|
273
|
-
"quality_bits": (
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
367
|
+
"quality_bits": utils.bit_field_definition(
|
|
368
|
+
{
|
|
369
|
+
0: """An echo is detected by the radar.""",
|
|
370
|
+
1: """An echo is detected by the lidar.""",
|
|
371
|
+
2: """The apparent echo detected by the radar is ground clutter or
|
|
372
|
+
some other non-atmospheric artifact.""",
|
|
373
|
+
3: """The lidar echo is due to clear-air molecular scattering.""",
|
|
374
|
+
4: """Liquid water cloud, rainfall or melting ice below this pixel
|
|
375
|
+
will have caused radar and lidar attenuation; if bit 5 is set
|
|
376
|
+
then a correction for the radar attenuation has been
|
|
377
|
+
performed; otherwise do not trust the absolute values of
|
|
378
|
+
reflectivity factor. No correction is performed for lidar
|
|
379
|
+
attenuation.""",
|
|
380
|
+
5: """Radar reflectivity has been corrected for liquid-water
|
|
381
|
+
attenuation using the microwave radiometer measurements of
|
|
382
|
+
liquid water path and the lidar estimation of the location of
|
|
383
|
+
liquid water cloud; be aware that errors in reflectivity may
|
|
384
|
+
result.""",
|
|
385
|
+
6: """Rain has caused radar attenuation; if bit 7 is set then a
|
|
386
|
+
correction for the radar attenuation has been performed;
|
|
387
|
+
otherwise do not trust the absolute values of reflectivity
|
|
388
|
+
factor. No correction is performed for lidar attenuation.""",
|
|
389
|
+
7: """Radar reflectivity has been corrected for rain attenuation
|
|
390
|
+
using rainfall rate from a disdrometer; be aware that errors
|
|
391
|
+
in reflectivity may result.""",
|
|
392
|
+
8: """Melting layer has caused radar attenuation; if bit 9 is set then a
|
|
393
|
+
correction for the radar attenuation has been performed;
|
|
394
|
+
otherwise do not trust the absolute values of reflectivity
|
|
395
|
+
factor. No correction is performed for lidar attenuation.""",
|
|
396
|
+
9: """Radar reflectivity has been corrected for melting layer
|
|
397
|
+
attenuation; be aware that errors in reflectivity may result.""",
|
|
398
|
+
}
|
|
289
399
|
),
|
|
290
400
|
}
|
|
291
401
|
|
|
292
402
|
CATEGORIZE_ATTRIBUTES = {
|
|
403
|
+
"height": COMMON_ATTRIBUTES["height"]._replace(dimensions=("height",)),
|
|
293
404
|
# Radar variables
|
|
294
405
|
"Z": MetaData(
|
|
295
406
|
long_name="Radar reflectivity factor",
|
|
296
407
|
units="dBZ",
|
|
297
408
|
comment=COMMENTS["Z"],
|
|
298
409
|
ancillary_variables="Z_error Z_bias Z_sensitivity",
|
|
410
|
+
dimensions=("time", "height"),
|
|
299
411
|
),
|
|
300
412
|
"Z_error": MetaData(
|
|
301
413
|
long_name="Error in radar reflectivity factor",
|
|
302
414
|
units="dB",
|
|
303
415
|
comment=COMMENTS["Z_error"],
|
|
416
|
+
dimensions=("time", "height"),
|
|
304
417
|
),
|
|
305
418
|
"Z_bias": MetaData(
|
|
306
419
|
long_name="Bias in radar reflectivity factor",
|
|
307
420
|
units="dB",
|
|
308
421
|
comment=COMMENTS["bias"],
|
|
422
|
+
dimensions=None,
|
|
309
423
|
),
|
|
310
424
|
"Z_sensitivity": MetaData(
|
|
311
425
|
long_name="Minimum detectable radar reflectivity",
|
|
312
426
|
units="dBZ",
|
|
313
427
|
comment=COMMENTS["Z_sensitivity"],
|
|
428
|
+
dimensions=("height",),
|
|
314
429
|
),
|
|
430
|
+
"v": COMMON_ATTRIBUTES["v"]._replace(dimensions=("time", "height")),
|
|
431
|
+
"width": COMMON_ATTRIBUTES["width"]._replace(dimensions=("time", "height")),
|
|
432
|
+
"ldr": COMMON_ATTRIBUTES["ldr"]._replace(dimensions=("time", "height")),
|
|
433
|
+
"sldr": COMMON_ATTRIBUTES["sldr"]._replace(dimensions=("time", "height")),
|
|
315
434
|
"v_sigma": MetaData(
|
|
316
|
-
long_name="Standard deviation of mean Doppler velocity",
|
|
435
|
+
long_name="Standard deviation of mean Doppler velocity",
|
|
436
|
+
units="m s-1",
|
|
437
|
+
dimensions=("time", "height"),
|
|
438
|
+
),
|
|
439
|
+
"zdr": RPG_ATTRIBUTES["zdr"]._replace(dimensions=("time", "height")),
|
|
440
|
+
"nyquist_velocity": COMMON_ATTRIBUTES["nyquist_velocity"]._replace(
|
|
441
|
+
dimensions=("time", "height")
|
|
317
442
|
),
|
|
318
443
|
# Lidar variables
|
|
319
444
|
"beta": MetaData(
|
|
@@ -321,77 +446,128 @@ CATEGORIZE_ATTRIBUTES = {
|
|
|
321
446
|
units="sr-1 m-1",
|
|
322
447
|
comment="SNR-screened attenuated backscatter coefficient.",
|
|
323
448
|
ancillary_variables="beta_error beta_bias",
|
|
449
|
+
dimensions=("time", "height"),
|
|
324
450
|
),
|
|
325
451
|
"beta_error": MetaData(
|
|
326
452
|
long_name="Error in attenuated backscatter coefficient",
|
|
327
453
|
units="dB",
|
|
454
|
+
dimensions=None,
|
|
328
455
|
),
|
|
329
456
|
"beta_bias": MetaData(
|
|
330
457
|
long_name="Bias in attenuated backscatter coefficient",
|
|
331
458
|
units="dB",
|
|
459
|
+
dimensions=None,
|
|
460
|
+
),
|
|
461
|
+
"lidar_wavelength": MetaData(
|
|
462
|
+
long_name="Laser wavelength", units="nm", dimensions=None
|
|
332
463
|
),
|
|
333
|
-
"lidar_wavelength": MetaData(long_name="Laser wavelength", units="nm"),
|
|
334
464
|
# MWR variables
|
|
335
465
|
"lwp_error": MetaData(
|
|
336
466
|
long_name="Error in liquid water path",
|
|
337
467
|
units="kg m-2",
|
|
468
|
+
dimensions=("time",),
|
|
469
|
+
),
|
|
470
|
+
"lwp": COMMON_ATTRIBUTES["lwp"]._replace(
|
|
471
|
+
ancillary_variables="lwp_error", dimensions=("time",)
|
|
338
472
|
),
|
|
339
|
-
"lwp": MetaData(ancillary_variables="lwp_error"),
|
|
340
473
|
# Model variables
|
|
341
|
-
"Tw": MetaData(
|
|
342
|
-
|
|
474
|
+
"Tw": MetaData(
|
|
475
|
+
long_name="Wet-bulb temperature",
|
|
476
|
+
units="K",
|
|
477
|
+
comment=COMMENTS["Tw"],
|
|
478
|
+
dimensions=("time", "height"),
|
|
479
|
+
),
|
|
480
|
+
"model_time": MetaData(
|
|
481
|
+
long_name="Model time UTC",
|
|
482
|
+
calendar="standard",
|
|
483
|
+
dimensions=("model_time",),
|
|
484
|
+
),
|
|
343
485
|
"model_height": MetaData(
|
|
344
486
|
long_name="Height of model variables above mean sea level",
|
|
345
487
|
units="m",
|
|
346
488
|
axis="Z",
|
|
489
|
+
dimensions=("model_height",),
|
|
490
|
+
),
|
|
491
|
+
"temperature": MetaData(
|
|
492
|
+
long_name="Temperature", units="K", dimensions=("model_time", "model_height")
|
|
493
|
+
),
|
|
494
|
+
"pressure": MetaData(
|
|
495
|
+
long_name="Pressure", units="Pa", dimensions=("model_time", "model_height")
|
|
347
496
|
),
|
|
348
497
|
"vwind": MetaData(
|
|
349
498
|
long_name="Meridional wind",
|
|
350
499
|
units="m s-1",
|
|
500
|
+
dimensions=("model_time", "model_height"),
|
|
351
501
|
),
|
|
352
502
|
"uwind": MetaData(
|
|
353
503
|
long_name="Zonal wind",
|
|
354
504
|
units="m s-1",
|
|
505
|
+
dimensions=("model_time", "model_height"),
|
|
506
|
+
),
|
|
507
|
+
"q": MetaData(
|
|
508
|
+
long_name="Specific humidity",
|
|
509
|
+
units="1",
|
|
510
|
+
dimensions=("model_time", "model_height"),
|
|
355
511
|
),
|
|
356
|
-
"q": MetaData(long_name="Specific humidity", units="1"),
|
|
357
512
|
# MISC
|
|
358
513
|
"category_bits": MetaData(
|
|
359
514
|
long_name="Target categorization bits",
|
|
360
515
|
comment=COMMENTS["category_bits"],
|
|
361
516
|
definition=DEFINITIONS["category_bits"],
|
|
362
517
|
units="1",
|
|
518
|
+
dimensions=("time", "height"),
|
|
363
519
|
),
|
|
364
520
|
"quality_bits": MetaData(
|
|
365
521
|
long_name="Data quality bits",
|
|
366
522
|
comment=COMMENTS["quality_bits"],
|
|
367
523
|
definition=DEFINITIONS["quality_bits"],
|
|
368
524
|
units="1",
|
|
369
|
-
|
|
370
|
-
"rainfall_rate": MetaData(
|
|
371
|
-
long_name="Rainfall rate",
|
|
372
|
-
standard_name="rainfall_rate",
|
|
373
|
-
units="m s-1",
|
|
374
|
-
comment="Fill values denote rain with undefined intensity.",
|
|
525
|
+
dimensions=("time", "height"),
|
|
375
526
|
),
|
|
376
527
|
"radar_liquid_atten": MetaData(
|
|
377
528
|
long_name="Two-way radar attenuation due to liquid water",
|
|
378
529
|
units="dB",
|
|
379
530
|
comment=COMMENTS["radar_liquid_atten"],
|
|
531
|
+
references="ITU-R P.840-9",
|
|
532
|
+
dimensions=("time", "height"),
|
|
533
|
+
),
|
|
534
|
+
"radar_rain_atten": MetaData(
|
|
535
|
+
long_name="Two-way radar attenuation due to rain",
|
|
536
|
+
units="dB",
|
|
537
|
+
references="Crane, R. (1980)",
|
|
538
|
+
comment=COMMENTS["radar_rain_atten"],
|
|
539
|
+
dimensions=("time", "height"),
|
|
540
|
+
),
|
|
541
|
+
"radar_melting_atten": MetaData(
|
|
542
|
+
long_name="Two-way radar attenuation due to melting ice",
|
|
543
|
+
units="dB",
|
|
544
|
+
references="Li, H., & Moisseev, D. (2019)",
|
|
545
|
+
comment=COMMENTS["radar_melting_atten"],
|
|
546
|
+
dimensions=("time", "height"),
|
|
380
547
|
),
|
|
381
548
|
"radar_gas_atten": MetaData(
|
|
382
549
|
long_name="Two-way radar attenuation due to atmospheric gases",
|
|
383
550
|
units="dB",
|
|
384
551
|
comment=COMMENTS["radar_gas_atten"],
|
|
385
|
-
references="
|
|
552
|
+
references="ITU-R P.676-13",
|
|
553
|
+
dimensions=("time", "height"),
|
|
386
554
|
),
|
|
387
555
|
"insect_prob": MetaData(
|
|
388
556
|
long_name="Insect probability",
|
|
389
557
|
units="1",
|
|
390
558
|
comment=COMMENTS["insect_prob"],
|
|
559
|
+
dimensions=("time", "height"),
|
|
391
560
|
),
|
|
392
561
|
"liquid_prob": MetaData(
|
|
393
562
|
long_name="Liquid probability",
|
|
394
563
|
units="1",
|
|
395
564
|
comment=COMMENTS["liquid_prob"],
|
|
565
|
+
dimensions=("time", "height"),
|
|
566
|
+
),
|
|
567
|
+
"rain_detected": MetaData(
|
|
568
|
+
long_name="Rain detected",
|
|
569
|
+
units="1",
|
|
570
|
+
comment="1 = rain detected, 0 = no rain detected",
|
|
571
|
+
dimensions=("time",),
|
|
396
572
|
),
|
|
397
573
|
}
|