cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,75 +1,115 @@
1
1
  """Module that generates Cloudnet categorize file."""
2
+
3
+ import dataclasses
4
+ import logging
5
+ from collections.abc import Sequence
6
+ from dataclasses import fields
7
+ from os import PathLike
8
+ from typing import TypedDict
9
+ from uuid import UUID
10
+
11
+ import numpy as np
12
+ from numpy.typing import NDArray
13
+ from typing_extensions import NotRequired
14
+
2
15
  from cloudnetpy import output, utils
3
- from cloudnetpy.categorize import atmos, classify
16
+ from cloudnetpy.categorize import attenuation, classify
17
+ from cloudnetpy.categorize.containers import Observations
18
+ from cloudnetpy.categorize.disdrometer import Disdrometer
4
19
  from cloudnetpy.categorize.lidar import Lidar
5
20
  from cloudnetpy.categorize.model import Model
6
21
  from cloudnetpy.categorize.mwr import Mwr
7
22
  from cloudnetpy.categorize.radar import Radar
8
- from cloudnetpy.exceptions import ValidTimeStampError
9
- from cloudnetpy.metadata import MetaData
23
+ from cloudnetpy.datasource import DataSource
24
+ from cloudnetpy.exceptions import DisdrometerDataError, ValidTimeStampError
25
+ from cloudnetpy.instruments.rpg import RPG_ATTRIBUTES
26
+ from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
27
+ from cloudnetpy.products.product_tools import CategoryBits, QualityBits
28
+
29
+
30
+ class CategorizeInput(TypedDict):
31
+ radar: str | PathLike
32
+ lidar: str | PathLike
33
+ model: str | PathLike
34
+ disdrometer: NotRequired[str | PathLike]
35
+ mwr: NotRequired[str | PathLike]
36
+ lv0_files: NotRequired[Sequence[str | PathLike]]
10
37
 
11
38
 
12
39
  def generate_categorize(
13
- input_files: dict, output_file: str, uuid: str | None = None
14
- ) -> str:
15
- """Generates Cloudnet Level 1c categorize file.
40
+ input_files: CategorizeInput,
41
+ output_file: str | PathLike,
42
+ uuid: str | UUID | None = None,
43
+ options: dict | None = None,
44
+ ) -> UUID:
45
+ """Generates a Cloudnet Level 1c categorize file.
16
46
 
17
- The measurements are rebinned into a common height / time grid,
18
- and classified as different types of scatterers such as ice, liquid,
19
- insects, etc. Next, the radar signal is corrected for atmospheric
20
- attenuation, and error estimates are computed. Results are saved
21
- in *ouput_file* which is a compressed netCDF4 file.
47
+ This function rebins measurements into a common height/time grid
48
+ and classifies them into different scatterer types, such as ice,
49
+ liquid, insects, etc. The radar signal is corrected for atmospheric
50
+ attenuation, and error estimates are computed. The results are saved in
51
+ *output_file*, a compressed netCDF4 file.
22
52
 
23
53
  Args:
24
- input_files: dict containing file names for calibrated `radar`, `lidar`,
25
- `model` and `mwr` files. Optionally also `lv0_files`, a list of
26
- RPG level 0 files.
27
- output_file: Full path of the output file.
28
- uuid: Set specific UUID for the file.
54
+ input_files (dict): Contains filenames for calibrated `radar`, `lidar`,
55
+ and `model` files. Optionally, it can also include `disdrometer`,
56
+ `mwr` (containing the LWP variable), and `lv0_files` (a list of RPG
57
+ Level 0 files).
58
+ output_file (str): The full path of the output file.
59
+ uuid (str): Specific UUID to assign to the generated file.
60
+ options (dict): Dictionary containing optional parameters.
29
61
 
30
62
  Returns:
31
- UUID of the generated file.
63
+ str: UUID of the generated file.
32
64
 
33
65
  Raises:
34
- RuntimeError: Failed to create the categorize file.
66
+ RuntimeError: Raised if the categorize file creation fails.
35
67
 
36
68
  Notes:
37
- Separate mwr-file is not needed when using RPG cloud radar which
38
- measures liquid water path. Then, the radar file can be used as
39
- a mwr-file as well, i.e. {'mwr': 'radar.nc'}.
69
+ A separate MWR file is not required when using an RPG cloud radar that
70
+ measures liquid water path (LWP). In this case, the radar file can also
71
+ serve as the MWR file (e.g., {'mwr': 'radar.nc'}). If no MWR file
72
+ is provided, liquid attenuation correction cannot be performed.
40
73
 
41
- If RPG L0 files are provided as an additional input, Voodoo method is used
42
- to detect liquid droplets.
74
+ If RPG L0 files are included as additional input, the Voodoo method
75
+ is used to detect liquid droplets.
43
76
 
44
77
  Examples:
45
78
  >>> from cloudnetpy.categorize import generate_categorize
46
- >>> input_files = {'radar': 'radar.nc',
47
- 'lidar': 'lidar.nc',
48
- 'model': 'model.nc',
49
- 'mwr': 'mwr.nc'}
79
+ >>> input_files = {
80
+ ... 'radar': 'radar.nc',
81
+ ... 'lidar': 'lidar.nc',
82
+ ... 'model': 'model.nc',
83
+ ... 'mwr': 'mwr.nc'
84
+ ... }
50
85
  >>> generate_categorize(input_files, 'output.nc')
51
86
 
52
- >>> input_files["lv0_files"] = ["file1.LV0", "file2.LV0"] # Add RGP LV0 files
53
- >>> generate_categorize(input_files, 'output.nc') # Use Voodoo method
54
-
87
+ >>> input_files['lv0_files'] = ['file1.LV0', 'file2.LV0'] # Add RPG LV0 files
88
+ >>> generate_categorize(input_files, 'output.nc') # Use the Voodoo method
55
89
  """
90
+ uuid = utils.get_uuid(uuid)
56
91
 
57
- def _interpolate_to_cloudnet_grid() -> list:
58
- wl_band = utils.get_wl_band(data["radar"].radar_frequency)
59
- data["model"].interpolate_to_common_height(wl_band)
60
- data["model"].interpolate_to_grid(time, height)
61
- data["mwr"].rebin_to_grid(time)
62
- radar_data_gap_indices = data["radar"].rebin_to_grid(time)
63
- lidar_data_gap_indices = data["lidar"].interpolate_to_grid(time, height)
64
- bad_time_indices = list(set(radar_data_gap_indices + lidar_data_gap_indices))
65
- valid_ind = [ind for ind in range(len(time)) if ind not in bad_time_indices]
66
- return valid_ind
92
+ def _interpolate_to_cloudnet_grid() -> list[int]:
93
+ if data.disdrometer is not None:
94
+ data.disdrometer.interpolate_to_grid(time)
95
+ if data.mwr is not None:
96
+ data.mwr.interpolate_to_grid(time)
97
+ data.model.calc_attenuations(data.radar.radar_frequency)
98
+ data.model.interpolate_to_common_height()
99
+ model_gap_ind = data.model.interpolate_to_grid(time, height)
100
+ radar_gap_ind = data.radar.rebin_to_grid(time)
101
+ lidar_gap_ind = data.lidar.interpolate_to_grid(time, height)
102
+ gap_indices = set(radar_gap_ind + lidar_gap_ind + model_gap_ind)
103
+ return [ind for ind in range(len(time)) if ind not in gap_indices]
67
104
 
68
105
  def _screen_bad_time_indices(valid_indices: list) -> None:
69
106
  n_time_full = len(time)
70
- data["radar"].time = time[valid_indices]
71
- for var in ("radar", "lidar", "mwr", "model"):
72
- for key, item in data[var].data.items():
107
+ data.radar.time = time[valid_indices]
108
+ for field in fields(data):
109
+ obj = getattr(data, field.name)
110
+ if not hasattr(obj, "data"):
111
+ continue
112
+ for key, item in obj.data.items():
73
113
  if utils.isscalar(item.data):
74
114
  continue
75
115
  array = item[:]
@@ -80,106 +120,155 @@ def generate_categorize(
80
120
  array = array[valid_indices, :]
81
121
  else:
82
122
  continue
83
- data[var].data[key].data = array
84
- for key, item in data["model"].data_dense.items():
85
- data["model"].data_dense[key] = item[valid_indices, :]
123
+ obj.data[key].data = array
124
+ for key, item in data.model.data_dense.items():
125
+ data.model.data_dense[key] = item[valid_indices, :]
86
126
 
87
127
  def _prepare_output() -> dict:
88
- data["radar"].add_meta()
89
- data["model"].screen_sparse_fields()
90
- for key in ("category_bits", "rainfall_rate", "insect_prob"):
91
- data["radar"].append_data(getattr(classification, key), key)
128
+ data.radar.add_meta()
129
+ data.model.screen_sparse_fields()
130
+
131
+ if data.disdrometer is not None:
132
+ data.radar.data.pop("rainfall_rate", None)
133
+ data.disdrometer.data.pop("n_particles", None)
134
+
135
+ data.radar.append_data(attenuations.gas.amount, "radar_gas_atten")
136
+ data.radar.append_data(attenuations.liquid.amount, "radar_liquid_atten")
137
+ data.radar.append_data(attenuations.rain.amount, "radar_rain_atten")
138
+ data.radar.append_data(attenuations.melting.amount, "radar_melting_atten")
139
+
140
+ data.radar.append_data(_classes_to_bits(quality), "quality_bits")
141
+
142
+ data.radar.append_data(classification.insect_prob, "insect_prob")
143
+ data.radar.append_data(classification.is_rain, "rain_detected")
144
+ data.radar.append_data(
145
+ _classes_to_bits(classification.category_bits), "category_bits"
146
+ )
147
+
92
148
  if classification.liquid_prob is not None:
93
- data["radar"].append_data(classification.liquid_prob, "liquid_prob")
94
- for key in ("radar_liquid_atten", "radar_gas_atten"):
95
- data["radar"].append_data(attenuations[key], key)
96
- data["radar"].append_data(quality["quality_bits"], "quality_bits")
149
+ data.radar.append_data(classification.liquid_prob, "liquid_prob")
150
+
97
151
  return {
98
- **data["radar"].data,
99
- **data["lidar"].data,
100
- **data["model"].data,
101
- **data["model"].data_sparse,
102
- **data["mwr"].data,
152
+ **data.radar.data,
153
+ **data.lidar.data,
154
+ **data.model.data,
155
+ **data.model.data_sparse,
156
+ **(data.mwr.data if data.mwr is not None else {}),
157
+ **(data.disdrometer.data if data.disdrometer is not None else {}),
103
158
  }
104
159
 
105
- def _define_dense_grid():
106
- return utils.time_grid(), data["radar"].height
160
+ def _define_dense_grid() -> tuple:
161
+ return utils.time_grid(), data.radar.height
107
162
 
108
- def _close_all():
109
- for obj in data.values():
110
- if isinstance(obj, Radar | Lidar | Mwr | Model):
111
- obj.close()
163
+ def _close_all() -> None:
164
+ if "data" in locals():
165
+ for field in fields(data):
166
+ obj = getattr(data, field.name)
167
+ if isinstance(obj, DataSource):
168
+ obj.close()
112
169
 
113
170
  try:
114
- data = {
115
- "radar": Radar(input_files["radar"]),
116
- "lidar": Lidar(input_files["lidar"]),
117
- "mwr": Mwr(input_files["mwr"]),
118
- "lv0_files": input_files.get("lv0_files", None),
119
- }
120
- assert data["radar"].altitude is not None
121
- data["model"] = Model(input_files["model"], data["radar"].altitude)
171
+ radar = Radar(input_files["radar"])
172
+ data = Observations(
173
+ radar=radar,
174
+ lidar=Lidar(input_files["lidar"]),
175
+ model=Model(input_files["model"], radar.altitude, options),
176
+ lv0_files=input_files.get("lv0_files"),
177
+ )
178
+ if "mwr" in input_files:
179
+ data.mwr = Mwr(input_files["mwr"])
180
+ if "disdrometer" in input_files:
181
+ try:
182
+ data.disdrometer = Disdrometer(input_files["disdrometer"])
183
+ except DisdrometerDataError as err:
184
+ logging.warning("Unable to use disdrometer: %s", err)
122
185
  time, height = _define_dense_grid()
186
+ data.radar.add_location(time)
123
187
  valid_ind = _interpolate_to_cloudnet_grid()
124
- if not valid_ind:
125
- raise ValidTimeStampError("No overlapping radar and lidar timestamps found")
188
+ if len(valid_ind) < 2:
189
+ msg = "Less than 2 overlapping radar, lidar and model timestamps found"
190
+ raise ValidTimeStampError(msg)
126
191
  _screen_bad_time_indices(valid_ind)
127
- if "rpg" in data["radar"].type.lower() or "basta" in data["radar"].type.lower():
128
- data["radar"].filter_speckle_noise()
129
- data["radar"].filter_1st_gate_artifact()
192
+
193
+ if any(source in data.radar.source_type.lower() for source in ("rpg", "basta")):
194
+ data.radar.filter_speckle_noise()
195
+ data.radar.filter_1st_gate_artifact()
130
196
  for variable in ("v", "v_sigma", "ldr"):
131
- data["radar"].filter_stripes(variable)
132
- data["radar"].remove_incomplete_pixels()
133
- data["model"].calc_wet_bulb()
197
+ data.radar.filter_stripes(variable)
198
+ data.radar.remove_incomplete_pixels()
199
+ data.model.calc_wet_bulb()
200
+
134
201
  classification = classify.classify_measurements(data)
135
- attenuations = atmos.get_attenuations(data, classification)
136
- data["radar"].correct_atten(attenuations)
137
- data["radar"].calc_errors(attenuations, classification)
202
+
203
+ attenuations = attenuation.get_attenuations(data, classification)
204
+
205
+ data.radar.correct_atten(attenuations)
206
+ data.radar.calc_errors(attenuations, classification.is_clutter)
207
+
138
208
  quality = classify.fetch_quality(data, classification, attenuations)
209
+
139
210
  cloudnet_arrays = _prepare_output()
140
- date = data["radar"].get_date()
211
+
212
+ date = data.radar.get_date()
141
213
  attributes = output.add_time_attribute(CATEGORIZE_ATTRIBUTES, date)
142
214
  attributes = output.add_time_attribute(attributes, date, "model_time")
143
215
  attributes = output.add_source_attribute(attributes, data)
144
216
  output.update_attributes(cloudnet_arrays, attributes)
145
- uuid = _save_cat(output_file, data, cloudnet_arrays, uuid)
217
+ _save_cat(output_file, data, cloudnet_arrays, uuid)
146
218
  return uuid
147
219
  finally:
148
220
  _close_all()
149
221
 
150
222
 
151
223
  def _save_cat(
152
- full_path: str, data_obs: dict, cloudnet_arrays: dict, uuid: str | None
153
- ) -> str:
224
+ full_path: str | PathLike,
225
+ data_obs: Observations,
226
+ cloudnet_arrays: dict,
227
+ uuid: UUID,
228
+ ) -> None:
154
229
  """Creates a categorize netCDF4 file and saves all data into it."""
155
-
156
230
  dims = {
157
- "time": len(data_obs["radar"].time),
158
- "height": len(data_obs["radar"].height),
159
- "model_time": len(data_obs["model"].time),
160
- "model_height": len(data_obs["model"].mean_height),
231
+ "time": len(data_obs.radar.time),
232
+ "height": len(data_obs.radar.height),
233
+ "model_time": len(data_obs.model.time),
234
+ "model_height": len(data_obs.model.mean_height),
161
235
  }
162
236
 
163
237
  file_type = "categorize"
238
+ if "liquid_prob" in cloudnet_arrays:
239
+ file_type += "-voodoo"
240
+
164
241
  with output.init_file(full_path, dims, cloudnet_arrays, uuid) as nc:
165
- uuid_out = nc.file_uuid
166
242
  nc.cloudnet_file_type = file_type
167
243
  output.copy_global(
168
- data_obs["radar"].dataset, nc, ("year", "month", "day", "location")
244
+ data_obs.radar.dataset,
245
+ nc,
246
+ ("year", "month", "day", "location"),
169
247
  )
170
- nc.title = f"Cloud categorization products from {data_obs['radar'].location}"
171
- nc.source_file_uuids = output.get_source_uuids(*data_obs.values())
248
+ nc.title = f"Cloud categorization products from {data_obs.radar.location}"
249
+ nc.source_file_uuids = output.get_source_uuids(data_obs)
250
+ is_voodoo = "liquid_prob" in cloudnet_arrays
172
251
  extra_references = (
173
- ["https://doi.org/10.5194/amt-15-5343-2022"]
174
- if "liquid_prob" in cloudnet_arrays
175
- else None
252
+ ["https://doi.org/10.5194/amt-15-5343-2022"] if is_voodoo else None
176
253
  )
177
254
  nc.references = output.get_references(
178
- identifier=file_type, extra=extra_references
255
+ identifier=file_type,
256
+ extra=extra_references,
179
257
  )
258
+ if is_voodoo:
259
+ import voodoonet.version # noqa: PLC0415
260
+
261
+ nc.voodoonet_version = voodoonet.version.__version__
180
262
  output.add_source_instruments(nc, data_obs)
181
263
  output.merge_history(nc, file_type, data_obs)
182
- return uuid_out
264
+
265
+
266
+ def _classes_to_bits(data: QualityBits | CategoryBits) -> NDArray[np.int_]:
267
+ shape = data.radar.shape if hasattr(data, "radar") else data.droplet.shape
268
+ quality = np.zeros(shape, dtype=np.int64)
269
+ for i, field in enumerate(dataclasses.fields(data)):
270
+ quality |= (1 << i) * getattr(data, field.name)
271
+ return quality
183
272
 
184
273
 
185
274
  COMMENTS = {
@@ -216,14 +305,17 @@ COMMENTS = {
216
305
  "humidity, but forcing pixels containing liquid cloud to saturation with\n"
217
306
  "respect to liquid water. It has been used to correct Z."
218
307
  ),
219
- "Tw": (
220
- "This variable was derived from model temperature, pressure and relative\n"
221
- "humidity."
308
+ "radar_rain_atten": (
309
+ "This variable was calculated from the disdrometer rainfall rate."
222
310
  ),
311
+ "radar_melting_atten": (
312
+ "This variable was calculated from the disdrometer rainfall rate."
313
+ ),
314
+ "Tw": "This variable was derived from model temperature, pressure and humidity.",
223
315
  "Z_sensitivity": (
224
316
  "This variable is an estimate of the radar sensitivity, i.e. the minimum\n"
225
317
  "detectable radar reflectivity, as a function of height. It includes the\n"
226
- "effect of ground clutter and gas attenuation but not liquid attenuation."
318
+ "effect of ground clutter and gas attenuation but not other attenuations."
227
319
  ),
228
320
  "Z_error": (
229
321
  "This variable is an estimate of the one-standard-deviation random error\n"
@@ -233,22 +325,22 @@ COMMENTS = {
233
325
  " finite number of pulses\n"
234
326
  "2) 10% uncertainty in gaseous attenuation correction (mainly due to error\n"
235
327
  " in model humidity field)\n"
236
- "3) Error in liquid water path (given by the variable lwp_error) and its\n"
328
+ "3) 20% uncertainty in rain attenuation correction (mainly due to error\n"
329
+ " in disdrometer rainfall rate)\n"
330
+ "4) 10%-20% uncertainty in melting layer attenuation correction (mainly due\n"
331
+ " to error in disdrometer rainfall rate)\n"
332
+ "5) Error in liquid water path (given by the variable lwp_error) and its\n"
237
333
  " partitioning with height)."
238
334
  ),
239
335
  "Z": (
240
- "This variable has been corrected for attenuation by gaseous attenuation\n"
241
- "(using the thermodynamic variables from a forecast model; see the\n"
242
- "radar_gas_atten variable) and liquid attenuation (using liquid water path\n"
243
- "from a microwave radiometer; see the radar_liquid_atten variable) but rain\n"
244
- "and melting-layer attenuation has not been corrected.\n"
245
- "Calibration convention: in the absence of attenuation, a cloud at 273 K\n"
246
- "containing one million 100-micron droplets per cubic metre will have\n"
247
- "a reflectivity of 0 dBZ at all frequencies."
336
+ "This variable has been corrected for attenuation by gaseous attenuation,\n"
337
+ "and possibly liquid water, rain and melting layer (see quality_bits\n"
338
+ "variable). Calibration convention: in the absence of attenuation, a cloud\n"
339
+ "at 273 K containing one million 100-micron droplets per cubic metre will\n"
340
+ "have\n a reflectivity of 0 dBZ at all frequencies."
248
341
  ),
249
342
  "bias": (
250
- "This variable is an estimate of the one-standard-deviation\n"
251
- "calibration error."
343
+ "This variable is an estimate of the one-standard-deviation\ncalibration error."
252
344
  ),
253
345
  "insect_prob": (
254
346
  "Ad-hoc estimation of the probability that the pixel contains insects."
@@ -259,61 +351,94 @@ COMMENTS = {
259
351
  }
260
352
 
261
353
  DEFINITIONS = {
262
- "category_bits": (
263
- "\n"
264
- "Bit 0: Small liquid droplets are present.\n"
265
- "Bit 1: Falling hydrometeors are present; if Bit 2 is set then these are most\n"
266
- " likely ice particles, otherwise they are drizzle or rain drops.\n"
267
- "Bit 2: Wet-bulb temperature is less than 0 degrees C, implying the phase of\n"
268
- " Bit-1 particles.\n"
269
- "Bit 3: Melting ice particles are present.\n"
270
- "Bit 4: Aerosol particles are present and visible to the lidar.\n"
271
- "Bit 5: Insects are present and visible to the radar."
354
+ "category_bits": utils.bit_field_definition(
355
+ {
356
+ 0: """Small liquid droplets are present.""",
357
+ 1: """Falling hydrometeors are present; if Bit 2 is set then these
358
+ are most likely ice particles, otherwise they are drizzle or
359
+ rain drops.""",
360
+ 2: """Wet-bulb temperature is less than 0 degrees C, implying the
361
+ phase of Bit-1 particles.""",
362
+ 3: """Melting ice particles are present.""",
363
+ 4: """Aerosol particles are present and visible to the lidar.""",
364
+ 5: """Insects are present and visible to the radar.""",
365
+ }
272
366
  ),
273
- "quality_bits": (
274
- "\n"
275
- "Bit 0: An echo is detected by the radar.\n"
276
- "Bit 1: An echo is detected by the lidar.\n"
277
- "Bit 2: The apparent echo detected by the radar is ground clutter or some\n"
278
- " other non-atmospheric artifact.\n"
279
- "Bit 3: The lidar echo is due to clear-air molecular scattering.\n"
280
- "Bit 4: Liquid water cloud, rainfall or melting ice below this pixel\n"
281
- " will have caused radar and lidar attenuation; if bit 5 is set then\n"
282
- " a correction for the radar attenuation has been performed;\n"
283
- " otherwise do not trust the absolute values of reflectivity factor.\n"
284
- " No correction is performed for lidar attenuation.\n"
285
- "Bit 5: Radar reflectivity has been corrected for liquid-water attenuation\n"
286
- " using the microwave radiometer measurements of liquid water path\n"
287
- " and the lidar estimation of the location of liquid water cloud;\n"
288
- " be aware that errors in reflectivity may result.\n"
367
+ "quality_bits": utils.bit_field_definition(
368
+ {
369
+ 0: """An echo is detected by the radar.""",
370
+ 1: """An echo is detected by the lidar.""",
371
+ 2: """The apparent echo detected by the radar is ground clutter or
372
+ some other non-atmospheric artifact.""",
373
+ 3: """The lidar echo is due to clear-air molecular scattering.""",
374
+ 4: """Liquid water cloud, rainfall or melting ice below this pixel
375
+ will have caused radar and lidar attenuation; if bit 5 is set
376
+ then a correction for the radar attenuation has been
377
+ performed; otherwise do not trust the absolute values of
378
+ reflectivity factor. No correction is performed for lidar
379
+ attenuation.""",
380
+ 5: """Radar reflectivity has been corrected for liquid-water
381
+ attenuation using the microwave radiometer measurements of
382
+ liquid water path and the lidar estimation of the location of
383
+ liquid water cloud; be aware that errors in reflectivity may
384
+ result.""",
385
+ 6: """Rain has caused radar attenuation; if bit 7 is set then a
386
+ correction for the radar attenuation has been performed;
387
+ otherwise do not trust the absolute values of reflectivity
388
+ factor. No correction is performed for lidar attenuation.""",
389
+ 7: """Radar reflectivity has been corrected for rain attenuation
390
+ using rainfall rate from a disdrometer; be aware that errors
391
+ in reflectivity may result.""",
392
+ 8: """Melting layer has caused radar attenuation; if bit 9 is set then a
393
+ correction for the radar attenuation has been performed;
394
+ otherwise do not trust the absolute values of reflectivity
395
+ factor. No correction is performed for lidar attenuation.""",
396
+ 9: """Radar reflectivity has been corrected for melting layer
397
+ attenuation; be aware that errors in reflectivity may result.""",
398
+ }
289
399
  ),
290
400
  }
291
401
 
292
402
  CATEGORIZE_ATTRIBUTES = {
403
+ "height": COMMON_ATTRIBUTES["height"]._replace(dimensions=("height",)),
293
404
  # Radar variables
294
405
  "Z": MetaData(
295
406
  long_name="Radar reflectivity factor",
296
407
  units="dBZ",
297
408
  comment=COMMENTS["Z"],
298
409
  ancillary_variables="Z_error Z_bias Z_sensitivity",
410
+ dimensions=("time", "height"),
299
411
  ),
300
412
  "Z_error": MetaData(
301
413
  long_name="Error in radar reflectivity factor",
302
414
  units="dB",
303
415
  comment=COMMENTS["Z_error"],
416
+ dimensions=("time", "height"),
304
417
  ),
305
418
  "Z_bias": MetaData(
306
419
  long_name="Bias in radar reflectivity factor",
307
420
  units="dB",
308
421
  comment=COMMENTS["bias"],
422
+ dimensions=None,
309
423
  ),
310
424
  "Z_sensitivity": MetaData(
311
425
  long_name="Minimum detectable radar reflectivity",
312
426
  units="dBZ",
313
427
  comment=COMMENTS["Z_sensitivity"],
428
+ dimensions=("height",),
314
429
  ),
430
+ "v": COMMON_ATTRIBUTES["v"]._replace(dimensions=("time", "height")),
431
+ "width": COMMON_ATTRIBUTES["width"]._replace(dimensions=("time", "height")),
432
+ "ldr": COMMON_ATTRIBUTES["ldr"]._replace(dimensions=("time", "height")),
433
+ "sldr": COMMON_ATTRIBUTES["sldr"]._replace(dimensions=("time", "height")),
315
434
  "v_sigma": MetaData(
316
- long_name="Standard deviation of mean Doppler velocity", units="m s-1"
435
+ long_name="Standard deviation of mean Doppler velocity",
436
+ units="m s-1",
437
+ dimensions=("time", "height"),
438
+ ),
439
+ "zdr": RPG_ATTRIBUTES["zdr"]._replace(dimensions=("time", "height")),
440
+ "nyquist_velocity": COMMON_ATTRIBUTES["nyquist_velocity"]._replace(
441
+ dimensions=("time", "height")
317
442
  ),
318
443
  # Lidar variables
319
444
  "beta": MetaData(
@@ -321,77 +446,128 @@ CATEGORIZE_ATTRIBUTES = {
321
446
  units="sr-1 m-1",
322
447
  comment="SNR-screened attenuated backscatter coefficient.",
323
448
  ancillary_variables="beta_error beta_bias",
449
+ dimensions=("time", "height"),
324
450
  ),
325
451
  "beta_error": MetaData(
326
452
  long_name="Error in attenuated backscatter coefficient",
327
453
  units="dB",
454
+ dimensions=None,
328
455
  ),
329
456
  "beta_bias": MetaData(
330
457
  long_name="Bias in attenuated backscatter coefficient",
331
458
  units="dB",
459
+ dimensions=None,
460
+ ),
461
+ "lidar_wavelength": MetaData(
462
+ long_name="Laser wavelength", units="nm", dimensions=None
332
463
  ),
333
- "lidar_wavelength": MetaData(long_name="Laser wavelength", units="nm"),
334
464
  # MWR variables
335
465
  "lwp_error": MetaData(
336
466
  long_name="Error in liquid water path",
337
467
  units="kg m-2",
468
+ dimensions=("time",),
469
+ ),
470
+ "lwp": COMMON_ATTRIBUTES["lwp"]._replace(
471
+ ancillary_variables="lwp_error", dimensions=("time",)
338
472
  ),
339
- "lwp": MetaData(ancillary_variables="lwp_error"),
340
473
  # Model variables
341
- "Tw": MetaData(long_name="Wet-bulb temperature", units="K", comment=COMMENTS["Tw"]),
342
- "model_time": MetaData(long_name="Model time UTC", calendar="standard"),
474
+ "Tw": MetaData(
475
+ long_name="Wet-bulb temperature",
476
+ units="K",
477
+ comment=COMMENTS["Tw"],
478
+ dimensions=("time", "height"),
479
+ ),
480
+ "model_time": MetaData(
481
+ long_name="Model time UTC",
482
+ calendar="standard",
483
+ dimensions=("model_time",),
484
+ ),
343
485
  "model_height": MetaData(
344
486
  long_name="Height of model variables above mean sea level",
345
487
  units="m",
346
488
  axis="Z",
489
+ dimensions=("model_height",),
490
+ ),
491
+ "temperature": MetaData(
492
+ long_name="Temperature", units="K", dimensions=("model_time", "model_height")
493
+ ),
494
+ "pressure": MetaData(
495
+ long_name="Pressure", units="Pa", dimensions=("model_time", "model_height")
347
496
  ),
348
497
  "vwind": MetaData(
349
498
  long_name="Meridional wind",
350
499
  units="m s-1",
500
+ dimensions=("model_time", "model_height"),
351
501
  ),
352
502
  "uwind": MetaData(
353
503
  long_name="Zonal wind",
354
504
  units="m s-1",
505
+ dimensions=("model_time", "model_height"),
506
+ ),
507
+ "q": MetaData(
508
+ long_name="Specific humidity",
509
+ units="1",
510
+ dimensions=("model_time", "model_height"),
355
511
  ),
356
- "q": MetaData(long_name="Specific humidity", units="1"),
357
512
  # MISC
358
513
  "category_bits": MetaData(
359
514
  long_name="Target categorization bits",
360
515
  comment=COMMENTS["category_bits"],
361
516
  definition=DEFINITIONS["category_bits"],
362
517
  units="1",
518
+ dimensions=("time", "height"),
363
519
  ),
364
520
  "quality_bits": MetaData(
365
521
  long_name="Data quality bits",
366
522
  comment=COMMENTS["quality_bits"],
367
523
  definition=DEFINITIONS["quality_bits"],
368
524
  units="1",
369
- ),
370
- "rainfall_rate": MetaData(
371
- long_name="Rainfall rate",
372
- standard_name="rainfall_rate",
373
- units="m s-1",
374
- comment="Fill values denote rain with undefined intensity.",
525
+ dimensions=("time", "height"),
375
526
  ),
376
527
  "radar_liquid_atten": MetaData(
377
528
  long_name="Two-way radar attenuation due to liquid water",
378
529
  units="dB",
379
530
  comment=COMMENTS["radar_liquid_atten"],
531
+ references="ITU-R P.840-9",
532
+ dimensions=("time", "height"),
533
+ ),
534
+ "radar_rain_atten": MetaData(
535
+ long_name="Two-way radar attenuation due to rain",
536
+ units="dB",
537
+ references="Crane, R. (1980)",
538
+ comment=COMMENTS["radar_rain_atten"],
539
+ dimensions=("time", "height"),
540
+ ),
541
+ "radar_melting_atten": MetaData(
542
+ long_name="Two-way radar attenuation due to melting ice",
543
+ units="dB",
544
+ references="Li, H., & Moisseev, D. (2019)",
545
+ comment=COMMENTS["radar_melting_atten"],
546
+ dimensions=("time", "height"),
380
547
  ),
381
548
  "radar_gas_atten": MetaData(
382
549
  long_name="Two-way radar attenuation due to atmospheric gases",
383
550
  units="dB",
384
551
  comment=COMMENTS["radar_gas_atten"],
385
- references="Liebe (1985, Radio Sci. 20(5), 1069-1089)",
552
+ references="ITU-R P.676-13",
553
+ dimensions=("time", "height"),
386
554
  ),
387
555
  "insect_prob": MetaData(
388
556
  long_name="Insect probability",
389
557
  units="1",
390
558
  comment=COMMENTS["insect_prob"],
559
+ dimensions=("time", "height"),
391
560
  ),
392
561
  "liquid_prob": MetaData(
393
562
  long_name="Liquid probability",
394
563
  units="1",
395
564
  comment=COMMENTS["liquid_prob"],
565
+ dimensions=("time", "height"),
566
+ ),
567
+ "rain_detected": MetaData(
568
+ long_name="Rain detected",
569
+ units="1",
570
+ comment="1 = rain detected, 0 = no rain detected",
571
+ dimensions=("time",),
396
572
  ),
397
573
  }