cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,17 +1,27 @@
1
- """Module containing low-level functions to classify gridded
2
- radar / lidar measurements.
3
- """
4
1
  import numpy as np
5
- import skimage
2
+ import numpy.typing as npt
6
3
  from numpy import ma
7
4
 
8
- import cloudnetpy.categorize.atmos
9
5
  from cloudnetpy import utils
10
- from cloudnetpy.categorize import droplet, falling, freezing, insects, melting
11
- from cloudnetpy.categorize.containers import ClassData, ClassificationResult
12
-
13
-
14
- def classify_measurements(data: dict) -> ClassificationResult:
6
+ from cloudnetpy.categorize import (
7
+ atmos_utils,
8
+ droplet,
9
+ falling,
10
+ freezing,
11
+ insects,
12
+ melting,
13
+ )
14
+ from cloudnetpy.categorize.attenuations import RadarAttenuation
15
+ from cloudnetpy.categorize.containers import (
16
+ ClassData,
17
+ ClassificationResult,
18
+ Observations,
19
+ )
20
+ from cloudnetpy.constants import T0
21
+ from cloudnetpy.products.product_tools import CategoryBits, QualityBits
22
+
23
+
24
+ def classify_measurements(data: Observations) -> ClassificationResult:
15
25
  """Classifies radar/lidar observations.
16
26
 
17
27
  This function classifies atmospheric scatterers from the input data.
@@ -19,8 +29,7 @@ def classify_measurements(data: dict) -> ClassificationResult:
19
29
  time / height grid before calling this function.
20
30
 
21
31
  Args:
22
- data: Containing :class:`Radar`, :class:`Lidar`, :class:`Model`
23
- and :class:`Mwr` instances.
32
+ data: A :class:`Observations` instance.
24
33
 
25
34
  Returns:
26
35
  A :class:`ClassificationResult` instance.
@@ -36,132 +45,142 @@ def classify_measurements(data: dict) -> ClassificationResult:
36
45
  Especially methods classifying insects, melting layer and liquid droplets.
37
46
 
38
47
  """
48
+ bits = CategoryBits(
49
+ droplet=np.array([], dtype=bool),
50
+ falling=np.array([], dtype=bool),
51
+ freezing=np.array([], dtype=bool),
52
+ melting=np.array([], dtype=bool),
53
+ aerosol=np.array([], dtype=bool),
54
+ insect=np.array([], dtype=bool),
55
+ )
56
+
39
57
  obs = ClassData(data)
40
- bits: list[np.ndarray] = [np.array([])] * 6
41
- bits[3] = melting.find_melting_layer(obs)
42
- bits[2] = freezing.find_freezing_region(obs, bits[3])
58
+
59
+ bits.melting = melting.find_melting_layer(obs)
60
+ bits.freezing = freezing.find_freezing_region(obs, bits.melting)
43
61
  liquid_from_lidar = droplet.find_liquid(obs)
44
62
  if obs.lv0_files is not None and len(obs.lv0_files) > 0:
45
- import voodoonet # pylint: disable=import-outside-toplevel
46
-
47
- target_time = voodoonet.utils.decimal_hour2unix(obs.date, obs.time)
48
- liquid_prob = voodoonet.infer(obs.lv0_files, target_time=target_time)
63
+ if "rpg-fmcw-94" not in obs.radar_type.lower():
64
+ msg = "VoodooNet is only implemented for RPG-FMCW-94 radar."
65
+ raise NotImplementedError(msg)
66
+ import voodoonet # noqa: PLC0415
67
+
68
+ options = voodoonet.VoodooOptions(progress_bar=False)
69
+ dumb_date = obs.date.isoformat().split("-")
70
+ target_time = voodoonet.utils.decimal_hour2unix(dumb_date, obs.time)
71
+ liquid_prob = voodoonet.infer(
72
+ list(obs.lv0_files), target_time=target_time, options=options
73
+ )
49
74
  liquid_from_radar = liquid_prob > 0.55
50
75
  liquid_from_radar = _remove_false_radar_liquid(
51
- liquid_from_radar, liquid_from_lidar
76
+ liquid_from_radar,
77
+ liquid_from_lidar,
52
78
  )
53
- bits[0] = liquid_from_radar | liquid_from_lidar
79
+ liquid_from_radar[~bits.freezing] = 0
80
+ is_liquid = liquid_from_radar | liquid_from_lidar
54
81
  else:
55
- bits[0] = droplet.correct_liquid_top(obs, liquid_from_lidar, bits[2], limit=500)
82
+ is_liquid = liquid_from_lidar
56
83
  liquid_prob = None
57
- bits[5], insect_prob = insects.find_insects(obs, bits[3], bits[0])
58
- bits[1] = falling.find_falling_hydrometeors(obs, bits[0], bits[5])
59
- bits, filtered_ice = _filter_falling(bits)
84
+ bits.droplet = droplet.correct_liquid_top(obs, is_liquid, bits.freezing, limit=500)
85
+ bits.insect, insect_prob = insects.find_insects(obs, bits.melting, bits.droplet)
86
+ bits.falling = falling.find_falling_hydrometeors(obs, bits.droplet, bits.insect)
87
+ filtered_ice = _filter_falling(bits)
60
88
  for _ in range(5):
61
- bits[3] = _fix_undetected_melting_layer(bits)
62
- bits = _filter_insects(bits)
63
- bits[4] = _find_aerosols(obs, bits[1], bits[0])
64
- bits[4][filtered_ice] = False
89
+ _fix_undetected_melting_layer(bits)
90
+ _filter_insects(bits)
91
+ bits.aerosol = _find_aerosols(obs, bits)
92
+ bits.aerosol[filtered_ice] = False
93
+ _fix_super_cold_liquid(obs, bits)
94
+
65
95
  return ClassificationResult(
66
- _bits_to_integer(bits),
67
- obs.is_rain,
68
- obs.is_clutter,
69
- obs.rainfall_rate,
70
- insect_prob,
71
- liquid_prob,
96
+ category_bits=bits,
97
+ is_rain=obs.is_rain,
98
+ is_clutter=obs.is_clutter,
99
+ insect_prob=insect_prob,
100
+ liquid_prob=liquid_prob,
72
101
  )
73
102
 
74
103
 
75
- def _remove_false_radar_liquid(
76
- liquid_from_radar: np.ndarray, liquid_from_lidar: np.ndarray
77
- ) -> np.ndarray:
78
- """Removes radar-liquid below lidar-detected liquid bases."""
79
- lidar_liquid_bases = cloudnetpy.categorize.atmos.find_cloud_bases(liquid_from_lidar)
80
- for prof, base in zip(*np.where(lidar_liquid_bases)):
81
- liquid_from_radar[prof, 0:base] = 0
82
- return liquid_from_radar
83
-
84
-
85
104
  def fetch_quality(
86
- data: dict, classification: ClassificationResult, attenuations: dict
87
- ) -> dict:
88
- """Returns Cloudnet quality bits.
105
+ data: Observations,
106
+ classification: ClassificationResult,
107
+ attenuations: RadarAttenuation,
108
+ ) -> QualityBits:
109
+ return QualityBits(
110
+ radar=~data.radar.data["Z"][:].mask,
111
+ lidar=~data.lidar.data["beta"][:].mask,
112
+ clutter=classification.is_clutter,
113
+ molecular=np.zeros(data.radar.data["Z"][:].shape, dtype=bool),
114
+ attenuated_liquid=attenuations.liquid.attenuated,
115
+ corrected_liquid=attenuations.liquid.attenuated
116
+ & ~attenuations.liquid.uncorrected,
117
+ attenuated_rain=attenuations.rain.attenuated,
118
+ corrected_rain=attenuations.rain.attenuated & ~attenuations.rain.uncorrected,
119
+ attenuated_melting=attenuations.melting.attenuated,
120
+ corrected_melting=attenuations.melting.attenuated
121
+ & ~attenuations.melting.uncorrected,
122
+ )
89
123
 
90
- Args:
91
- data: Containing :class:`Radar` and :class:`Lidar` instances.
92
- classification: A :class:`ClassificationResult` instance.
93
- attenuations: Dictionary containing keys `liquid_corrected`,
94
- `liquid_uncorrected`.
95
124
 
96
- Returns:
97
- Dictionary containing `quality_bits`, an integer array with the bits:
125
+ def _fix_super_cold_liquid(obs: ClassData, bits: CategoryBits) -> None:
126
+ """Supercooled liquid droplets do not exist in atmosphere below around -38 C."""
127
+ t_limit = T0 - 38
128
+ super_cold_liquid = np.where((obs.tw < t_limit) & bits.droplet)
129
+ bits.droplet[super_cold_liquid] = False
130
+ bits.falling[super_cold_liquid] = True
98
131
 
99
- - bit 0: Pixel contains radar data
100
- - bit 1: Pixel contains lidar data
101
- - bit 2: Pixel contaminated by radar clutter
102
- - bit 3: Molecular scattering present (currently not implemented!)
103
- - bit 4: Pixel was affected by liquid attenuation
104
- - bit 5: Liquid attenuation was corrected
105
- - bit 6: Data gap in radar or lidar data
106
132
 
107
- """
108
- bits: list[np.ndarray] = [np.ndarray([])] * 7
109
- radar_echo = data["radar"].data["Z"][:]
110
- bits[0] = ~radar_echo.mask
111
- bits[1] = ~data["lidar"].data["beta"][:].mask
112
- bits[2] = classification.is_clutter
113
- bits[4] = attenuations["liquid_corrected"] | attenuations["liquid_uncorrected"]
114
- bits[5] = attenuations["liquid_corrected"]
115
- qbits = _bits_to_integer(bits)
116
- return {"quality_bits": qbits}
133
+ def _remove_false_radar_liquid(
134
+ liquid_from_radar: npt.NDArray,
135
+ liquid_from_lidar: npt.NDArray,
136
+ ) -> npt.NDArray[np.bool_]:
137
+ """Removes radar-liquid below lidar-detected liquid bases."""
138
+ lidar_liquid_bases = atmos_utils.find_cloud_bases(liquid_from_lidar)
139
+ for prof, base in zip(*np.where(lidar_liquid_bases), strict=True):
140
+ liquid_from_radar[prof, 0:base] = 0
141
+ return liquid_from_radar
117
142
 
118
143
 
119
144
  def _find_aerosols(
120
- obs: ClassData, is_falling: np.ndarray, is_liquid: np.ndarray
121
- ) -> np.ndarray:
145
+ obs: ClassData,
146
+ bits: CategoryBits,
147
+ ) -> npt.NDArray[np.bool_]:
122
148
  """Estimates aerosols from lidar backscattering.
123
149
 
124
150
  Aerosols are lidar signals that are: a) not falling, b) not liquid droplets.
125
151
 
126
152
  Args:
127
153
  obs: A :class:`ClassData` instance.
128
- is_falling: 2-D boolean array of falling hydrometeors.
129
- is_liquid: 2-D boolean array of liquid droplets.
154
+ bits: A :class:`CategoryBits instance.
130
155
 
131
156
  Returns:
132
157
  2-D boolean array containing aerosols.
133
158
 
134
159
  """
135
160
  is_beta = ~obs.beta.mask
136
- return is_beta & ~is_falling & ~is_liquid
161
+ return is_beta & ~bits.falling & ~bits.droplet
137
162
 
138
163
 
139
- def _fix_undetected_melting_layer(bits: list) -> np.ndarray:
140
- melting_layer = bits[3]
141
- drizzle_and_falling = _find_drizzle_and_falling(*bits[:3])
164
+ def _fix_undetected_melting_layer(bits: CategoryBits) -> None:
165
+ drizzle_and_falling = _find_drizzle_and_falling(bits)
142
166
  transition = ma.diff(drizzle_and_falling, axis=1) == -1
143
- melting_layer[:, 1:][transition] = True
144
- return melting_layer
167
+ bits.melting[:, 1:][transition] = True
145
168
 
146
169
 
147
- def _find_drizzle_and_falling(
148
- is_liquid: np.ndarray, is_falling: np.ndarray, is_freezing: np.ndarray
149
- ) -> np.ndarray:
170
+ def _find_drizzle_and_falling(bits: CategoryBits) -> npt.NDArray:
150
171
  """Classifies pixels as falling, drizzle and others.
151
172
 
152
173
  Args:
153
- is_liquid: 2D boolean array denoting liquid layers.
154
- is_falling: 2D boolean array denoting falling pixels.
155
- is_freezing: 2D boolean array denoting subzero temperatures.
174
+ bits: A :class:`CategoryBits instance.
156
175
 
157
176
  Returns:
158
177
  2D array where values are 1 (falling, drizzle, supercooled liquids),
159
178
  2 (drizzle), and masked (all others).
160
179
 
161
180
  """
162
- falling_dry = is_falling & ~is_liquid
163
- supercooled_liquids = is_liquid & is_freezing
164
- drizzle = falling_dry & ~is_freezing
181
+ falling_dry = bits.falling & ~bits.droplet
182
+ supercooled_liquids = bits.droplet & bits.freezing
183
+ drizzle = falling_dry & ~bits.freezing
165
184
  drizzle_and_falling = falling_dry.astype(int) + drizzle.astype(int)
166
185
  drizzle_and_falling = ma.copy(drizzle_and_falling)
167
186
  drizzle_and_falling[supercooled_liquids] = 1
@@ -169,28 +188,10 @@ def _find_drizzle_and_falling(
169
188
  return drizzle_and_falling
170
189
 
171
190
 
172
- def _bits_to_integer(bits: list) -> np.ndarray:
173
- """Creates array of integers from individual boolean arrays.
174
-
175
- Args:
176
- bits: List of bit fields (of similar sizes) to be saved in the resulting
177
- array of integers. bits[0] is saved as bit 0, bits[1] as bit 1, etc.
178
-
179
- Returns:
180
- Array of integers containing the information of the individual boolean arrays.
181
-
182
- """
183
- int_array = np.zeros_like(bits[0], dtype=int)
184
- for n, bit in enumerate(bits):
185
- ind = np.where(bit) # works also if bit is None
186
- int_array[ind] = utils.setbit(int_array[ind].astype(int), n)
187
- return int_array
188
-
189
-
190
- def _filter_insects(bits: list) -> list:
191
- is_melting_layer = bits[3]
192
- is_insects = bits[5]
193
- is_falling = bits[1]
191
+ def _filter_insects(bits: CategoryBits) -> None:
192
+ is_melting_layer = bits.melting
193
+ is_insects = bits.insect
194
+ is_falling = bits.falling
194
195
 
195
196
  # Remove above melting layer
196
197
  above_melting = utils.ffill(is_melting_layer)
@@ -201,7 +202,7 @@ def _filter_insects(bits: list) -> list:
201
202
  # remove around melting layer:
202
203
  original_insects = np.copy(is_insects)
203
204
  n_gates = 5
204
- for x, y in zip(*np.where(is_melting_layer)):
205
+ for x, y in zip(*np.where(is_melting_layer), strict=True):
205
206
  try:
206
207
  # change insects to drizzle below melting layer pixel
207
208
  ind1 = np.arange(y - n_gates, y)
@@ -219,17 +220,18 @@ def _filter_insects(bits: list) -> list:
219
220
  is_insects[ind1[ind11], y - 1 : y + 2] = False
220
221
  except IndexError:
221
222
  continue
222
- bits[1] = is_falling
223
- bits[5] = is_insects
224
- return bits
223
+ bits.falling = is_falling
224
+ bits.insect = is_insects
225
225
 
226
226
 
227
- def _filter_falling(bits: list) -> tuple:
227
+ def _filter_falling(bits: CategoryBits) -> tuple:
228
228
  # filter falling ice speckle noise
229
- is_freezing = bits[2]
230
- is_falling = bits[1]
231
- is_falling_filtered = skimage.morphology.remove_small_objects(
232
- is_falling, 10, connectivity=1
229
+ is_freezing = bits.freezing
230
+ is_falling = bits.falling
231
+ is_falling_filtered = utils.remove_small_objects(
232
+ is_falling,
233
+ max_size=10,
234
+ connectivity=1,
233
235
  )
234
236
  is_filtered = is_falling & ~np.array(is_falling_filtered)
235
237
  ice_ind = np.where(is_freezing & is_filtered)
@@ -237,6 +239,6 @@ def _filter_falling(bits: list) -> tuple:
237
239
  # in warm these are (probably) insects
238
240
  insect_ind = np.where(~is_freezing & is_filtered)
239
241
  is_falling[insect_ind] = False
240
- bits[1] = is_falling
241
- bits[5][insect_ind] = True
242
- return bits, ice_ind
242
+ bits.falling = is_falling
243
+ bits.insect[insect_ind] = True
244
+ return ice_ind
@@ -1,23 +1,39 @@
1
- import logging
1
+ from collections.abc import Sequence
2
2
  from dataclasses import dataclass
3
+ from os import PathLike
3
4
 
4
5
  import numpy as np
5
- import skimage
6
+ import numpy.typing as npt
6
7
  from numpy import ma
7
8
 
8
9
  from cloudnetpy import utils
10
+ from cloudnetpy.constants import MM_H_TO_M_S, T0
11
+ from cloudnetpy.products.product_tools import CategoryBits
12
+
13
+ from .disdrometer import Disdrometer
14
+ from .lidar import Lidar
15
+ from .model import Model
16
+ from .mwr import Mwr
17
+ from .radar import Radar
9
18
 
10
19
 
11
20
  @dataclass
12
- class ClassificationResult:
13
- """Result of classification"""
21
+ class Observations:
22
+ radar: Radar
23
+ lidar: Lidar
24
+ model: Model
25
+ mwr: Mwr | None = None
26
+ disdrometer: Disdrometer | None = None
27
+ lv0_files: Sequence[str | PathLike] | None = None
14
28
 
15
- category_bits: np.ndarray
16
- is_rain: np.ndarray
17
- is_clutter: np.ndarray
18
- rainfall_rate: np.ndarray
19
- insect_prob: np.ndarray
20
- liquid_prob: np.ndarray | None
29
+
30
+ @dataclass
31
+ class ClassificationResult:
32
+ category_bits: CategoryBits
33
+ is_rain: npt.NDArray
34
+ is_clutter: npt.NDArray
35
+ insect_prob: npt.NDArray
36
+ liquid_prob: npt.NDArray | None
21
37
 
22
38
 
23
39
  class ClassData:
@@ -42,86 +58,100 @@ class ClassData:
42
58
  radar_type (str): Radar identifier.
43
59
  is_rain (ndarray): 2D boolean array denoting rain.
44
60
  is_clutter (ndarray): 2D boolean array denoting clutter.
45
- rainfall_rate: 1D rain rate.
46
61
  altitude: site altitude.
47
62
 
48
63
  """
49
64
 
50
- def __init__(self, data: dict):
51
- self.z = data["radar"].data["Z"][:]
52
- self.v = data["radar"].data["v"][:]
53
- self.v_sigma = data["radar"].data["v_sigma"][:]
65
+ def __init__(self, data: Observations) -> None:
66
+ self.data = data
67
+ self.z = data.radar.data["Z"][:]
68
+ self.v = data.radar.data["v"][:]
69
+ self.v_sigma = data.radar.data["v_sigma"][:]
54
70
  for key in ("width", "ldr", "sldr"):
55
- if key in data["radar"].data.keys():
56
- setattr(self, key, data["radar"].data[key][:])
57
- self.time = data["radar"].time
58
- self.height = data["radar"].height
59
- self.radar_type = data["radar"].type
60
- self.tw = data["model"].data["Tw"][:]
61
- self.model_type = data["model"].type
62
- self.beta = data["lidar"].data["beta"][:]
63
- self.lwp = data["mwr"].data["lwp"][:]
64
- self.is_rain = _find_rain_from_radar_echo(self.z, self.time)
65
- self.rainfall_rate = _find_rainfall_rate(self.is_rain, data["radar"])
71
+ if key in data.radar.data:
72
+ setattr(self, key, data.radar.data[key][:])
73
+ self.time = data.radar.time
74
+ self.height = data.radar.height
75
+ self.radar_type = data.radar.source_type
76
+ self.tw = data.model.data["Tw"][:]
77
+ self.model_type = data.model.source_type
78
+ self.beta = data.lidar.data["beta"][:]
79
+ self.lwp = (
80
+ data.mwr.data["lwp"][:]
81
+ if data.mwr is not None
82
+ else ma.masked_all(self.time.shape)
83
+ )
84
+ self.is_rain = self._find_profiles_with_rain()
66
85
  self.is_clutter = _find_clutter(self.v, self.is_rain)
67
- self.altitude = data["radar"].altitude
68
- self.lv0_files = data["lv0_files"]
69
- self.date = data["radar"].get_date()
70
-
71
-
72
- def _find_rain_from_radar_echo(
73
- z: np.ndarray, time: np.ndarray, time_buffer: int = 5
74
- ) -> np.ndarray:
75
- """Find profiles affected by rain.
76
-
77
- Rain is present in such profiles where the radar echo in
78
- the third range gate is > 0 dB. To make sure we do not include any
79
- rainy profiles, we also flag a few profiles before and after
80
- detections as raining.
81
-
82
- Args:
83
- z: Radar echo.
84
- time: Time vector.
85
- time_buffer: Time in minutes.
86
-
87
- Returns:
88
- 1D Boolean array denoting profiles with rain.
89
-
90
- """
91
- is_rain = ma.array(z[:, 3] > 0, dtype=bool).filled(False)
92
- is_rain = skimage.morphology.remove_small_objects(
93
- is_rain, 2, connectivity=1
94
- ) # Filter hot pixels
95
- n_profiles = len(time)
96
- n_steps = utils.n_elements(time, time_buffer, "time")
97
- for ind in np.where(is_rain)[0]:
98
- ind1 = max(0, ind - n_steps)
99
- ind2 = min(ind + n_steps, n_profiles)
100
- is_rain[ind1 : ind2 + 1] = True
101
- return is_rain
102
-
103
-
104
- def _find_rainfall_rate(is_rain: np.ndarray, radar) -> np.ndarray:
105
- rainfall_rate = ma.zeros(len(is_rain))
106
- rainfall_rate[is_rain] = ma.masked
107
- if "rainfall_rate" in radar.data:
108
- radar_rainfall_rate = radar.data["rainfall_rate"].data
109
- ind = np.where(~radar_rainfall_rate.mask)
110
- rainfall_rate[ind] = radar_rainfall_rate[ind]
111
- else:
112
- logging.info("No measured rain rate available")
113
- return rainfall_rate
86
+ self.altitude = data.radar.altitude
87
+ self.lv0_files = data.lv0_files
88
+ self.date = data.radar.get_date()
89
+
90
+ def _find_profiles_with_rain(self) -> npt.NDArray:
91
+ is_rain = self._find_rain_from_radar_echo()
92
+ rain_from_disdrometer = self._find_rain_from_disdrometer()
93
+ ind = ~rain_from_disdrometer.mask
94
+ is_rain[ind] = rain_from_disdrometer[ind]
95
+ # Filter out snowfall:
96
+ if (
97
+ self.data.disdrometer is not None
98
+ and "synop_WaWa" in self.data.disdrometer.data
99
+ ):
100
+ wawa = self.data.disdrometer.data["synop_WaWa"].data
101
+ liquid_rain_only = (wawa >= 57) & (wawa <= 68)
102
+ else:
103
+ mask = ma.getmaskarray(self.tw)
104
+ first_valid_ind = np.nonzero(np.all(~mask, axis=0))[0][0]
105
+ liquid_rain_only = self.tw[:, first_valid_ind] > T0 + 5
106
+ return is_rain & liquid_rain_only
107
+
108
+ def _find_rain_from_radar_echo(self) -> npt.NDArray:
109
+ first_gate_with_data = np.argmin(self.z.mask.all(axis=0))
110
+ gate_number = first_gate_with_data + 3
111
+ threshold = {"z": 3, "v": 0, "ldr": -15}
112
+ z = self.z[:, gate_number]
113
+ v = self.v[:, gate_number]
114
+ if hasattr(self, "ldr"):
115
+ ldr = self.ldr[:, gate_number]
116
+ elif hasattr(self, "sldr"):
117
+ ldr = self.sldr[:, gate_number]
118
+ else:
119
+ ldr = np.full(self.time.shape, threshold["ldr"] - 1)
120
+
121
+ return np.where(
122
+ (~ma.getmaskarray(z))
123
+ & (z > threshold["z"])
124
+ & (v < threshold["v"])
125
+ & (ldr < threshold["ldr"]),
126
+ 1,
127
+ 0,
128
+ )
129
+
130
+ def _find_rain_from_disdrometer(self) -> ma.MaskedArray:
131
+ if self.data.disdrometer is None:
132
+ return ma.masked_all(self.time.shape, dtype=int)
133
+ threshold_mm_h = 0.25 # Standard threshold for drizzle -> rain
134
+ threshold_particles = 30 # This is arbitrary and should be better tested
135
+ threshold_rate = threshold_mm_h * MM_H_TO_M_S
136
+ rainfall_rate = self.data.disdrometer.data["rainfall_rate"].data
137
+ n_particles = self.data.disdrometer.data["n_particles"].data
138
+ return ma.array(
139
+ (rainfall_rate > threshold_rate) & (n_particles > threshold_particles),
140
+ dtype=int,
141
+ )
114
142
 
115
143
 
116
144
  def _find_clutter(
117
145
  v: np.ma.MaskedArray,
118
- is_rain: np.ndarray,
146
+ is_rain: npt.NDArray,
119
147
  n_gates: int = 10,
120
148
  v_lim: float = 0.05,
121
- ) -> np.ndarray:
149
+ ) -> npt.NDArray:
122
150
  """Estimates clutter from doppler velocity.
123
151
 
124
152
  Args:
153
+ v: 2D radar velocity.
154
+ is_rain: 2D boolean array denoting rain.
125
155
  n_gates: Number of range gates from the ground where clutter is expected
126
156
  to be found. Default is 10.
127
157
  v_lim: Velocity threshold. Smaller values are classified as clutter.
@@ -132,6 +162,7 @@ def _find_clutter(
132
162
 
133
163
  """
134
164
  is_clutter = np.zeros(v.shape, dtype=bool)
135
- tiny_velocity = (np.abs(v[:, :n_gates]) < v_lim).filled(False)
165
+ filled = False
166
+ tiny_velocity = (np.abs(v[:, :n_gates]) < v_lim).filled(filled)
136
167
  is_clutter[:, :n_gates] = tiny_velocity * utils.transpose(~is_rain)
137
168
  return is_clutter
@@ -0,0 +1,40 @@
1
+ """Mwr module, containing the :class:`Mwr` class."""
2
+
3
+ from os import PathLike
4
+
5
+ import numpy.typing as npt
6
+
7
+ from cloudnetpy.datasource import DataSource
8
+ from cloudnetpy.exceptions import DisdrometerDataError
9
+ from cloudnetpy.utils import interpolate_1d
10
+
11
+
12
+ class Disdrometer(DataSource):
13
+ """Disdrometer class, child of DataSource.
14
+
15
+ Args:
16
+ ----
17
+ full_path: Cloudnet Level 1b disdrometer file.
18
+
19
+ """
20
+
21
+ def __init__(self, full_path: str | PathLike) -> None:
22
+ super().__init__(full_path)
23
+ self._init_rainfall_rate()
24
+
25
+ def interpolate_to_grid(self, time_grid: npt.NDArray) -> None:
26
+ for key, array in self.data.items():
27
+ method = "nearest" if key == "synop_WaWa" else "linear"
28
+ self.data[key].data = interpolate_1d(
29
+ self.time, array.data, time_grid, max_time=1, method=method
30
+ )
31
+
32
+ def _init_rainfall_rate(self) -> None:
33
+ keys = ("rainfall_rate", "n_particles", "synop_WaWa")
34
+ for key in keys:
35
+ if key not in self.dataset.variables:
36
+ if key == "synop_WaWa":
37
+ continue
38
+ msg = f"variable {key} is missing"
39
+ raise DisdrometerDataError(msg)
40
+ self.append_data(self.dataset.variables[key][:], key)