cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,19 @@
1
- """This module contains RPG Cloud Radar related functions."""
1
+ import datetime
2
2
  import logging
3
3
  import math
4
- from typing import Sequence
4
+ from collections.abc import Sequence
5
+ from os import PathLike
6
+ from uuid import UUID
5
7
 
6
8
  import numpy as np
9
+ import numpy.typing as npt
7
10
  from numpy import ma
11
+ from rpgpy import RPGFileError
8
12
 
9
13
  from cloudnetpy import output, utils
10
- from cloudnetpy.categorize.atmos_utils import mmh2ms
11
14
  from cloudnetpy.cloudnetarray import CloudnetArray
12
- from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
15
+ from cloudnetpy.constants import G_TO_KG, HPA_TO_PA, KM_H_TO_M_S, MM_H_TO_M_S
16
+ from cloudnetpy.exceptions import ValidTimeStampError
13
17
  from cloudnetpy.instruments import instruments
14
18
  from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
15
19
  from cloudnetpy.instruments.instruments import Instrument
@@ -18,12 +22,12 @@ from cloudnetpy.metadata import MetaData
18
22
 
19
23
 
20
24
  def rpg2nc(
21
- path_to_l1_files: str,
22
- output_file: str,
25
+ path_to_l1_files: str | PathLike,
26
+ output_file: str | PathLike,
23
27
  site_meta: dict,
24
- uuid: str | None = None,
25
- date: str | None = None,
26
- ) -> tuple[str, list]:
28
+ uuid: str | UUID | None = None,
29
+ date: str | datetime.date | None = None,
30
+ ) -> tuple[UUID, list[str]]:
27
31
  """Converts RPG-FMCW-94 cloud radar data into Cloudnet Level 1b netCDF file.
28
32
 
29
33
  This function reads one day of RPG Level 1 cloud radar binary files,
@@ -56,12 +60,14 @@ def rpg2nc(
56
60
  >>> rpg2nc('/path/to/files/', 'test.nc', site_meta)
57
61
 
58
62
  """
63
+ if isinstance(date, str):
64
+ date = datetime.date.fromisoformat(date)
65
+ uuid = utils.get_uuid(uuid)
59
66
  l1_files = utils.get_sorted_filenames(path_to_l1_files, ".LV1")
60
67
  fmcw94_objects, valid_files = _get_fmcw94_objects(l1_files, date)
61
68
  one_day_of_data = create_one_day_data_record(fmcw94_objects)
62
- if not valid_files:
63
- return "", []
64
- print_info(one_day_of_data)
69
+ one_day_of_data["nyquist_velocity"] = _expand_nyquist(one_day_of_data)
70
+ _print_info(one_day_of_data)
65
71
  fmcw = Fmcw(one_day_of_data, site_meta)
66
72
  fmcw.convert_time_to_fraction_hour()
67
73
  fmcw.mask_invalid_ldr()
@@ -71,15 +77,20 @@ def rpg2nc(
71
77
  fmcw.linear_to_db(("Zh", "antenna_gain"))
72
78
  fmcw.convert_units()
73
79
  fmcw.add_site_geolocation()
74
- fmcw.add_zenith_angle()
80
+ valid_ind = fmcw.add_zenith_angle()
81
+ fmcw.screen_time_indices(valid_ind)
75
82
  fmcw.add_height()
83
+ if len(np.unique(fmcw.data["time"][:].astype("f4"))) != len(fmcw.data["time"][:]):
84
+ msg = "Convert time to f8 to keep values unique in netCDF"
85
+ logging.info(msg)
86
+ fmcw.data["time"].data_type = "f8"
76
87
  attributes = output.add_time_attribute(RPG_ATTRIBUTES, fmcw.date)
77
88
  output.update_attributes(fmcw.data, attributes)
78
- uuid = output.save_level1b(fmcw, output_file, uuid)
89
+ output.save_level1b(fmcw, output_file, uuid)
79
90
  return uuid, valid_files
80
91
 
81
92
 
82
- def print_info(data: dict) -> None:
93
+ def _print_info(data: dict) -> None:
83
94
  dual_pol = data["dual_polarization"]
84
95
  if dual_pol == 0:
85
96
  mode = "single polarisation"
@@ -87,65 +98,88 @@ def print_info(data: dict) -> None:
87
98
  mode = "LDR"
88
99
  else:
89
100
  mode = "STSR"
90
- logging.info(f"RPG cloud radar in {mode} mode")
101
+ logging.info("RPG cloud radar in %s mode", mode)
91
102
 
92
103
 
93
104
  RpgObjects = Sequence[Fmcw94Bin] | Sequence[HatproBinCombined]
94
105
 
95
106
 
96
107
  def create_one_day_data_record(rpg_objects: RpgObjects) -> dict:
97
- """Concatenates all RPG data from one day."""
108
+ """Concatenates all RPG FMCW / HATPRO data from one day."""
98
109
  rpg_raw_data, rpg_header = _stack_rpg_data(rpg_objects)
99
- if len(rpg_objects) > 1:
100
- rpg_header = _reduce_header(rpg_header)
110
+ if "range" in rpg_header:
111
+ rpg_header["range"] = rpg_objects[0].header["range"]
112
+ should_be_constant = [
113
+ "customer_name",
114
+ "model_number",
115
+ "dual_polarization",
116
+ "antenna_separation",
117
+ "antenna_diameter",
118
+ "antenna_gain",
119
+ "half_power_beam_width",
120
+ "radar_frequency",
121
+ ]
122
+ to_be_removed = ["customer_name"]
123
+ for key in should_be_constant:
124
+ if key not in rpg_header:
125
+ continue
126
+ unique_values = np.unique(rpg_header[key])
127
+ if len(unique_values) > 1:
128
+ msg = f"More than one value for {key} found: {unique_values}"
129
+ raise ValueError(msg)
130
+ if key in to_be_removed:
131
+ del rpg_header[key]
132
+ else:
133
+ rpg_header[key] = unique_values[0]
134
+
101
135
  rpg_raw_data = _mask_invalid_data(rpg_raw_data)
102
136
  return {**rpg_header, **rpg_raw_data}
103
137
 
104
138
 
105
- def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
106
- """Combines data from hourly RPG objects.
107
-
108
- Notes:
109
- Ignores variable names starting with an underscore.
110
-
111
- """
139
+ def _expand_nyquist(data: dict) -> npt.NDArray:
140
+ """Expands Nyquist velocity from time X chirp => time X range."""
141
+ nyquist_velocity = ma.array(data["nyquist_velocity"])
142
+ chirp_start_indices = ma.array(data["chirp_start_indices"])
143
+ n_time = chirp_start_indices.shape[0]
144
+ n_range = len(data["range"])
145
+ expanded_nyquist = np.empty((n_time, n_range))
146
+ for t in range(n_time):
147
+ starts = chirp_start_indices[t].compressed()
148
+ v_nyq = nyquist_velocity[t].compressed()
149
+ ends = np.r_[starts[1:], n_range]
150
+ seg_lengths = ends - starts
151
+ expanded_nyquist[t, :] = np.repeat(v_nyq, seg_lengths)
152
+ return expanded_nyquist
112
153
 
113
- def _stack(source, target, fun):
114
- for name, value in source.items():
115
- if not name.startswith("_"):
116
- target[name] = fun((target[name], value)) if name in target else value
117
154
 
155
+ def _stack_rpg_data(rpg_objects: RpgObjects) -> tuple[dict, dict]:
118
156
  data: dict = {}
119
157
  header: dict = {}
120
158
  for rpg in rpg_objects:
121
- _stack(rpg.data, data, ma.concatenate)
122
- _stack(rpg.header, header, ma.vstack)
159
+ for src, dst in ((rpg.data, data), (rpg.header, header)):
160
+ for name, value in src.items():
161
+ if name.startswith("_"):
162
+ continue
163
+ arr = dst.get(name)
164
+ fun = (
165
+ ma.concatenate
166
+ if any(isinstance(x, ma.MaskedArray) for x in (value, arr))
167
+ else np.concatenate
168
+ )
169
+ dst[name] = fun((arr, value)) if arr is not None else value
123
170
  return data, header
124
171
 
125
172
 
126
- def _reduce_header(header: dict) -> dict:
127
- """Removes duplicate header data. Otherwise, we would need n_files dimension."""
128
- reduced_header = {}
129
- for key, data in header.items():
130
- first_profile_value = data[0]
131
- is_identical_value = bool(
132
- np.isclose(data, first_profile_value, rtol=1e-2).all()
133
- )
134
- if is_identical_value is False:
135
- msg = f"Inconsistent header: {key}"
136
- if key in ("latitude", "longitude", "sample_duration"):
137
- logging.warning(f"{msg}: {data}")
138
- else:
139
- raise InconsistentDataError(msg)
140
- reduced_header[key] = first_profile_value
141
- return reduced_header
142
-
143
-
144
173
  def _mask_invalid_data(data_in: dict) -> dict:
145
174
  """Masks zeros and other fill values from data."""
146
175
  data = data_in.copy()
147
176
  fill_values = (-999, 1e-10)
177
+ extra_keys = ("air_temperature", "air_pressure")
148
178
  for name in data:
179
+ if np.issubdtype(data[name].dtype, np.integer) or (
180
+ data[name].ndim < 2 and name not in extra_keys
181
+ ):
182
+ continue
149
183
  data[name] = ma.masked_equal(data[name], 0)
150
184
  for value in fill_values:
151
185
  data[name][data[name] == value] = ma.masked
@@ -154,7 +188,9 @@ def _mask_invalid_data(data_in: dict) -> dict:
154
188
  return data
155
189
 
156
190
 
157
- def _get_fmcw94_objects(files: list, expected_date: str | None) -> tuple[list, list]:
191
+ def _get_fmcw94_objects(
192
+ files: list[str], expected_date: datetime.date | None
193
+ ) -> tuple[list[Fmcw94Bin], list[str]]:
158
194
  """Creates a list of Rpg() objects from the file names."""
159
195
  objects = []
160
196
  valid_files = []
@@ -163,46 +199,98 @@ def _get_fmcw94_objects(files: list, expected_date: str | None) -> tuple[list, l
163
199
  obj = Fmcw94Bin(file)
164
200
  if expected_date is not None:
165
201
  _validate_date(obj, expected_date)
166
- except (TypeError, ValueError, IndexError) as err:
202
+ except (RPGFileError, TypeError, ValueError, IndexError) as err:
167
203
  logging.warning(err)
168
204
  continue
169
205
  objects.append(obj)
170
206
  valid_files.append(file)
171
- if objects:
172
- objects, valid_files = _remove_files_with_bad_height(objects, valid_files)
173
- if not valid_files:
174
- raise ValidTimeStampError
207
+ if not objects:
208
+ msg = "No valid files found"
209
+ raise ValidTimeStampError(msg)
210
+ objects = _interpolate_to_common_height(objects)
211
+ objects = _pad_chirp_related_fields(objects)
212
+ objects = _expand_time_related_fields(objects)
175
213
  return objects, valid_files
176
214
 
177
215
 
178
- def _remove_files_with_bad_height(objects: list, files: list) -> tuple[list, list]:
179
- lengths = [obj.data["Zh"].shape[1] for obj in objects]
180
- most_common = np.bincount(lengths).argmax()
181
- files = [
182
- file
183
- for file, obj, length in zip(files, objects, lengths)
184
- if length == most_common
185
- ]
186
- objects = [obj for obj, length in zip(objects, lengths) if length == most_common]
187
- n_removed = len(lengths) - len(files)
188
- if n_removed > 0:
189
- logging.warning(
190
- f"Removed {n_removed} RPG-FMCW-94 files due to inconsistent height vector"
191
- )
192
- return objects, files
216
+ def _interpolate_to_common_height(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
217
+ range_arrays = [obj.header["range"] for obj in objects]
218
+ if all(np.array_equal(range_arrays[0], r) for r in range_arrays[1:]):
219
+ return objects
220
+ # Use range with the highest range gate for interpolation
221
+ target_range = max(range_arrays, key=lambda r: r[-1])
222
+ for obj in objects:
223
+ src_range = obj.header["range"]
224
+ if np.array_equal(src_range, target_range):
225
+ continue
226
+ for key, arr in obj.data.items():
227
+ if arr.ndim == 2 and arr.shape[1] == src_range.size:
228
+ obj.data[key] = utils.interpolate_2D_along_y(
229
+ src_range, arr, target_range
230
+ )
231
+ _interpolate_chirp_start_indices(obj, target_range)
232
+ obj.header["range"] = target_range
233
+ return objects
234
+
235
+
236
+ def _interpolate_chirp_start_indices(obj: Fmcw94Bin, target_range: np.ndarray) -> None:
237
+ range_orig = obj.header["range"]
238
+ vals = range_orig[obj.header["chirp_start_indices"]]
239
+ indices = np.abs(target_range[:, None] - vals).argmin(axis=0)
240
+ # Chirp start indices should always start from 0:
241
+ if indices[0] != 0:
242
+ indices[0] = 0
243
+ obj.header["chirp_start_indices"] = indices
244
+
245
+
246
+ def _pad_chirp_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
247
+ """Pads chirp-related header fields with masked values to have the same length."""
248
+ chirp_lens = [len(obj.header["chirp_start_indices"]) for obj in objects]
249
+ if all(chirp_lens[0] == length for length in chirp_lens[1:]):
250
+ return objects
251
+ max_chirp_len = max(chirp_lens)
252
+ for obj in objects:
253
+ n_chirps = len(obj.header["chirp_start_indices"])
254
+ if n_chirps == max_chirp_len:
255
+ continue
256
+ for key, arr in obj.header.items():
257
+ if not isinstance(arr, str) and arr.ndim == 1 and arr.size == n_chirps:
258
+ pad_len = max_chirp_len - n_chirps
259
+ masked_arr = ma.array(arr, dtype=arr.dtype)
260
+ pad = ma.masked_all(pad_len, dtype=arr.dtype)
261
+ obj.header[key] = ma.concatenate([masked_arr, pad])
262
+ return objects
263
+
264
+
265
+ def _expand_time_related_fields(objects: list[Fmcw94Bin]) -> list[Fmcw94Bin]:
266
+ for obj in objects:
267
+ n_time = obj.data["time"].size
268
+ for key in obj.header:
269
+ if key in ("range", "time") or key.startswith("_"):
270
+ continue
271
+ arr = obj.header[key]
272
+ # Handle outliers in latitude and longitude (e.g. Galati 2024-02-11):
273
+ if key in ("latitude", "longitude"):
274
+ arr = ma.median(arr)
275
+ if utils.isscalar(arr):
276
+ obj.header[key] = np.repeat(arr, n_time)
277
+ else:
278
+ obj.header[key] = np.tile(arr, (n_time, 1))
279
+ return objects
193
280
 
194
281
 
195
- def _validate_date(obj, expected_date: str) -> None:
282
+ def _validate_date(obj: Fmcw94Bin, expected_date: datetime.date) -> None:
196
283
  for t in obj.data["time"][:]:
197
- date_str = "-".join(utils.seconds2date(t)[:3])
198
- if date_str != expected_date:
199
- raise ValueError("Ignoring a file (time stamps not what expected)")
284
+ date = utils.seconds2date(t).date()
285
+ if date != expected_date:
286
+ msg = "Ignoring a file (time stamps not what expected)"
287
+ raise ValueError(msg)
200
288
 
201
289
 
202
290
  class Rpg(CloudnetInstrument):
203
291
  """Base class for RPG FMCW-94 cloud radar and HATPRO mwr."""
204
292
 
205
- def __init__(self, raw_data: dict, site_meta: dict):
293
+ def __init__(self, raw_data: dict, site_meta: dict) -> None:
206
294
  super().__init__()
207
295
  self.raw_data = raw_data
208
296
  self.site_meta = site_meta
@@ -212,17 +300,21 @@ class Rpg(CloudnetInstrument):
212
300
 
213
301
  def convert_time_to_fraction_hour(self, data_type: str | None = None) -> None:
214
302
  """Converts time to fraction hour."""
215
- key = "time"
216
- fraction_hour = utils.seconds2hours(self.raw_data[key])
217
- self.data[key] = CloudnetArray(
218
- np.array(fraction_hour), key, data_type=data_type
303
+ ms2s = 1e-3
304
+ total_time_sec = self.raw_data["time"] + self.raw_data.get("time_ms", 0) * ms2s
305
+ fraction_hour = utils.seconds2hours(total_time_sec)
306
+
307
+ self.data["time"] = CloudnetArray(
308
+ np.array(fraction_hour),
309
+ "time",
310
+ data_type=data_type,
219
311
  )
220
312
 
221
- def _get_date(self) -> list:
313
+ def _get_date(self) -> datetime.date:
222
314
  time_first = self.raw_data["time"][0]
223
315
  time_last = self.raw_data["time"][-1]
224
- date_first = utils.seconds2date(time_first)[:3]
225
- date_last = utils.seconds2date(time_last)[:3]
316
+ date_first = utils.seconds2date(time_first).date()
317
+ date_last = utils.seconds2date(time_last).date()
226
318
  if date_first != date_last:
227
319
  logging.warning("Measurements from different days")
228
320
  return date_first
@@ -237,7 +329,7 @@ class Rpg(CloudnetInstrument):
237
329
  class Fmcw(Rpg):
238
330
  """Class for RPG cloud radars."""
239
331
 
240
- def __init__(self, raw_data: dict, site_properties: dict):
332
+ def __init__(self, raw_data: dict, site_properties: dict) -> None:
241
333
  super().__init__(raw_data, site_properties)
242
334
  self.instrument = self._get_instrument(raw_data)
243
335
 
@@ -246,7 +338,8 @@ class Fmcw(Rpg):
246
338
  threshold = -35
247
339
  if "ldr" in self.data:
248
340
  self.data["ldr"].data = ma.masked_less_equal(
249
- self.data["ldr"].data, threshold
341
+ self.data["ldr"].data,
342
+ threshold,
250
343
  )
251
344
 
252
345
  def mask_invalid_width(self) -> None:
@@ -262,211 +355,250 @@ class Fmcw(Rpg):
262
355
  self.data[key].data[ind] = ma.masked
263
356
 
264
357
  def add_zenith_angle(self) -> list:
265
- """Adds zenith angle and returns time indices where zenith angle is stable."""
358
+ """Adds zenith angle and returns time indices with valid zenith angle."""
266
359
  elevation = self.data["elevation"].data
267
360
  zenith = 90 - elevation
268
- if elevation.mask.all():
269
- zenith[:] = 0
270
- logging.warning("Can not determine zenith angle, assuming 0 degrees")
271
- is_stable_zenith = np.isclose(zenith, ma.median(zenith), atol=0.1)
272
- n_removed = len(is_stable_zenith) - np.count_nonzero(is_stable_zenith)
361
+ is_valid_zenith = _filter_zenith_angle(zenith)
362
+ n_removed = len(is_valid_zenith) - np.count_nonzero(is_valid_zenith)
363
+ if n_removed == len(zenith):
364
+ msg = "No profiles with valid zenith angle"
365
+ raise ValidTimeStampError(msg)
273
366
  if n_removed > 0:
274
367
  logging.warning(
275
- f"Filtering {n_removed} profiles due to varying zenith angle"
368
+ "Filtering %s profiles due to invalid zenith angle",
369
+ n_removed,
276
370
  )
277
371
  self.data["zenith_angle"] = CloudnetArray(zenith, "zenith_angle")
278
372
  del self.data["elevation"]
279
- return list(is_stable_zenith)
373
+ return list(is_valid_zenith)
280
374
 
281
- def convert_units(self):
375
+ def convert_units(self) -> None:
282
376
  """Converts units."""
283
- self.data["rainfall_rate"].data = mmh2ms(self.data["rainfall_rate"].data)
284
- self.data["lwp"].data *= 1e-3 # g -> kg
377
+ self.data["rainfall_rate"].data = self.data["rainfall_rate"].data * MM_H_TO_M_S
378
+ self.data["lwp"].data *= G_TO_KG
379
+ self.data["relative_humidity"].data /= 100
380
+ self.data["air_pressure"].data *= HPA_TO_PA
381
+ self.data["wind_speed"].data *= KM_H_TO_M_S
285
382
 
286
383
  @staticmethod
287
- def _get_instrument(data: dict):
384
+ def _get_instrument(data: dict) -> Instrument:
288
385
  frequency = data["radar_frequency"]
289
386
  if math.isclose(frequency, 35, abs_tol=0.1):
290
387
  return instruments.FMCW35
291
388
  if math.isclose(frequency, 94, abs_tol=0.1):
292
389
  return instruments.FMCW94
293
- raise RuntimeError(f"Unknown RPG cloud radar frequency: {frequency}")
390
+ msg = f"Unknown RPG cloud radar frequency: {frequency}"
391
+ raise RuntimeError(msg)
294
392
 
295
393
 
296
394
  class Hatpro(Rpg):
297
395
  """Class for RPG HATPRO mwr."""
298
396
 
299
- def __init__(self, raw_data: dict, site_properties: dict):
397
+ def __init__(
398
+ self, raw_data: dict, site_properties: dict, instrument: Instrument
399
+ ) -> None:
300
400
  super().__init__(raw_data, site_properties)
301
- self.instrument = instruments.HATPRO
401
+ self.instrument = instrument
402
+
403
+
404
+ def _filter_zenith_angle(zenith: ma.MaskedArray) -> npt.NDArray:
405
+ """Returns indices of profiles with stable zenith angle close to 0 deg."""
406
+ zenith = ma.array(zenith)
407
+ if zenith.mask.all():
408
+ return np.zeros(zenith.shape, dtype=bool)
409
+ limits = [-5, 15]
410
+ ind_close_to_zenith = np.where(
411
+ np.logical_and(zenith > limits[0], zenith < limits[1]),
412
+ )
413
+ if not ind_close_to_zenith[0].size:
414
+ return np.zeros_like(zenith, dtype=bool)
415
+ valid_range_median = ma.median(zenith[ind_close_to_zenith])
416
+ is_stable_zenith = np.isclose(zenith, valid_range_median, atol=0.1)
417
+ is_stable_zenith[zenith.mask] = False
418
+ return np.array(is_stable_zenith)
302
419
 
303
420
 
304
421
  DEFINITIONS = {
305
- "model_number": "\n"
306
- "0: Single polarisation radar.\n"
307
- "1: Dual polarisation radar.",
308
- "dual_polarization": (
309
- "\n"
310
- "Value 0: Single polarisation radar.\n"
311
- "Value 1: Dual polarisation radar in linear depolarisation ratio (LDR) mode.\n"
312
- "Value 2: Dual polarisation radar in simultaneous transmission simultaneous\n"
313
- "reception (STSR) mode."
314
- ),
315
- "FFT_window": (
316
- "\n"
317
- "Value 0: Square\n"
318
- "Value 1: Parzen\n"
319
- "Value 2: Blackman\n"
320
- "Value 3: Welch\n"
321
- "Value 4: Slepian2\n"
322
- "Value 5: Slepian3"
323
- ),
324
- "quality_flag": (
325
- "\n"
326
- "Bit 0: ADC saturation.\n"
327
- "Bit 1: Spectral width too high.\n"
328
- "Bit 2: No transmission power levelling."
422
+ "model_number": utils.status_field_definition(
423
+ {
424
+ 0: "Single polarisation radar.",
425
+ 1: "Dual polarisation radar.",
426
+ }
427
+ ),
428
+ "dual_polarization": utils.status_field_definition(
429
+ {
430
+ 0: """Single polarisation radar.""",
431
+ 1: """Dual polarisation radar in linear depolarisation ratio (LDR)
432
+ mode.""",
433
+ 2: """Dual polarisation radar in simultaneous transmission
434
+ simultaneous reception (STSR) mode.""",
435
+ }
436
+ ),
437
+ "FFT_window": utils.status_field_definition(
438
+ {
439
+ 0: "Square",
440
+ 1: "Parzen",
441
+ 2: "Blackman",
442
+ 3: "Welch",
443
+ 4: "Slepian2",
444
+ 5: "Slepian3",
445
+ }
446
+ ),
447
+ "quality_flag": utils.bit_field_definition(
448
+ {
449
+ 0: "ADC saturation.",
450
+ 1: "Spectral width too high.",
451
+ 2: "No transmission power levelling.",
452
+ }
329
453
  ),
330
454
  }
331
455
 
332
456
  RPG_ATTRIBUTES = {
333
457
  # LDR-mode radars:
334
- "ldr": MetaData(long_name="Linear depolarisation ratio", units="dB"),
335
- "rho_cx": MetaData(long_name="Co-cross-channel correlation coefficient", units="1"),
336
- "phi_cx": MetaData(long_name="Co-cross-channel differential phase", units="rad"),
458
+ "ldr": MetaData(
459
+ long_name="Linear depolarisation ratio",
460
+ units="dB",
461
+ dimensions=("time", "range"),
462
+ ),
463
+ "rho_cx": MetaData(
464
+ long_name="Co-cross-channel correlation coefficient",
465
+ units="1",
466
+ dimensions=("time", "range"),
467
+ ),
468
+ "phi_cx": MetaData(
469
+ long_name="Co-cross-channel differential phase",
470
+ units="rad",
471
+ dimensions=("time", "range"),
472
+ ),
337
473
  # STSR-mode radars
338
- "zdr": MetaData(long_name="Differential reflectivity", units="dB"),
339
- "rho_hv": MetaData(long_name="Correlation coefficient", units="1"),
340
- "phi_dp": MetaData(long_name="Differential phase", units="rad"),
341
- "srho_hv": MetaData(long_name="Slanted correlation coefficient", units="1"),
342
- "kdp": MetaData(long_name="Specific differential phase shift", units="rad km-1"),
474
+ "zdr": MetaData(
475
+ long_name="Differential reflectivity", units="dB", dimensions=("time", "range")
476
+ ),
477
+ "rho_hv": MetaData(
478
+ long_name="Correlation coefficient", units="1", dimensions=("time", "range")
479
+ ),
480
+ "phi_dp": MetaData(
481
+ long_name="Differential phase", units="rad", dimensions=("time", "range")
482
+ ),
483
+ "srho_hv": MetaData(
484
+ long_name="Slanted correlation coefficient",
485
+ units="1",
486
+ dimensions=("time", "range"),
487
+ ),
488
+ "kdp": MetaData(
489
+ long_name="Specific differential phase shift",
490
+ units="rad km-1",
491
+ dimensions=("time", "range"),
492
+ ),
343
493
  "differential_attenuation": MetaData(
344
- long_name="Differential attenuation", units="dB km-1"
494
+ long_name="Differential attenuation",
495
+ units="dB km-1",
496
+ dimensions=("time", "range"),
345
497
  ),
346
498
  # All radars
347
499
  "file_code": MetaData(
348
500
  long_name="File code",
349
501
  units="1",
350
502
  comment="Indicates the RPG software version.",
503
+ dimensions=("time",),
504
+ ),
505
+ "program_number": MetaData(
506
+ long_name="Program number", units="1", dimensions=("time",)
351
507
  ),
352
- "program_number": MetaData(long_name="Program number", units="1"),
353
508
  "model_number": MetaData(
354
509
  long_name="Model number",
355
510
  units="1",
356
511
  definition=DEFINITIONS["model_number"],
512
+ dimensions=None,
357
513
  ),
358
514
  "antenna_separation": MetaData(
359
- long_name="Antenna separation",
360
- units="m",
515
+ long_name="Antenna separation", units="m", dimensions=None
361
516
  ),
362
517
  "antenna_diameter": MetaData(
363
- long_name="Antenna diameter",
364
- units="m",
365
- ),
366
- "antenna_gain": MetaData(
367
- long_name="Antenna gain",
368
- units="dB",
518
+ long_name="Antenna diameter", units="m", dimensions=None
369
519
  ),
520
+ "antenna_gain": MetaData(long_name="Antenna gain", units="dB", dimensions=None),
370
521
  "half_power_beam_width": MetaData(
371
- long_name="Half power beam width",
372
- units="degree",
522
+ long_name="Half power beam width", units="degree", dimensions=None
373
523
  ),
374
524
  "dual_polarization": MetaData(
375
525
  long_name="Dual polarisation type",
376
526
  units="1",
377
527
  definition=DEFINITIONS["dual_polarization"],
528
+ dimensions=None,
529
+ ),
530
+ "sample_duration": MetaData(
531
+ long_name="Sample duration", units="s", dimensions=("time",)
378
532
  ),
379
- "sample_duration": MetaData(long_name="Sample duration", units="s"),
380
533
  "calibration_interval": MetaData(
381
- long_name="Calibration interval in samples",
382
- units="1",
534
+ long_name="Calibration interval in samples", units="1", dimensions=("time",)
383
535
  ),
384
536
  "number_of_spectral_samples": MetaData(
385
537
  long_name="Number of spectral samples in each chirp sequence",
386
538
  units="1",
387
- ),
388
- "chirp_start_indices": MetaData(
389
- long_name="Chirp sequences start indices",
390
- units="1",
539
+ dimensions=("time", "chirp_sequence"),
391
540
  ),
392
541
  "number_of_averaged_chirps": MetaData(
393
542
  long_name="Number of averaged chirps in sequence",
394
543
  units="1",
544
+ dimensions=("time", "chirp_sequence"),
545
+ ),
546
+ "chirp_start_indices": MetaData(
547
+ long_name="Chirp sequences start indices",
548
+ units="1",
549
+ dimensions=("time", "chirp_sequence"),
395
550
  ),
396
551
  "integration_time": MetaData(
397
552
  long_name="Integration time",
398
553
  units="s",
399
554
  comment="Effective integration time of chirp sequence",
555
+ dimensions=("time", "chirp_sequence"),
400
556
  ),
401
557
  "range_resolution": MetaData(
402
558
  long_name="Vertical resolution of range",
403
559
  units="m",
560
+ dimensions=("time", "chirp_sequence"),
404
561
  ),
405
562
  "FFT_window": MetaData(
406
563
  long_name="FFT window type",
407
564
  units="1",
408
565
  definition=DEFINITIONS["FFT_window"],
566
+ dimensions=("time",),
409
567
  ),
410
568
  "input_voltage_range": MetaData(
411
- long_name="ADC input voltage range (+/-)",
412
- units="mV",
569
+ long_name="ADC input voltage range (+/-)", units="mV", dimensions=("time",)
413
570
  ),
414
571
  "noise_threshold": MetaData(
415
572
  long_name="Noise filter threshold factor",
416
573
  units="1",
417
574
  comment="Multiple of the standard deviation of Doppler spectra.",
575
+ dimensions=("time",),
418
576
  ),
419
- "time_ms": MetaData(
420
- long_name="Time ms",
421
- units="ms",
422
- ),
577
+ "time_ms": MetaData(long_name="Time ms", units="ms", dimensions=("time",)),
423
578
  "quality_flag": MetaData(
424
579
  long_name="Quality flag",
425
580
  definition=DEFINITIONS["quality_flag"],
426
581
  units="1",
582
+ dimensions=("time",),
427
583
  ),
428
- "voltage": MetaData(
429
- long_name="Voltage",
430
- units="V",
431
- ),
584
+ "voltage": MetaData(long_name="Voltage", units="V", dimensions=("time",)),
432
585
  "brightness_temperature": MetaData(
433
- long_name="Brightness temperature",
434
- units="K",
435
- ),
436
- "if_power": MetaData(
437
- long_name="IF power at ACD",
438
- units="uW",
586
+ long_name="Brightness temperature", units="K", dimensions=("time",)
439
587
  ),
588
+ "if_power": MetaData(long_name="IF power at ACD", units="uW", dimensions=("time",)),
440
589
  "status_flag": MetaData(
441
- long_name="Status flag for heater and blower",
442
- units="1",
590
+ long_name="Status flag for heater and blower", units="1", dimensions=("time",)
443
591
  ),
444
592
  "transmitted_power": MetaData(
445
- long_name="Transmitted power",
446
- units="W",
593
+ long_name="Transmitted power", units="W", dimensions=("time",)
447
594
  ),
448
595
  "transmitter_temperature": MetaData(
449
- long_name="Transmitter temperature",
450
- units="K",
596
+ long_name="Transmitter temperature", units="K", dimensions=("time",)
451
597
  ),
452
598
  "receiver_temperature": MetaData(
453
- long_name="Receiver temperature",
454
- units="K",
599
+ long_name="Receiver temperature", units="K", dimensions=("time",)
455
600
  ),
456
601
  "pc_temperature": MetaData(
457
- long_name="PC temperature",
458
- units="K",
459
- ),
460
- "skewness": MetaData(
461
- long_name="Skewness of spectra",
462
- units="1",
463
- ),
464
- "kurtosis": MetaData(
465
- long_name="Kurtosis of spectra",
466
- units="1",
467
- ),
468
- "correlation_coefficient": MetaData(
469
- long_name="Correlation coefficient",
470
- units="1",
602
+ long_name="PC temperature", units="K", dimensions=("time",)
471
603
  ),
472
604
  }