cloudnetpy 1.77.2__tar.gz → 1.78.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. {cloudnetpy-1.77.2/cloudnetpy.egg-info → cloudnetpy-1.78.1}/PKG-INFO +1 -1
  2. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/insects.py +1 -1
  3. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/radar.py +89 -66
  4. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/cloudnetarray.py +19 -82
  5. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/concat_lib.py +66 -57
  6. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/datasource.py +0 -47
  7. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/bowtie.py +2 -2
  8. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/copernicus.py +0 -1
  9. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/galileo.py +0 -1
  10. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/mira.py +0 -10
  11. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/plotting/plot_tools.py +1 -1
  12. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +1 -1
  13. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/output.py +5 -1
  14. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/utils.py +32 -66
  15. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/version.py +2 -2
  16. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1/cloudnetpy.egg-info}/PKG-INFO +1 -1
  17. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/LICENSE +0 -0
  18. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/MANIFEST.in +0 -0
  19. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/README.md +0 -0
  20. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/__init__.py +0 -0
  21. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/__init__.py +0 -0
  22. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/atmos_utils.py +0 -0
  23. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuation.py +0 -0
  24. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuations/__init__.py +0 -0
  25. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuations/gas_attenuation.py +0 -0
  26. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuations/liquid_attenuation.py +0 -0
  27. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuations/melting_attenuation.py +0 -0
  28. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/attenuations/rain_attenuation.py +0 -0
  29. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/categorize.py +0 -0
  30. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/classify.py +0 -0
  31. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/containers.py +0 -0
  32. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/disdrometer.py +0 -0
  33. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/droplet.py +0 -0
  34. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/falling.py +0 -0
  35. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/freezing.py +0 -0
  36. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/itu.py +0 -0
  37. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/lidar.py +0 -0
  38. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/melting.py +0 -0
  39. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/model.py +0 -0
  40. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/categorize/mwr.py +0 -0
  41. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/cli.py +0 -0
  42. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/constants.py +0 -0
  43. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/exceptions.py +0 -0
  44. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/__init__.py +0 -0
  45. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/basta.py +0 -0
  46. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/ceilo.py +0 -0
  47. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/ceilometer.py +0 -0
  48. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/cl61d.py +0 -0
  49. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/cloudnet_instrument.py +0 -0
  50. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/disdrometer/__init__.py +0 -0
  51. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/disdrometer/common.py +0 -0
  52. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/disdrometer/parsivel.py +0 -0
  53. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/disdrometer/thies.py +0 -0
  54. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/fd12p.py +0 -0
  55. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/hatpro.py +0 -0
  56. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/instruments.py +0 -0
  57. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/lufft.py +0 -0
  58. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/mrr.py +0 -0
  59. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/nc_lidar.py +0 -0
  60. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/nc_radar.py +0 -0
  61. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/pollyxt.py +0 -0
  62. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/radiometrics.py +0 -0
  63. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/rain_e_h3.py +0 -0
  64. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/rpg.py +0 -0
  65. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/rpg_reader.py +0 -0
  66. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/toa5.py +0 -0
  67. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/vaisala.py +0 -0
  68. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/instruments/weather_station.py +0 -0
  69. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/metadata.py +0 -0
  70. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/__init__.py +0 -0
  71. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/file_handler.py +0 -0
  72. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/metadata.py +0 -0
  73. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/model_metadata.py +0 -0
  74. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/plotting/__init__.py +0 -0
  75. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/plotting/plot_meta.py +0 -0
  76. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/plotting/plotting.py +0 -0
  77. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/__init__.py +0 -0
  78. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/advance_methods.py +0 -0
  79. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/grid_methods.py +0 -0
  80. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/model_products.py +0 -0
  81. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/observation_products.py +0 -0
  82. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/product_resampling.py +0 -0
  83. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/products/tools.py +0 -0
  84. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/statistics/__init__.py +0 -0
  85. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/statistics/statistical_methods.py +0 -0
  86. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/__init__.py +0 -0
  87. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/__init__.py +0 -0
  88. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/conftest.py +0 -0
  89. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_cf/__init__.py +0 -0
  90. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +0 -0
  91. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +0 -0
  92. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/__init__.py +0 -0
  93. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +0 -0
  94. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +0 -0
  95. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/__init__.py +0 -0
  96. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +0 -0
  97. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +0 -0
  98. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/__init__.py +0 -0
  99. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/conftest.py +0 -0
  100. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +0 -0
  101. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_model_products.py +0 -0
  102. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +0 -0
  103. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +0 -0
  104. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_plotting.py +0 -0
  105. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +0 -0
  106. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/tests/unit/test_tools.py +0 -0
  107. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/model_evaluation/utils.py +0 -0
  108. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/plotting/__init__.py +0 -0
  109. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/plotting/plot_meta.py +0 -0
  110. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/plotting/plotting.py +0 -0
  111. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/__init__.py +0 -0
  112. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/classification.py +0 -0
  113. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/der.py +0 -0
  114. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/drizzle.py +0 -0
  115. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/drizzle_error.py +0 -0
  116. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/drizzle_tools.py +0 -0
  117. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/epsilon.py +0 -0
  118. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/ier.py +0 -0
  119. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/iwc.py +0 -0
  120. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/lwc.py +0 -0
  121. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/mie_lu_tables.nc +0 -0
  122. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/mwr_tools.py +0 -0
  123. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/products/product_tools.py +0 -0
  124. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy/py.typed +0 -0
  125. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy.egg-info/SOURCES.txt +0 -0
  126. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy.egg-info/dependency_links.txt +0 -0
  127. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy.egg-info/entry_points.txt +0 -0
  128. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy.egg-info/requires.txt +0 -0
  129. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/cloudnetpy.egg-info/top_level.txt +0 -0
  130. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/docs/source/conf.py +0 -0
  131. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/pyproject.toml +0 -0
  132. {cloudnetpy-1.77.2 → cloudnetpy-1.78.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloudnetpy
3
- Version: 1.77.2
3
+ Version: 1.78.1
4
4
  Summary: Python package for Cloudnet processing
5
5
  Author: Simo Tukiainen
6
6
  License: MIT License
@@ -83,7 +83,7 @@ def _get_smoothed_v(
83
83
  obs: ClassData,
84
84
  sigma: tuple[float, float] = (5, 5),
85
85
  ) -> ma.MaskedArray:
86
- smoothed_v = gaussian_filter(obs.v, sigma)
86
+ smoothed_v = gaussian_filter(obs.v.filled(0), sigma)
87
87
  return ma.masked_where(obs.v.mask, smoothed_v)
88
88
 
89
89
 
@@ -24,8 +24,6 @@ class Radar(DataSource):
24
24
  folding_velocity (float): Radar's folding velocity (m/s).
25
25
  location (str): Location of the radar, copied from the global attribute
26
26
  `location` of the input file.
27
- sequence_indices (list): Indices denoting the different altitude
28
- regimes of the radar.
29
27
  source_type (str): Type of the radar, copied from the global attribute
30
28
  `source` of the *radar_file*. Can be free form string but must
31
29
  include either 'rpg' or 'mira' denoting one of the two supported
@@ -39,21 +37,17 @@ class Radar(DataSource):
39
37
  def __init__(self, full_path: str):
40
38
  super().__init__(full_path, radar=True)
41
39
  self.radar_frequency = float(self.getvar("radar_frequency"))
42
- self.folding_velocity = self._get_folding_velocity()
43
- self.sequence_indices = self._get_sequence_indices()
44
40
  self.location = getattr(self.dataset, "location", "")
45
41
  self.source_type = getattr(self.dataset, "source", "")
46
42
  self.height: np.ndarray
47
43
  self.altitude: float
48
44
  self._init_data()
49
- self._init_sigma_v()
50
- self._get_folding_velocity_full()
51
45
 
52
46
  def rebin_to_grid(self, time_new: np.ndarray) -> list:
53
- """Rebins radar data in time using mean.
47
+ """Rebins radar data in time.
54
48
 
55
49
  Args:
56
- time_new: Target time array as fraction hour. Updates *time* attribute.
50
+ time_new: Target time array as fraction hour.
57
51
 
58
52
  """
59
53
  bad_time_indices = []
@@ -64,21 +58,25 @@ class Radar(DataSource):
64
58
  bad_time_indices = array.rebin_data(self.time, time_new)
65
59
  array.lin2db()
66
60
  case "v":
67
- array.rebin_velocity(
68
- self.time,
61
+ array.data = self._rebin_velocity(
62
+ array.data,
69
63
  time_new,
70
- self.folding_velocity,
71
- self.sequence_indices,
72
64
  )
73
65
  case "v_sigma":
74
- array.calc_linear_std(self.time, time_new)
66
+ array.data, _ = utils.rebin_2d(
67
+ self.time,
68
+ array.data,
69
+ time_new,
70
+ "std",
71
+ mask_zeros=True,
72
+ )
75
73
  case "width":
76
74
  array.rebin_data(self.time, time_new)
77
75
  case "rainfall_rate":
78
- array.rebin_data(self.time, time_new, mask_zeros=False)
76
+ array.rebin_data(self.time, time_new)
79
77
  case _:
80
78
  continue
81
- return bad_time_indices
79
+ return list(bad_time_indices)
82
80
 
83
81
  def remove_incomplete_pixels(self) -> None:
84
82
  """Mask radar pixels where one or more required quantities are missing.
@@ -337,63 +335,88 @@ class Radar(DataSource):
337
335
 
338
336
  def _init_data(self) -> None:
339
337
  self.append_data(self.getvar("Zh"), "Z", units="dBZ")
340
- for key in ("v", "ldr", "width", "sldr", "rainfall_rate"):
341
- try:
342
- self._variables_to_cloudnet_arrays((key,))
343
- except KeyError:
344
- continue
345
-
346
- def _init_sigma_v(self) -> None:
347
- """Initializes std of the velocity field. The std will be calculated
348
- later when re-binning the data.
349
- """
350
338
  self.append_data(self.getvar("v"), "v_sigma")
339
+ for key in ("v", "ldr", "width", "sldr", "rainfall_rate", "nyquist_velocity"):
340
+ if key in self.dataset.variables:
341
+ data = self.dataset.variables[key]
342
+ self.append_data(data, key)
351
343
 
352
- def _get_sequence_indices(self) -> list:
353
- """Mira has only one sequence and one folding velocity. RPG has
354
- several sequences with different folding velocities.
355
- """
356
- if self.height is None:
357
- msg = "Height not found in the input file"
358
- raise RuntimeError(msg)
359
- all_indices = np.arange(len(self.height))
360
- if not utils.isscalar(self.folding_velocity):
361
- starting_indices = self.getvar("chirp_start_indices")
362
- return np.split(all_indices, starting_indices[1:])
363
- return [all_indices]
364
-
365
- def _get_folding_velocity(self) -> np.ndarray | float:
344
+ def _rebin_velocity(
345
+ self,
346
+ data: np.ndarray,
347
+ time_new: np.ndarray,
348
+ ) -> np.ndarray:
349
+ """Rebins Doppler velocity in polar coordinates."""
350
+ folding_velocity = self._get_expanded_folding_velocity()
351
+ # with the new shape (maximum value in every bin)
352
+ max_folding_binned, _ = utils.rebin_2d(
353
+ self.time,
354
+ folding_velocity,
355
+ time_new,
356
+ "max",
357
+ )
358
+ # store this in the file
359
+ self.append_data(max_folding_binned, "nyquist_velocity")
360
+ # with original shape (repeat maximum value for each point in every bin)
361
+ max_folding_full, _ = utils.rebin_2d(
362
+ self.time,
363
+ folding_velocity,
364
+ time_new,
365
+ "max",
366
+ keepdim=True,
367
+ )
368
+ data_scaled = data * (np.pi / max_folding_full)
369
+ vel_x = ma.cos(data_scaled)
370
+ vel_y = ma.sin(data_scaled)
371
+ vel_x_mean, _ = utils.rebin_2d(self.time, vel_x, time_new)
372
+ vel_y_mean, _ = utils.rebin_2d(self.time, vel_y, time_new)
373
+ vel_scaled = ma.arctan2(vel_y_mean, vel_x_mean)
374
+ return vel_scaled / (np.pi / max_folding_binned)
375
+
376
+ def _get_expanded_folding_velocity(self) -> np.ndarray:
366
377
  if "nyquist_velocity" in self.dataset.variables:
367
- return self.getvar("nyquist_velocity")
368
- if "prf" in self.dataset.variables:
378
+ fvel = self.getvar("nyquist_velocity")
379
+ elif "prf" in self.dataset.variables:
369
380
  prf = self.getvar("prf")
370
- return _prf_to_folding_velocity(prf, self.radar_frequency)
371
- msg = "Unable to determine folding velocity"
372
- raise RuntimeError(msg)
373
-
374
- def _get_folding_velocity_full(self) -> None:
375
- folding_velocity: list | np.ndarray = []
376
- if utils.isscalar(self.folding_velocity):
377
- folding_velocity = np.repeat(
378
- self.folding_velocity,
379
- len(self.sequence_indices[0]),
380
- )
381
+ fvel = _prf_to_folding_velocity(prf, self.radar_frequency)
381
382
  else:
382
- folding_velocity = list(folding_velocity)
383
- self.folding_velocity = np.array(self.folding_velocity)
384
- for indices, velocity in zip(
385
- self.sequence_indices,
386
- self.folding_velocity,
387
- strict=True,
388
- ):
389
- folding_velocity.append(np.repeat(velocity, len(indices)))
390
- folding_velocity = np.hstack(folding_velocity)
391
- self.append_data(folding_velocity, "nyquist_velocity")
392
-
393
-
394
- def _prf_to_folding_velocity(prf: np.ndarray, radar_frequency: float) -> float:
383
+ msg = "Unable to determine folding velocity"
384
+ raise RuntimeError(msg)
385
+
386
+ n_time = self.getvar("time").size
387
+ n_height = self.height.size
388
+
389
+ if fvel.shape == (n_time, n_height):
390
+ # Folding velocity is already expanded in radar file
391
+ # Not yet in current files
392
+ return fvel
393
+ if utils.isscalar(fvel):
394
+ # e.g. MIRA
395
+ return np.broadcast_to(fvel, (n_time, n_height))
396
+
397
+ # RPG radars have chirp segments
398
+ starts = self.getvar("chirp_start_indices")
399
+ n_seg = starts.size if starts.ndim == 1 else starts.shape[1]
400
+
401
+ starts = np.broadcast_to(starts, (n_time, n_seg))
402
+ fvel = np.broadcast_to(fvel, (n_time, n_seg))
403
+
404
+ # Indices should start from zero (first range gate)
405
+ # In pre-processed RV Meteor files the first index is 1, so normalize:
406
+ # Normalize starts so indices begin from zero
407
+ first_values = starts[:, [0]]
408
+ if not np.all(np.isin(first_values, [0, 1])):
409
+ msg = "First value of chirp_start_indices must be 0 or 1"
410
+ raise ValueError(msg)
411
+ starts = starts - first_values
412
+
413
+ chirp_size = np.diff(starts, append=n_height)
414
+ return np.repeat(fvel.ravel(), chirp_size.ravel()).reshape((n_time, n_height))
415
+
416
+
417
+ def _prf_to_folding_velocity(prf: np.ndarray, radar_frequency: float) -> np.ndarray:
395
418
  ghz_to_hz = 1e9
396
419
  if len(prf) != 1:
397
420
  msg = "Unable to determine folding velocity"
398
421
  raise RuntimeError(msg)
399
- return float(prf[0] * constants.c / (4 * radar_frequency * ghz_to_hz))
422
+ return prf[0] * constants.c / (4 * radar_frequency * ghz_to_hz)
@@ -1,6 +1,5 @@
1
1
  """CloudnetArray class."""
2
2
 
3
- import math
4
3
  from collections.abc import Sequence
5
4
 
6
5
  import netCDF4
@@ -58,29 +57,22 @@ class CloudnetArray:
58
57
  """Masks data from given indices."""
59
58
  self.data[ind] = ma.masked
60
59
 
61
- def rebin_data(
62
- self, time: np.ndarray, time_new: np.ndarray, *, mask_zeros: bool = True
63
- ) -> list:
60
+ def rebin_data(self, time: np.ndarray, time_new: np.ndarray) -> np.ndarray:
64
61
  """Rebins `data` in time.
65
62
 
66
63
  Args:
67
64
  time: 1D time array.
68
65
  time_new: 1D new time array.
69
- mask_zeros: Whether to mask 0 values in the returned array. Default is True.
70
66
 
71
67
  Returns:
72
68
  Time indices without data.
73
69
 
74
70
  """
75
71
  if self.data.ndim == 1:
76
- self.data = utils.rebin_1d(time, self.data, time_new, mask_zeros=mask_zeros)
77
- bad_indices = list(np.where(self.data == ma.masked)[0])
72
+ self.data = utils.rebin_1d(time, self.data, time_new)
73
+ bad_indices = np.nonzero(self.data.mask)[0]
78
74
  else:
79
- if not isinstance(self.data, ma.MaskedArray):
80
- self.data = ma.masked_array(self.data)
81
- self.data, bad_indices = utils.rebin_2d(
82
- time, self.data, time_new, mask_zeros=mask_zeros
83
- )
75
+ self.data, bad_indices = utils.rebin_2d(time, self.data, time_new)
84
76
  return bad_indices
85
77
 
86
78
  def fetch_attributes(self) -> list:
@@ -108,6 +100,21 @@ class CloudnetArray:
108
100
  if data:
109
101
  setattr(self, key, data)
110
102
 
103
+ def filter_isolated_pixels(self) -> None:
104
+ """Filters hot pixels from radar data."""
105
+ self._filter(utils.filter_isolated_pixels)
106
+
107
+ def filter_vertical_stripes(self) -> None:
108
+ """Filters vertical artifacts from radar data."""
109
+ self._filter(utils.filter_x_pixels)
110
+
111
+ def _filter(self, fun) -> None:
112
+ if not isinstance(self.data, ma.MaskedArray):
113
+ self.data = ma.masked_array(self.data)
114
+ is_data = (~self.data.mask).astype(int)
115
+ is_data_filtered = fun(is_data)
116
+ self.data[is_data_filtered == 0] = ma.masked
117
+
111
118
  def _init_data(self) -> np.ndarray:
112
119
  if isinstance(self.variable, netCDF4.Variable):
113
120
  return self.variable[:]
@@ -139,73 +146,3 @@ class CloudnetArray:
139
146
 
140
147
  def __getitem__(self, ind: tuple) -> np.ndarray:
141
148
  return self.data[ind]
142
-
143
- def filter_isolated_pixels(self) -> None:
144
- """Filters hot pixels from radar data."""
145
- self._filter(utils.filter_isolated_pixels)
146
-
147
- def filter_vertical_stripes(self) -> None:
148
- """Filters vertical artifacts from radar data."""
149
- self._filter(utils.filter_x_pixels)
150
-
151
- def _filter(self, fun) -> None:
152
- if not isinstance(self.data, ma.MaskedArray):
153
- self.data = ma.masked_array(self.data)
154
- is_data = (~self.data.mask).astype(int)
155
- is_data_filtered = fun(is_data)
156
- self.data[is_data_filtered == 0] = ma.masked
157
-
158
- def calc_linear_std(self, time: np.ndarray, time_new: np.ndarray) -> None:
159
- """Calculates std of radar velocity.
160
-
161
- Args:
162
- time: 1D time array.
163
- time_new: 1D new time array.
164
-
165
- Notes:
166
- The result is masked if the bin contains masked values.
167
- """
168
- data_as_float = self.data.astype(float)
169
- data_as_float = ma.masked_array(data_as_float)
170
- self.data, _ = utils.rebin_2d(time, data_as_float, time_new, "std")
171
-
172
- def rebin_velocity(
173
- self,
174
- time: np.ndarray,
175
- time_new: np.ndarray,
176
- folding_velocity: float | np.ndarray,
177
- sequence_indices: list,
178
- ) -> None:
179
- """Rebins Doppler velocity in polar coordinates.
180
-
181
- Args:
182
- time: 1D time array.
183
- time_new: 1D new time array.
184
- folding_velocity: Folding velocity (m/s). Can be a float when
185
- it's the same for all altitudes, or np.ndarray when it
186
- matches difference altitude regions (defined in `sequence_indices`).
187
- sequence_indices: List containing indices of different folding regions,
188
- e.g. [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10]].
189
-
190
- """
191
-
192
- def _get_scaled_vfold() -> np.ndarray:
193
- vfold_scaled = math.pi / folding_velocity
194
- if isinstance(vfold_scaled, float):
195
- vfold_scaled = np.array([float(vfold_scaled)])
196
- return vfold_scaled
197
-
198
- def _scale_by_vfold(data_in: np.ndarray, fun) -> np.ndarray:
199
- data_out = ma.copy(data_in)
200
- for i, ind in enumerate(sequence_indices):
201
- data_out[:, ind] = fun(data_in[:, ind], folding_velocity_scaled[i])
202
- return data_out
203
-
204
- folding_velocity_scaled = _get_scaled_vfold()
205
- data_scaled = _scale_by_vfold(self.data, np.multiply)
206
- vel_x = ma.cos(data_scaled)
207
- vel_y = ma.sin(data_scaled)
208
- vel_x_mean, _ = utils.rebin_2d(time, vel_x, time_new)
209
- vel_y_mean, _ = utils.rebin_2d(time, vel_y, time_new)
210
- mean_vel_scaled = np.arctan2(vel_y_mean, vel_x_mean)
211
- self.data = _scale_by_vfold(mean_vel_scaled, np.divide)
@@ -9,9 +9,9 @@ from typing import Literal
9
9
 
10
10
  import netCDF4
11
11
  import numpy as np
12
+ from numpy import ma
12
13
 
13
14
  from cloudnetpy import utils
14
- from cloudnetpy.exceptions import InconsistentDataError
15
15
 
16
16
 
17
17
  def truncate_netcdf_file(
@@ -89,7 +89,7 @@ def concatenate_files(
89
89
  variables: list | None = None,
90
90
  new_attributes: dict | None = None,
91
91
  ignore: list | None = None,
92
- allow_difference: list | None = None,
92
+ interp_dimension: str = "range",
93
93
  ) -> list:
94
94
  """Concatenate netCDF files in one dimension.
95
95
 
@@ -101,22 +101,21 @@ def concatenate_files(
101
101
  Default is None when all variables with 'concat_dimension' will be saved.
102
102
  new_attributes: Optional new global attributes as {'attribute_name': value}.
103
103
  ignore: List of variables to be ignored.
104
- allow_difference: Names of scalar variables that can differ from one file to
105
- another (value from the first file is saved).
104
+ interp_dimension: Dimension name for interpolation if the dimensions
105
+ are not the same.
106
106
 
107
107
  Returns:
108
108
  List of filenames that were successfully concatenated.
109
109
 
110
110
  Notes:
111
- Arrays without 'concat_dimension', scalars, and global attributes will be taken
112
- from the first file. Groups, possibly present in a NETCDF4 formatted file,
113
- are ignored.
111
+ Arrays without 'concat_dimension' and scalars are expanded to the
112
+ concat_dimension. Global attributes are taken from the first file.
113
+ Groups, possibly present in a NETCDF4 formatted file, are ignored.
114
114
 
115
115
  """
116
- with _Concat(filenames, output_file, concat_dimension) as concat:
117
- concat.get_common_variables()
116
+ with _Concat(filenames, output_file, concat_dimension, interp_dimension) as concat:
118
117
  concat.create_global_attributes(new_attributes)
119
- return concat.concat_data(variables, ignore, allow_difference)
118
+ return concat.concat_data(variables, ignore)
120
119
 
121
120
 
122
121
  class _Concat:
@@ -127,19 +126,14 @@ class _Concat:
127
126
  filenames: Iterable[PathLike | str],
128
127
  output_file: str,
129
128
  concat_dimension: str = "time",
129
+ interp_dim: str = "range",
130
130
  ):
131
131
  self.filenames = sorted(map(Path, filenames), key=lambda f: f.name)
132
132
  self.concat_dimension = concat_dimension
133
+ self.interp_dim = interp_dim
133
134
  self.first_filename = self.filenames[0]
134
135
  self.first_file = netCDF4.Dataset(self.first_filename)
135
136
  self.concatenated_file = self._init_output_file(output_file)
136
- self.common_variables = set()
137
-
138
- def get_common_variables(self) -> None:
139
- """Finds variables which should have the same values in all files."""
140
- for key, value in self.first_file.variables.items():
141
- if self.concat_dimension not in value.dimensions:
142
- self.common_variables.add(key)
143
137
 
144
138
  def create_global_attributes(self, new_attributes: dict | None) -> None:
145
139
  """Copies global attributes from one of the source files."""
@@ -150,17 +144,16 @@ class _Concat:
150
144
 
151
145
  def concat_data(
152
146
  self,
153
- variables: list | None,
154
- ignore: list | None,
155
- allow_vary: list | None,
147
+ keep: list | None = None,
148
+ ignore: list | None = None,
156
149
  ) -> list:
157
150
  """Concatenates data arrays."""
158
- self._write_initial_data(variables, ignore)
151
+ self._write_initial_data(keep, ignore)
159
152
  output = [self.first_filename]
160
153
  if len(self.filenames) > 1:
161
154
  for filename in self.filenames[1:]:
162
155
  try:
163
- self._append_data(filename, allow_vary)
156
+ self._append_data(filename)
164
157
  except RuntimeError as e:
165
158
  if "NetCDF: HDF error" in str(e):
166
159
  msg = f"Caught a NetCDF HDF error. Skipping file '{filename}'."
@@ -170,24 +163,28 @@ class _Concat:
170
163
  output.append(filename)
171
164
  return output
172
165
 
173
- def _write_initial_data(self, variables: list | None, ignore: list | None) -> None:
174
- for key in self.first_file.variables:
166
+ def _write_initial_data(self, keep: list | None, ignore: list | None) -> None:
167
+ len_concat_dim = self.first_file[self.concat_dimension].size
168
+ auto_scale = False
169
+
170
+ for key, var in self.first_file.variables.items():
175
171
  if (
176
- variables is not None
177
- and key not in variables
178
- and key not in self.common_variables
172
+ # This filtering only affects variables having the concat_dimension
173
+ keep is not None
174
+ and key not in keep
179
175
  and key != self.concat_dimension
176
+ and self.concat_dimension in var.dimensions
180
177
  ):
181
178
  continue
182
179
  if ignore and key in ignore:
183
180
  continue
184
181
 
185
- auto_scale = False
186
- self.first_file[key].set_auto_scale(auto_scale)
187
- array = self.first_file[key][:]
188
- dimensions = self.first_file[key].dimensions
189
- fill_value = getattr(self.first_file[key], "_FillValue", None)
190
- var = self.concatenated_file.createVariable(
182
+ var.set_auto_scale(auto_scale)
183
+ array, dimensions = self._expand_array(var, len_concat_dim)
184
+
185
+ fill_value = var.get_fill_value()
186
+
187
+ var_new = self.concatenated_file.createVariable(
191
188
  key,
192
189
  array.dtype,
193
190
  dimensions,
@@ -196,37 +193,49 @@ class _Concat:
196
193
  shuffle=False,
197
194
  fill_value=fill_value,
198
195
  )
199
- auto_scale = False
200
- var.set_auto_scale(auto_scale)
201
- var[:] = array
202
- _copy_attributes(self.first_file[key], var)
203
-
204
- def _append_data(self, filename: str | PathLike, allow_vary: list | None) -> None:
196
+ var_new.set_auto_scale(auto_scale)
197
+ var_new[:] = array
198
+ _copy_attributes(var, var_new)
199
+
200
+ def _expand_array(
201
+ self, var: netCDF4.Variable, n_data: int
202
+ ) -> tuple[ma.MaskedArray, tuple[str, ...]]:
203
+ dimensions = var.dimensions
204
+ arr = var[:]
205
+ if self.concat_dimension not in dimensions and var.name != self.interp_dim:
206
+ dimensions = (self.concat_dimension, *dimensions)
207
+ arr = np.repeat(arr[np.newaxis, ...], n_data, axis=0)
208
+
209
+ return arr, dimensions
210
+
211
+ def _append_data(self, filename: str | PathLike) -> None:
205
212
  with netCDF4.Dataset(filename) as file:
206
213
  auto_scale = False
207
214
  file.set_auto_scale(auto_scale)
208
215
  ind0 = len(self.concatenated_file.variables[self.concat_dimension])
209
216
  ind1 = ind0 + len(file.variables[self.concat_dimension])
217
+ n_points = ind1 - ind0
218
+
210
219
  for key in self.concatenated_file.variables:
211
- if key not in file.variables:
212
- continue
213
- array = file[key][:]
214
- if key in self.common_variables:
215
- if allow_vary is not None and key in allow_vary:
216
- continue
217
- if not np.array_equal(self.first_file[key][:], array):
218
- msg = (
219
- f"Inconsistent values in variable '{key}' between "
220
- f"files '{self.first_filename}' and '{filename}'"
221
- )
222
- raise InconsistentDataError(msg)
220
+ if key not in file.variables or key == self.interp_dim:
223
221
  continue
224
- if array.ndim == 0:
225
- continue
226
- if array.ndim == 1:
227
- self.concatenated_file.variables[key][ind0:ind1] = array
228
- else:
229
- self.concatenated_file.variables[key][ind0:ind1, :] = array
222
+
223
+ array, dimensions = self._expand_array(file[key], n_points)
224
+
225
+ # Nearest neighbour interpolation in the interp_dim dimension
226
+ # if the dimensions are not the same between the files
227
+ if self.interp_dim in dimensions and (
228
+ self.first_file[self.interp_dim].size != file[self.interp_dim].size
229
+ ):
230
+ x = file.variables[self.interp_dim][:]
231
+ x_target = self.first_file.variables[self.interp_dim][:]
232
+ idx = np.abs(x[:, None] - x_target[None, :]).argmin(axis=0)
233
+ array = array[:, idx]
234
+ out_of_bounds = (x_target < x.min()) | (x_target > x.max())
235
+ fill_value = self.first_file.variables[key].get_fill_value()
236
+ array[:, out_of_bounds] = fill_value
237
+
238
+ self.concatenated_file.variables[key][ind0:ind1, ...] = array
230
239
 
231
240
  def _init_output_file(self, output_file: str) -> netCDF4.Dataset:
232
241
  data_model: Literal["NETCDF4", "NETCDF4_CLASSIC"] = (
@@ -182,53 +182,6 @@ class DataSource:
182
182
  return np.array(range_instrument + self.altitude)
183
183
  return None
184
184
 
185
- def _variables_to_cloudnet_arrays(self, keys: tuple) -> None:
186
- """Transforms netCDF4-variables into CloudnetArrays.
187
-
188
- Args:
189
- keys: netCDF4-variables to be converted. The results
190
- are saved in *self.data* dictionary with *fields*
191
- strings as keys.
192
-
193
- Notes:
194
- The attributes of the variables are not copied. Just the data.
195
-
196
- """
197
- for key in keys:
198
- self.append_data(self.dataset.variables[key], key)
199
-
200
- def _unknown_variable_to_cloudnet_array(
201
- self,
202
- possible_names: tuple,
203
- key: str,
204
- units: str | None = None,
205
- *,
206
- ignore_mask: bool = False,
207
- ) -> None:
208
- """Transforms single netCDF4 variable into CloudnetArray.
209
-
210
- Args:
211
- possible_names: Tuple of strings containing the possible
212
- names of the variable in the input NetCDF file.
213
- key: Key for self.data dictionary and name-attribute
214
- for the saved CloudnetArray object.
215
- units: Units attribute for the CloudnetArray object.
216
- ignore_mask: If true, always writes an ordinary numpy array.
217
-
218
- Raises:
219
- RuntimeError: No variable found.
220
-
221
- """
222
- for name in possible_names:
223
- if name in self.dataset.variables:
224
- array: netCDF4.Variable | np.ndarray = self.dataset.variables[name]
225
- if ignore_mask is True:
226
- array = np.array(array)
227
- self.append_data(array, key, units=units)
228
- return
229
- msg = f"Missing variable {possible_names[0]} in the input file."
230
- raise RuntimeError(msg)
231
-
232
185
  def __enter__(self):
233
186
  return self
234
187
 
@@ -80,8 +80,8 @@ class Bowtie(NcRadar):
80
80
  self.data["relative_humidity"].data /= 100
81
81
 
82
82
  def fix_chirp_start_indices(self) -> None:
83
- ind = self.data["chirp_start_indices"].data
84
- self.data["chirp_start_indices"].data = np.array([int(i) for i in ind])
83
+ array = self.data["chirp_start_indices"].data
84
+ self.data["chirp_start_indices"].data = np.array(array, dtype=np.int32)
85
85
  self.data["chirp_start_indices"].data_type = "int32"
86
86
 
87
87
  def check_date(self, date: str):
@@ -69,7 +69,6 @@ def copernicus2nc(
69
69
  valid_filenames = utils.get_files_with_variables(
70
70
  valid_filenames, ["time", "ZED_HC"]
71
71
  )
72
- valid_filenames = utils.get_files_with_common_range(valid_filenames)
73
72
  variables = list(keymap.keys())
74
73
  concat_lib.concatenate_files(
75
74
  valid_filenames,
@@ -68,7 +68,6 @@ def galileo2nc(
68
68
  valid_filenames = utils.get_files_with_variables(
69
69
  valid_filenames, ["time", "ZED_HC"]
70
70
  )
71
- valid_filenames = utils.get_files_with_common_range(valid_filenames)
72
71
  variables = list(keymap.keys())
73
72
  concat_lib.concatenate_files(
74
73
  valid_filenames,