cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. cloudnetpy/categorize/__init__.py +1 -2
  2. cloudnetpy/categorize/atmos_utils.py +297 -67
  3. cloudnetpy/categorize/attenuation.py +31 -0
  4. cloudnetpy/categorize/attenuations/__init__.py +37 -0
  5. cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
  6. cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
  7. cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
  8. cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
  9. cloudnetpy/categorize/categorize.py +332 -156
  10. cloudnetpy/categorize/classify.py +127 -125
  11. cloudnetpy/categorize/containers.py +107 -76
  12. cloudnetpy/categorize/disdrometer.py +40 -0
  13. cloudnetpy/categorize/droplet.py +23 -21
  14. cloudnetpy/categorize/falling.py +53 -24
  15. cloudnetpy/categorize/freezing.py +25 -12
  16. cloudnetpy/categorize/insects.py +35 -23
  17. cloudnetpy/categorize/itu.py +243 -0
  18. cloudnetpy/categorize/lidar.py +36 -41
  19. cloudnetpy/categorize/melting.py +34 -26
  20. cloudnetpy/categorize/model.py +84 -37
  21. cloudnetpy/categorize/mwr.py +18 -14
  22. cloudnetpy/categorize/radar.py +215 -102
  23. cloudnetpy/cli.py +578 -0
  24. cloudnetpy/cloudnetarray.py +43 -89
  25. cloudnetpy/concat_lib.py +218 -78
  26. cloudnetpy/constants.py +28 -10
  27. cloudnetpy/datasource.py +61 -86
  28. cloudnetpy/exceptions.py +49 -20
  29. cloudnetpy/instruments/__init__.py +5 -0
  30. cloudnetpy/instruments/basta.py +29 -12
  31. cloudnetpy/instruments/bowtie.py +135 -0
  32. cloudnetpy/instruments/ceilo.py +138 -115
  33. cloudnetpy/instruments/ceilometer.py +164 -80
  34. cloudnetpy/instruments/cl61d.py +21 -5
  35. cloudnetpy/instruments/cloudnet_instrument.py +74 -36
  36. cloudnetpy/instruments/copernicus.py +108 -30
  37. cloudnetpy/instruments/da10.py +54 -0
  38. cloudnetpy/instruments/disdrometer/common.py +126 -223
  39. cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
  40. cloudnetpy/instruments/disdrometer/thies.py +254 -87
  41. cloudnetpy/instruments/fd12p.py +201 -0
  42. cloudnetpy/instruments/galileo.py +65 -23
  43. cloudnetpy/instruments/hatpro.py +123 -49
  44. cloudnetpy/instruments/instruments.py +113 -1
  45. cloudnetpy/instruments/lufft.py +39 -17
  46. cloudnetpy/instruments/mira.py +268 -61
  47. cloudnetpy/instruments/mrr.py +187 -0
  48. cloudnetpy/instruments/nc_lidar.py +19 -8
  49. cloudnetpy/instruments/nc_radar.py +109 -55
  50. cloudnetpy/instruments/pollyxt.py +135 -51
  51. cloudnetpy/instruments/radiometrics.py +313 -59
  52. cloudnetpy/instruments/rain_e_h3.py +171 -0
  53. cloudnetpy/instruments/rpg.py +321 -189
  54. cloudnetpy/instruments/rpg_reader.py +74 -40
  55. cloudnetpy/instruments/toa5.py +49 -0
  56. cloudnetpy/instruments/vaisala.py +95 -343
  57. cloudnetpy/instruments/weather_station.py +774 -105
  58. cloudnetpy/metadata.py +90 -19
  59. cloudnetpy/model_evaluation/file_handler.py +55 -52
  60. cloudnetpy/model_evaluation/metadata.py +46 -20
  61. cloudnetpy/model_evaluation/model_metadata.py +1 -1
  62. cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
  63. cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
  64. cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
  65. cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
  66. cloudnetpy/model_evaluation/products/model_products.py +43 -35
  67. cloudnetpy/model_evaluation/products/observation_products.py +41 -35
  68. cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
  69. cloudnetpy/model_evaluation/products/tools.py +29 -20
  70. cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
  71. cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
  72. cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
  73. cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
  74. cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
  75. cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
  76. cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
  77. cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
  78. cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
  79. cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
  80. cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
  81. cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
  82. cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
  83. cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
  84. cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
  85. cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
  86. cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
  87. cloudnetpy/model_evaluation/utils.py +2 -1
  88. cloudnetpy/output.py +170 -111
  89. cloudnetpy/plotting/__init__.py +2 -1
  90. cloudnetpy/plotting/plot_meta.py +562 -822
  91. cloudnetpy/plotting/plotting.py +1142 -704
  92. cloudnetpy/products/__init__.py +1 -0
  93. cloudnetpy/products/classification.py +370 -88
  94. cloudnetpy/products/der.py +85 -55
  95. cloudnetpy/products/drizzle.py +77 -34
  96. cloudnetpy/products/drizzle_error.py +15 -11
  97. cloudnetpy/products/drizzle_tools.py +79 -59
  98. cloudnetpy/products/epsilon.py +211 -0
  99. cloudnetpy/products/ier.py +27 -50
  100. cloudnetpy/products/iwc.py +55 -48
  101. cloudnetpy/products/lwc.py +96 -70
  102. cloudnetpy/products/mwr_tools.py +186 -0
  103. cloudnetpy/products/product_tools.py +170 -128
  104. cloudnetpy/utils.py +455 -240
  105. cloudnetpy/version.py +2 -2
  106. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
  107. cloudnetpy-1.87.3.dist-info/RECORD +127 -0
  108. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
  109. cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
  110. docs/source/conf.py +2 -2
  111. cloudnetpy/categorize/atmos.py +0 -361
  112. cloudnetpy/products/mwr_multi.py +0 -68
  113. cloudnetpy/products/mwr_single.py +0 -75
  114. cloudnetpy-1.49.9.dist-info/RECORD +0 -112
  115. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
  116. {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,10 @@
1
1
  import importlib
2
2
  import logging
3
+ import os.path
4
+ from os import PathLike
3
5
 
4
6
  import numpy as np
7
+ import numpy.typing as npt
5
8
  from numpy import ma
6
9
 
7
10
  from cloudnetpy.datasource import DataSource
@@ -32,12 +35,13 @@ class ModelManager(DataSource):
32
35
 
33
36
  def __init__(
34
37
  self,
35
- model_file: str,
38
+ model_file: str | PathLike,
36
39
  model: str,
37
- output_file: str,
40
+ output_file: str | PathLike,
38
41
  product: str,
42
+ *,
39
43
  check_file: bool = True,
40
- ):
44
+ ) -> None:
41
45
  super().__init__(model_file)
42
46
  self.model = model
43
47
  self.model_info = MODELS[model]
@@ -52,31 +56,33 @@ class ModelManager(DataSource):
52
56
  self.wind = self._calculate_wind_speed()
53
57
  self.resolution_h = self._get_horizontal_resolution()
54
58
 
55
- def _read_cycle_name(self, model_file: str):
56
- """Get cycle name from model_metadata.py for saving variable name(s)"""
59
+ def _read_cycle_name(self, model_file: str | PathLike) -> str:
60
+ """Get cycle name from model_metadata.py for saving variable name(s)."""
61
+ basename = os.path.basename(model_file)
57
62
  try:
58
63
  cycles = self.model_info.cycle
59
64
  if cycles is None:
60
65
  return ""
61
66
  cycles_split = [x.strip() for x in cycles.split(",")]
62
67
  for cycle in cycles_split:
63
- if cycle in model_file:
68
+ if cycle in basename:
64
69
  return f"_{cycle}"
65
70
  except AttributeError:
66
71
  return ""
67
72
  return ""
68
73
 
69
- def _generate_products(self):
70
- """Process needed data of model to a ModelManager object"""
71
- cls = getattr(importlib.import_module(__name__), "ModelManager")
74
+ def _generate_products(self) -> None:
75
+ """Process needed data of model to a ModelManager object."""
76
+ cls = importlib.import_module(__name__).ModelManager
72
77
  try:
73
78
  name = f"_get_{self._product}"
74
79
  getattr(cls, name)(self)
75
- except AttributeError as e:
76
- logging.error(f"Invalid product name: {e}")
80
+ except AttributeError:
81
+ msg = f"Invalid product name: {self._product}"
82
+ logging.exception(msg)
77
83
  raise
78
84
 
79
- def _get_cf(self):
85
+ def _get_cf(self) -> None:
80
86
  """Collect cloud fraction straight from model file."""
81
87
  cf_name = self.get_model_var_names(("cf",))[0]
82
88
  cf = self.getvar(cf_name)
@@ -85,14 +91,14 @@ class ModelManager(DataSource):
85
91
  self.append_data(cf, f"{self.model}{self.cycle}_cf")
86
92
  self.keys[self._product] = f"{self.model}{self.cycle}_cf"
87
93
 
88
- def _get_iwc(self):
89
- iwc = self.get_water_continent("iwc")
94
+ def _get_iwc(self) -> None:
95
+ iwc = self.get_water_content("iwc")
90
96
  iwc[iwc < 1e-7] = ma.masked
91
97
  self.append_data(iwc, f"{self.model}{self.cycle}_iwc")
92
98
  self.keys[self._product] = f"{self.model}{self.cycle}_iwc"
93
99
 
94
- def _get_lwc(self):
95
- lwc = self.get_water_continent("lwc")
100
+ def _get_lwc(self) -> None:
101
+ lwc = self.get_water_content("lwc")
96
102
  lwc[lwc < 1e-5] = ma.masked
97
103
  self.append_data(lwc, f"{self.model}{self.cycle}_lwc")
98
104
  self.keys[self._product] = f"{self.model}{self.cycle}_lwc"
@@ -104,7 +110,7 @@ class ModelManager(DataSource):
104
110
  var.append(VARIABLES[arg].long_name)
105
111
  return var
106
112
 
107
- def get_water_continent(self, var: str) -> np.ndarray:
113
+ def get_water_content(self, var: str) -> npt.NDArray:
108
114
  p_name = self.get_model_var_names(("p",))[0]
109
115
  t_name = self.get_model_var_names(("T",))[0]
110
116
  lwc_name = self.get_model_var_names((var,))[0]
@@ -117,16 +123,20 @@ class ModelManager(DataSource):
117
123
  return wc
118
124
 
119
125
  @staticmethod
120
- def _calc_water_content(q: np.ndarray, p: np.ndarray, t: np.ndarray) -> np.ndarray:
126
+ def _calc_water_content(
127
+ q: npt.NDArray, p: npt.NDArray, t: npt.NDArray
128
+ ) -> npt.NDArray:
121
129
  return q * p / (287 * t)
122
130
 
123
- def _add_variables(self):
124
- """Add basic variables off model and cycle"""
131
+ def _add_variables(self) -> None:
132
+ """Add basic variables off model and cycle."""
125
133
 
126
- def _add_common_variables():
127
- """Model variables that are always the same within cycles"""
134
+ def _add_common_variables() -> None:
135
+ """Model variables that are always the same within cycles."""
128
136
  wanted_vars = self.model_vars.common_var
129
- assert wanted_vars is not None
137
+ if wanted_vars is None:
138
+ msg = f"Model {self.model} has no common variables"
139
+ raise ValueError(msg)
130
140
  wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
131
141
  for var in wanted_vars_split:
132
142
  if var in self.dataset.variables:
@@ -135,10 +145,12 @@ class ModelManager(DataSource):
135
145
  data = self.cut_off_extra_levels(self.dataset.variables[var][:])
136
146
  self.append_data(data, f"{var}")
137
147
 
138
- def _add_cycle_variables():
139
- """Add cycle depending variables"""
148
+ def _add_cycle_variables() -> None:
149
+ """Add cycle depending variables."""
140
150
  wanted_vars = self.model_vars.cycle_var
141
- assert wanted_vars is not None
151
+ if wanted_vars is None:
152
+ msg = f"Model {self.model} has no cycle variables"
153
+ raise ValueError(msg)
142
154
  wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
143
155
  for var in wanted_vars_split:
144
156
  if var in self.dataset.variables:
@@ -153,21 +165,17 @@ class ModelManager(DataSource):
153
165
  _add_common_variables()
154
166
  _add_cycle_variables()
155
167
 
156
- def cut_off_extra_levels(self, data: np.ndarray) -> np.ndarray:
157
- """Remove unused levels (over 22km) from model data"""
168
+ def cut_off_extra_levels(self, data: npt.NDArray) -> npt.NDArray:
169
+ """Remove unused levels (over 22km) from model data."""
158
170
  try:
159
171
  level = self.model_info.level
160
172
  except KeyError:
161
173
  return data
162
174
 
163
- if data.ndim > 1:
164
- data = data[:, :level]
165
- else:
166
- data = data[:level]
167
- return data
175
+ return data[:, :level] if data.ndim > 1 else data[:level]
168
176
 
169
- def _calculate_wind_speed(self) -> np.ndarray:
170
- """Real wind from x- and y-components"""
177
+ def _calculate_wind_speed(self) -> npt.NDArray:
178
+ """Real wind from x- and y-components."""
171
179
  u = self.getvar("uwind")
172
180
  v = self.getvar("vwind")
173
181
  u = self.cut_off_extra_levels(u)
@@ -1,12 +1,14 @@
1
1
  import logging
2
- from datetime import datetime
2
+ from datetime import datetime, timezone
3
+ from os import PathLike
3
4
 
4
5
  import numpy as np
6
+ import numpy.typing as npt
5
7
  from numpy import ma
6
8
 
7
9
  from cloudnetpy import utils
8
10
  from cloudnetpy.datasource import DataSource
9
- from cloudnetpy.products.product_tools import CategorizeBits
11
+ from cloudnetpy.products.product_tools import CategorizeBits, CategoryBits
10
12
 
11
13
 
12
14
  class ObservationManager(DataSource):
@@ -24,7 +26,7 @@ class ObservationManager(DataSource):
24
26
  should be processed using CloudnetPy for this class to work properly.
25
27
  """
26
28
 
27
- def __init__(self, obs: str, obs_file: str):
29
+ def __init__(self, obs: str, obs_file: str | PathLike) -> None:
28
30
  super().__init__(obs_file)
29
31
  self.obs = obs
30
32
  self._file = obs_file
@@ -42,22 +44,23 @@ class ObservationManager(DataSource):
42
44
  0,
43
45
  0,
44
46
  0,
47
+ tzinfo=timezone.utc,
45
48
  )
46
49
 
47
- def _get_radar_frequency(self) -> np.ndarray | None:
50
+ def _get_radar_frequency(self) -> npt.NDArray | None:
48
51
  try:
49
52
  return self.getvar("radar_frequency")
50
53
  except (KeyError, RuntimeError):
51
54
  return None
52
55
 
53
- def _get_z_sensitivity(self) -> np.ndarray | None:
56
+ def _get_z_sensitivity(self) -> npt.NDArray | None:
54
57
  try:
55
58
  return self.getvar("Z_sensitivity")
56
59
  except (KeyError, RuntimeError):
57
60
  return None
58
61
 
59
- def _generate_product(self):
60
- """Process needed data of observation to a ObservationManager object"""
62
+ def _generate_product(self) -> None:
63
+ """Process needed data of observation to a ObservationManager object."""
61
64
  try:
62
65
  if self.obs == "cf":
63
66
  self.append_data(self._generate_cf(), "cf")
@@ -66,31 +69,31 @@ class ObservationManager(DataSource):
66
69
  if self.obs == "iwc":
67
70
  self._generate_iwc_masks()
68
71
  self.append_data(self.getvar("height"), "height")
69
- except (KeyError, RuntimeError) as e:
70
- logging.error(f"Invalid product name: {e}")
72
+ except (KeyError, RuntimeError):
73
+ msg = f"Failed to read {self.obs} from {self._file}"
74
+ logging.exception(msg)
71
75
  raise
72
76
 
73
- def _generate_cf(self) -> np.ndarray:
74
- """Generates cloud fractions using categorize bits and masking conditions"""
77
+ def _generate_cf(self) -> npt.NDArray:
78
+ """Generates cloud fractions using categorize bits and masking conditions."""
75
79
  categorize_bits = CategorizeBits(self._file)
76
80
  cloud_mask = self._classify_basic_mask(categorize_bits.category_bits)
77
- cloud_mask = self._mask_cloud_bits(cloud_mask)
78
- return cloud_mask
81
+ return self._mask_cloud_bits(cloud_mask)
79
82
 
80
83
  @staticmethod
81
- def _classify_basic_mask(bits: dict) -> np.ndarray:
82
- cloud_mask = bits["droplet"] + bits["falling"] * 2
83
- cloud_mask[bits["falling"] & bits["cold"]] = (
84
- cloud_mask[bits["falling"] & bits["cold"]] + 2
84
+ def _classify_basic_mask(bits: CategoryBits) -> npt.NDArray:
85
+ cloud_mask = bits.droplet + bits.falling * 2
86
+ cloud_mask[bits.falling & bits.freezing] = (
87
+ cloud_mask[bits.falling & bits.freezing] + 2
85
88
  )
86
- cloud_mask[bits["aerosol"]] = 6
87
- cloud_mask[bits["insect"]] = 7
88
- cloud_mask[bits["aerosol"] & bits["insect"]] = 8
89
+ cloud_mask[bits.aerosol] = 6
90
+ cloud_mask[bits.insect] = 7
91
+ cloud_mask[bits.aerosol & bits.insect] = 8
89
92
  return cloud_mask
90
93
 
91
94
  @staticmethod
92
- def _mask_cloud_bits(cloud_mask: np.ndarray) -> np.ndarray:
93
- """Creates cloud fraction"""
95
+ def _mask_cloud_bits(cloud_mask: npt.NDArray) -> npt.NDArray:
96
+ """Creates cloud fraction."""
94
97
  for i in [1, 3, 4, 5]:
95
98
  cloud_mask[cloud_mask == i] = 1
96
99
  for i in [2, 6, 7, 8]:
@@ -98,28 +101,30 @@ class ObservationManager(DataSource):
98
101
  return cloud_mask
99
102
 
100
103
  def _check_rainrate(self) -> bool:
101
- """Check if rainrate in file"""
104
+ """Check if rainrate in file."""
102
105
  try:
103
106
  self.getvar("rainrate")
104
- return True
105
- except RuntimeError:
107
+ except KeyError:
106
108
  return False
109
+ return True
107
110
 
108
111
  def _get_rainrate_threshold(self) -> int:
109
- assert self.radar_freq is not None
112
+ if self.radar_freq is None:
113
+ msg = "Radar frequency not found from file"
114
+ raise RuntimeError(msg)
110
115
  wband = utils.get_wl_band(float(self.radar_freq))
111
116
  rainrate_threshold = 8
112
- if 90 < wband < 100:
117
+ if wband == "W":
113
118
  rainrate_threshold = 2
114
119
  return rainrate_threshold
115
120
 
116
- def _rain_index(self) -> np.ndarray:
121
+ def _rain_index(self) -> npt.NDArray:
117
122
  rainrate = self.getvar("rainrate")
118
123
  rainrate_threshold = self._get_rainrate_threshold()
119
124
  return rainrate > rainrate_threshold
120
125
 
121
126
  def _generate_iwc_masks(self) -> None:
122
- """Generates ice water content variables with different masks"""
127
+ """Generates ice water content variables with different masks."""
123
128
  # TODO: Differences with CloudnetPy (status=2) and Legacy data (status=3)
124
129
  iwc = self.getvar(self.obs)
125
130
  iwc_status = self.getvar("iwc_retrieval_status")
@@ -127,21 +132,22 @@ class ObservationManager(DataSource):
127
132
  self._get_rain_iwc(iwc_status)
128
133
  self._mask_iwc(iwc, iwc_status)
129
134
 
130
- def _mask_iwc(self, iwc: np.ndarray, iwc_status: np.ndarray) -> None:
131
- """Leaves only reliable data and corrected liquid attenuation"""
135
+ def _mask_iwc(self, iwc: npt.NDArray, iwc_status: npt.NDArray) -> None:
136
+ """Leaves only reliable data and corrected liquid attenuation."""
132
137
  iwc_mask = ma.copy(iwc)
133
138
  iwc_mask[np.bitwise_and(iwc_status != 1, iwc_status != 2)] = ma.masked
134
139
  self.append_data(iwc_mask, "iwc")
135
140
 
136
- def _mask_iwc_att(self, iwc: np.ndarray, iwc_status: np.ndarray) -> None:
141
+ def _mask_iwc_att(self, iwc: npt.NDArray, iwc_status: npt.NDArray) -> None:
137
142
  """Leaves only where reliable data, corrected liquid attenuation
138
- and uncorrected liquid attenuation"""
143
+ and uncorrected liquid attenuation.
144
+ """
139
145
  iwc_att = ma.copy(iwc)
140
146
  iwc_att[iwc_status > 3] = ma.masked
141
147
  self.append_data(iwc_att, "iwc_att")
142
148
 
143
- def _get_rain_iwc(self, iwc_status: np.ndarray) -> None:
144
- """Finds columns where is rain, return boolean of x-axis shape"""
149
+ def _get_rain_iwc(self, iwc_status: npt.NDArray) -> None:
150
+ """Finds columns where is rain, return boolean of x-axis shape."""
145
151
  iwc_rain = np.zeros(iwc_status.shape, dtype=bool)
146
152
  iwc_rain[iwc_status == 5] = 1
147
153
  iwc_rain = np.any(iwc_rain, axis=1)
@@ -1,4 +1,6 @@
1
1
  import logging
2
+ from os import PathLike
3
+ from uuid import UUID
2
4
 
3
5
  import cloudnetpy.model_evaluation.products.tools as tl
4
6
  from cloudnetpy.model_evaluation.file_handler import (
@@ -12,22 +14,25 @@ from cloudnetpy.model_evaluation.products.grid_methods import ProductGrid
12
14
  from cloudnetpy.model_evaluation.products.model_products import ModelManager
13
15
  from cloudnetpy.model_evaluation.products.observation_products import ObservationManager
14
16
  from cloudnetpy.model_evaluation.utils import file_exists
17
+ from cloudnetpy.utils import get_uuid
15
18
 
16
19
 
17
20
  def process_L3_day_product(
18
21
  model: str,
19
22
  obs: str,
20
- model_files: list,
21
- product_file: str,
22
- output_file: str,
23
- uuid: str | None = None,
23
+ model_files: list[str | PathLike],
24
+ product_file: str | PathLike,
25
+ output_file: str | PathLike,
26
+ uuid: str | UUID | None = None,
27
+ *,
24
28
  overwrite: bool = False,
25
- ):
29
+ ) -> UUID:
26
30
  """Main function to generate downsample of observations to match model grid.
27
31
 
28
32
  This function will generate a L3 product nc-file. It includes the information of
29
33
  downsampled observation products for each model cycles and model products
30
34
  and other variables of each cycles.
35
+
31
36
  Args:
32
37
  model (str): Name of model
33
38
  obs (str): Name of product to generate
@@ -61,11 +66,16 @@ def process_L3_day_product(
61
66
  >>> process_L3_day_product(model, product, [model_file], input_file,
62
67
  output_file)
63
68
  """
69
+ uuid = get_uuid(uuid)
64
70
  product_obj = ObservationManager(obs, product_file)
65
71
  tl.check_model_file_list(model, model_files)
66
72
  for m_file in model_files:
67
73
  model_obj = ModelManager(
68
- m_file, model, output_file, obs, check_file=not overwrite
74
+ m_file,
75
+ model,
76
+ output_file,
77
+ obs,
78
+ check_file=not overwrite,
69
79
  )
70
80
  try:
71
81
  AdvanceProductMethods(model_obj, m_file, product_obj)
@@ -76,7 +86,7 @@ def process_L3_day_product(
76
86
  update_attributes(model_obj.data, attributes)
77
87
  if not file_exists(output_file) or overwrite:
78
88
  tl.add_date(model_obj, product_obj)
79
- uuid = save_downsampled_file(
89
+ save_downsampled_file(
80
90
  f"{obs}_{model}",
81
91
  output_file,
82
92
  (model_obj, product_obj),
@@ -1,27 +1,32 @@
1
1
  import datetime
2
2
  import logging
3
+ import os.path
4
+ from collections.abc import Sequence
3
5
  from datetime import timedelta
6
+ from os import PathLike
4
7
 
5
8
  import numpy as np
9
+ import numpy.typing as npt
6
10
  from numpy import ma
7
11
 
8
12
  from cloudnetpy.model_evaluation.products.model_products import ModelManager
9
13
  from cloudnetpy.model_evaluation.products.observation_products import ObservationManager
10
14
 
11
15
 
12
- def check_model_file_list(name: str, models: list) -> None:
13
- """Check that files in models are from same model and date"""
16
+ def check_model_file_list(name: str, models: Sequence[str | PathLike]) -> None:
17
+ """Check that files in models are from same model and date."""
14
18
  for m in models:
15
- if name not in m:
19
+ if name not in os.path.basename(m):
16
20
  logging.error("Invalid model file set")
17
- raise AttributeError(f"{m} not from {name}")
21
+ msg = f"{m} not from {name}"
22
+ raise AttributeError(msg)
18
23
 
19
24
 
20
- def time2datetime(time: np.ndarray, date: datetime.datetime) -> np.ndarray:
25
+ def time2datetime(time: npt.NDArray, date: datetime.datetime) -> npt.NDArray:
21
26
  return np.asarray([date + timedelta(hours=float(t)) for t in time])
22
27
 
23
28
 
24
- def rebin_edges(arr: np.ndarray) -> np.ndarray:
29
+ def rebin_edges(arr: npt.NDArray) -> npt.NDArray:
25
30
  """Rebins array bins by half and adds boundaries."""
26
31
  new_arr = [(arr[i] + arr[i + 1]) / 2 for i in range(len(arr) - 1)]
27
32
  new_arr.insert(0, arr[0] - ((arr[0] + arr[1]) / 2))
@@ -30,9 +35,11 @@ def rebin_edges(arr: np.ndarray) -> np.ndarray:
30
35
 
31
36
 
32
37
  def calculate_advection_time(
33
- resolution: int, wind: ma.MaskedArray, sampling: int
34
- ) -> np.ndarray:
35
- """Calculates time which variable takes to go through the time window
38
+ resolution: int,
39
+ wind: ma.MaskedArray,
40
+ sampling: int,
41
+ ) -> npt.NDArray:
42
+ """Calculates time which variable takes to go through the time window.
36
43
 
37
44
  Notes:
38
45
  Wind speed is stronger in upper levels, so advection time is more
@@ -40,8 +47,6 @@ def calculate_advection_time(
40
47
  but visible in a tropics.
41
48
 
42
49
  sampling = 1 -> hour, sampling 1/6 -> 10min
43
-
44
- References:
45
50
  """
46
51
  t_adv = resolution * 1000 / wind / 60**2
47
52
  t_adv[t_adv.mask] = 0
@@ -49,8 +54,12 @@ def calculate_advection_time(
49
54
  return np.asarray([[timedelta(hours=float(t)) for t in time] for time in t_adv])
50
55
 
51
56
 
52
- def get_1d_indices(window: tuple, data: np.ndarray, mask: np.ndarray | None = None):
53
- indices = (window[0] <= data) & (data < window[-1])
57
+ def get_1d_indices(
58
+ window: tuple,
59
+ data: npt.NDArray,
60
+ mask: npt.NDArray | None = None,
61
+ ) -> npt.NDArray:
62
+ indices: npt.NDArray = np.array((window[0] <= data) & (data < window[-1]))
54
63
  if mask is not None:
55
64
  indices[mask] = ma.masked
56
65
  return indices
@@ -59,17 +68,17 @@ def get_1d_indices(window: tuple, data: np.ndarray, mask: np.ndarray | None = No
59
68
  def get_adv_indices(
60
69
  model_t: int,
61
70
  adv_t: float,
62
- data: np.ndarray,
63
- mask: np.ndarray | None = None,
64
- ):
71
+ data: npt.NDArray,
72
+ mask: npt.NDArray | None = None,
73
+ ) -> npt.NDArray:
65
74
  adv_indices = ((model_t - adv_t / 2) <= data) & (data < (model_t + adv_t / 2))
66
75
  if mask is not None:
67
76
  adv_indices[mask] = ma.masked
68
77
  return adv_indices
69
78
 
70
79
 
71
- def get_obs_window_size(ind_x: np.ndarray, ind_y: np.ndarray) -> tuple | None:
72
- """Returns shape (tuple) of window area, where values are True"""
80
+ def get_obs_window_size(ind_x: npt.NDArray, ind_y: npt.NDArray) -> tuple | None:
81
+ """Returns shape (tuple) of window area, where values are True."""
73
82
  x = np.where(ind_x)[0]
74
83
  y = np.where(ind_y)[0]
75
84
  if np.any(x) and np.any(y):
@@ -82,6 +91,6 @@ def add_date(model_obj: ModelManager, obs_obj: ObservationManager) -> None:
82
91
  model_obj.date.append(getattr(obs_obj.dataset, a))
83
92
 
84
93
 
85
- def average_column_sum(data: np.ndarray) -> np.ndarray:
86
- """Returns average sum of columns which have any data"""
94
+ def average_column_sum(data: npt.NDArray) -> npt.NDArray:
95
+ """Returns average sum of columns which have any data."""
87
96
  return np.nanmean(np.nansum(data, 1) > 0)
@@ -3,13 +3,14 @@ import os
3
3
  import sys
4
4
 
5
5
  import numpy as np
6
+ import numpy.typing as npt
6
7
  from numpy import ma
7
8
 
8
9
  sys.path.append(os.path.dirname(os.path.abspath(__file__)))
9
10
 
10
11
 
11
12
  class DayStatistics:
12
- """Class for calculating statistical analysis of day scale products
13
+ """Class for calculating statistical analysis of day scale products.
13
14
 
14
15
  Class generates one statistical method at the time with given model data
15
16
  and observation data of wanted product.
@@ -20,8 +21,8 @@ class DayStatistics:
20
21
  done with. A list includes observed product name (str), model variable (str)
21
22
  name and a name of observation variable (str). Example: ['cf', 'ECMWF',
22
23
  'Cloud fraction by volume']
23
- model (np.ndarray): Ndarray of model simulation of product
24
- observation (np.ndarray): Ndrray of Downsampled observation of product
24
+ model (npt.NDArray): Ndarray of model simulation of product
25
+ observation (npt.NDArray): Ndrray of Downsampled observation of product
25
26
 
26
27
  Raises:
27
28
  RuntimeError: A function of given method not found
@@ -44,9 +45,9 @@ class DayStatistics:
44
45
  self,
45
46
  method: str,
46
47
  product_info: list,
47
- model: np.ndarray,
48
- observation: np.ndarray,
49
- ):
48
+ model: npt.NDArray,
49
+ observation: npt.NDArray,
50
+ ) -> None:
50
51
  self.method = method
51
52
  self.product = product_info
52
53
  self.model_data = model
@@ -72,20 +73,22 @@ class DayStatistics:
72
73
  full_name = "vertical_profile"
73
74
  return full_name, params
74
75
 
75
- def _generate_day_statistics(self):
76
+ def _generate_day_statistics(self) -> None:
76
77
  full_name, params = self._get_method_attr()
77
78
  cls = __import__("statistical_methods")
78
79
  try:
79
80
  self.model_stat, self.observation_stat = getattr(cls, f"{full_name}")(
80
- *params
81
+ *params,
81
82
  )
82
- self.title = getattr(cls, "day_stat_title")(self.method, self.product)
83
- except RuntimeError as error:
84
- logging.error(f"Method {full_name} not found or missing: {error}")
83
+ self.title = cls.day_stat_title(self.method, self.product)
84
+ except RuntimeError:
85
+ msg = f"Failed to calculate {self.method} of {self.product[0]}"
86
+ logging.exception(msg)
85
87
 
86
88
 
87
89
  def relative_error(
88
- model: ma.MaskedArray, observation: ma.MaskedArray
90
+ model: ma.MaskedArray,
91
+ observation: ma.MaskedArray,
89
92
  ) -> tuple[float, str]:
90
93
  model, observation = combine_masked_indices(model, observation)
91
94
  error = ((model - observation) / observation) * 100
@@ -93,7 +96,8 @@ def relative_error(
93
96
 
94
97
 
95
98
  def absolute_error(
96
- model: ma.MaskedArray, observation: ma.MaskedArray
99
+ model: ma.MaskedArray,
100
+ observation: ma.MaskedArray,
97
101
  ) -> tuple[float, str]:
98
102
  model, observation = combine_masked_indices(model, observation)
99
103
  error = (observation - model) * 100
@@ -101,9 +105,10 @@ def absolute_error(
101
105
 
102
106
 
103
107
  def combine_masked_indices(
104
- model: ma.MaskedArray, observation: ma.MaskedArray
108
+ model: ma.MaskedArray,
109
+ observation: ma.MaskedArray,
105
110
  ) -> tuple[ma.MaskedArray, ma.MaskedArray]:
106
- """Connects two array masked indices to one and add in two array same mask"""
111
+ """Connects two array masked indices to one and add in two array same mask."""
107
112
  observation[np.where(np.isnan(observation))] = ma.masked
108
113
  model[model < np.min(observation)] = ma.masked
109
114
  combine_mask = model.mask + observation.mask
@@ -113,9 +118,10 @@ def combine_masked_indices(
113
118
 
114
119
 
115
120
  def calc_common_area_sum(
116
- model: ma.MaskedArray, observation: ma.MaskedArray
121
+ model: ma.MaskedArray,
122
+ observation: ma.MaskedArray,
117
123
  ) -> tuple[float, str]:
118
- def _indices_of_mask_sum():
124
+ def _indices_of_mask_sum() -> float:
119
125
  # Calculate percentage value of common area of indices from two arrays.
120
126
  # Results is total number of common indices with value
121
127
  observation[np.where(np.isnan(observation))] = ma.masked
@@ -131,17 +137,21 @@ def calc_common_area_sum(
131
137
 
132
138
 
133
139
  def histogram(
134
- product: list, model: ma.MaskedArray, observation: ma.MaskedArray
140
+ product: list,
141
+ model: ma.MaskedArray,
142
+ observation: ma.MaskedArray,
135
143
  ) -> tuple:
136
144
  if "cf" in product:
137
145
  model = ma.round(model[~model.mask].data, decimals=1).flatten()
138
146
  observation = ma.round(
139
- observation[~observation.mask].data, decimals=1
147
+ observation[~observation.mask].data,
148
+ decimals=1,
140
149
  ).flatten()
141
150
  else:
142
151
  model = ma.round(model[~model.mask].data, decimals=6).flatten()
143
152
  observation = ma.round(
144
- observation[~observation.mask].data, decimals=6
153
+ observation[~observation.mask].data,
154
+ decimals=6,
145
155
  ).flatten()
146
156
  observation = observation[~np.isnan(observation)]
147
157
  hist_bins = np.histogram(observation, density=True)[-1]
@@ -3,11 +3,11 @@ import pytest
3
3
  args = ["site", "date", "input", "output", "full_path"]
4
4
 
5
5
 
6
- def pytest_addoption(parser):
6
+ def pytest_addoption(parser) -> None:
7
7
  for arg in args:
8
8
  parser.addoption(f"--{arg}", action="store")
9
9
 
10
10
 
11
- @pytest.fixture
12
- def params(request):
11
+ @pytest.fixture()
12
+ def params(request) -> dict:
13
13
  return {arg: request.config.getoption(f"--{arg}") for arg in args}