cloudnetpy 1.49.9__py3-none-any.whl → 1.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -2
- cloudnetpy/categorize/atmos_utils.py +297 -67
- cloudnetpy/categorize/attenuation.py +31 -0
- cloudnetpy/categorize/attenuations/__init__.py +37 -0
- cloudnetpy/categorize/attenuations/gas_attenuation.py +30 -0
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +84 -0
- cloudnetpy/categorize/attenuations/melting_attenuation.py +78 -0
- cloudnetpy/categorize/attenuations/rain_attenuation.py +84 -0
- cloudnetpy/categorize/categorize.py +332 -156
- cloudnetpy/categorize/classify.py +127 -125
- cloudnetpy/categorize/containers.py +107 -76
- cloudnetpy/categorize/disdrometer.py +40 -0
- cloudnetpy/categorize/droplet.py +23 -21
- cloudnetpy/categorize/falling.py +53 -24
- cloudnetpy/categorize/freezing.py +25 -12
- cloudnetpy/categorize/insects.py +35 -23
- cloudnetpy/categorize/itu.py +243 -0
- cloudnetpy/categorize/lidar.py +36 -41
- cloudnetpy/categorize/melting.py +34 -26
- cloudnetpy/categorize/model.py +84 -37
- cloudnetpy/categorize/mwr.py +18 -14
- cloudnetpy/categorize/radar.py +215 -102
- cloudnetpy/cli.py +578 -0
- cloudnetpy/cloudnetarray.py +43 -89
- cloudnetpy/concat_lib.py +218 -78
- cloudnetpy/constants.py +28 -10
- cloudnetpy/datasource.py +61 -86
- cloudnetpy/exceptions.py +49 -20
- cloudnetpy/instruments/__init__.py +5 -0
- cloudnetpy/instruments/basta.py +29 -12
- cloudnetpy/instruments/bowtie.py +135 -0
- cloudnetpy/instruments/ceilo.py +138 -115
- cloudnetpy/instruments/ceilometer.py +164 -80
- cloudnetpy/instruments/cl61d.py +21 -5
- cloudnetpy/instruments/cloudnet_instrument.py +74 -36
- cloudnetpy/instruments/copernicus.py +108 -30
- cloudnetpy/instruments/da10.py +54 -0
- cloudnetpy/instruments/disdrometer/common.py +126 -223
- cloudnetpy/instruments/disdrometer/parsivel.py +453 -94
- cloudnetpy/instruments/disdrometer/thies.py +254 -87
- cloudnetpy/instruments/fd12p.py +201 -0
- cloudnetpy/instruments/galileo.py +65 -23
- cloudnetpy/instruments/hatpro.py +123 -49
- cloudnetpy/instruments/instruments.py +113 -1
- cloudnetpy/instruments/lufft.py +39 -17
- cloudnetpy/instruments/mira.py +268 -61
- cloudnetpy/instruments/mrr.py +187 -0
- cloudnetpy/instruments/nc_lidar.py +19 -8
- cloudnetpy/instruments/nc_radar.py +109 -55
- cloudnetpy/instruments/pollyxt.py +135 -51
- cloudnetpy/instruments/radiometrics.py +313 -59
- cloudnetpy/instruments/rain_e_h3.py +171 -0
- cloudnetpy/instruments/rpg.py +321 -189
- cloudnetpy/instruments/rpg_reader.py +74 -40
- cloudnetpy/instruments/toa5.py +49 -0
- cloudnetpy/instruments/vaisala.py +95 -343
- cloudnetpy/instruments/weather_station.py +774 -105
- cloudnetpy/metadata.py +90 -19
- cloudnetpy/model_evaluation/file_handler.py +55 -52
- cloudnetpy/model_evaluation/metadata.py +46 -20
- cloudnetpy/model_evaluation/model_metadata.py +1 -1
- cloudnetpy/model_evaluation/plotting/plot_tools.py +32 -37
- cloudnetpy/model_evaluation/plotting/plotting.py +327 -117
- cloudnetpy/model_evaluation/products/advance_methods.py +92 -83
- cloudnetpy/model_evaluation/products/grid_methods.py +88 -63
- cloudnetpy/model_evaluation/products/model_products.py +43 -35
- cloudnetpy/model_evaluation/products/observation_products.py +41 -35
- cloudnetpy/model_evaluation/products/product_resampling.py +17 -7
- cloudnetpy/model_evaluation/products/tools.py +29 -20
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +30 -20
- cloudnetpy/model_evaluation/tests/e2e/conftest.py +3 -3
- cloudnetpy/model_evaluation/tests/e2e/process_cf/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_cf/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_iwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/main.py +9 -5
- cloudnetpy/model_evaluation/tests/e2e/process_lwc/tests.py +15 -14
- cloudnetpy/model_evaluation/tests/unit/conftest.py +42 -41
- cloudnetpy/model_evaluation/tests/unit/test_advance_methods.py +41 -48
- cloudnetpy/model_evaluation/tests/unit/test_grid_methods.py +216 -194
- cloudnetpy/model_evaluation/tests/unit/test_model_products.py +23 -21
- cloudnetpy/model_evaluation/tests/unit/test_observation_products.py +37 -38
- cloudnetpy/model_evaluation/tests/unit/test_plot_tools.py +43 -40
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +30 -36
- cloudnetpy/model_evaluation/tests/unit/test_statistical_methods.py +68 -31
- cloudnetpy/model_evaluation/tests/unit/test_tools.py +33 -26
- cloudnetpy/model_evaluation/utils.py +2 -1
- cloudnetpy/output.py +170 -111
- cloudnetpy/plotting/__init__.py +2 -1
- cloudnetpy/plotting/plot_meta.py +562 -822
- cloudnetpy/plotting/plotting.py +1142 -704
- cloudnetpy/products/__init__.py +1 -0
- cloudnetpy/products/classification.py +370 -88
- cloudnetpy/products/der.py +85 -55
- cloudnetpy/products/drizzle.py +77 -34
- cloudnetpy/products/drizzle_error.py +15 -11
- cloudnetpy/products/drizzle_tools.py +79 -59
- cloudnetpy/products/epsilon.py +211 -0
- cloudnetpy/products/ier.py +27 -50
- cloudnetpy/products/iwc.py +55 -48
- cloudnetpy/products/lwc.py +96 -70
- cloudnetpy/products/mwr_tools.py +186 -0
- cloudnetpy/products/product_tools.py +170 -128
- cloudnetpy/utils.py +455 -240
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/METADATA +44 -40
- cloudnetpy-1.87.3.dist-info/RECORD +127 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/WHEEL +1 -1
- cloudnetpy-1.87.3.dist-info/entry_points.txt +2 -0
- docs/source/conf.py +2 -2
- cloudnetpy/categorize/atmos.py +0 -361
- cloudnetpy/products/mwr_multi.py +0 -68
- cloudnetpy/products/mwr_single.py +0 -75
- cloudnetpy-1.49.9.dist-info/RECORD +0 -112
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info/licenses}/LICENSE +0 -0
- {cloudnetpy-1.49.9.dist-info → cloudnetpy-1.87.3.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import importlib
|
|
2
2
|
import logging
|
|
3
|
+
import os.path
|
|
4
|
+
from os import PathLike
|
|
3
5
|
|
|
4
6
|
import numpy as np
|
|
7
|
+
import numpy.typing as npt
|
|
5
8
|
from numpy import ma
|
|
6
9
|
|
|
7
10
|
from cloudnetpy.datasource import DataSource
|
|
@@ -32,12 +35,13 @@ class ModelManager(DataSource):
|
|
|
32
35
|
|
|
33
36
|
def __init__(
|
|
34
37
|
self,
|
|
35
|
-
model_file: str,
|
|
38
|
+
model_file: str | PathLike,
|
|
36
39
|
model: str,
|
|
37
|
-
output_file: str,
|
|
40
|
+
output_file: str | PathLike,
|
|
38
41
|
product: str,
|
|
42
|
+
*,
|
|
39
43
|
check_file: bool = True,
|
|
40
|
-
):
|
|
44
|
+
) -> None:
|
|
41
45
|
super().__init__(model_file)
|
|
42
46
|
self.model = model
|
|
43
47
|
self.model_info = MODELS[model]
|
|
@@ -52,31 +56,33 @@ class ModelManager(DataSource):
|
|
|
52
56
|
self.wind = self._calculate_wind_speed()
|
|
53
57
|
self.resolution_h = self._get_horizontal_resolution()
|
|
54
58
|
|
|
55
|
-
def _read_cycle_name(self, model_file: str):
|
|
56
|
-
"""Get cycle name from model_metadata.py for saving variable name(s)"""
|
|
59
|
+
def _read_cycle_name(self, model_file: str | PathLike) -> str:
|
|
60
|
+
"""Get cycle name from model_metadata.py for saving variable name(s)."""
|
|
61
|
+
basename = os.path.basename(model_file)
|
|
57
62
|
try:
|
|
58
63
|
cycles = self.model_info.cycle
|
|
59
64
|
if cycles is None:
|
|
60
65
|
return ""
|
|
61
66
|
cycles_split = [x.strip() for x in cycles.split(",")]
|
|
62
67
|
for cycle in cycles_split:
|
|
63
|
-
if cycle in
|
|
68
|
+
if cycle in basename:
|
|
64
69
|
return f"_{cycle}"
|
|
65
70
|
except AttributeError:
|
|
66
71
|
return ""
|
|
67
72
|
return ""
|
|
68
73
|
|
|
69
|
-
def _generate_products(self):
|
|
70
|
-
"""Process needed data of model to a ModelManager object"""
|
|
71
|
-
cls =
|
|
74
|
+
def _generate_products(self) -> None:
|
|
75
|
+
"""Process needed data of model to a ModelManager object."""
|
|
76
|
+
cls = importlib.import_module(__name__).ModelManager
|
|
72
77
|
try:
|
|
73
78
|
name = f"_get_{self._product}"
|
|
74
79
|
getattr(cls, name)(self)
|
|
75
|
-
except AttributeError
|
|
76
|
-
|
|
80
|
+
except AttributeError:
|
|
81
|
+
msg = f"Invalid product name: {self._product}"
|
|
82
|
+
logging.exception(msg)
|
|
77
83
|
raise
|
|
78
84
|
|
|
79
|
-
def _get_cf(self):
|
|
85
|
+
def _get_cf(self) -> None:
|
|
80
86
|
"""Collect cloud fraction straight from model file."""
|
|
81
87
|
cf_name = self.get_model_var_names(("cf",))[0]
|
|
82
88
|
cf = self.getvar(cf_name)
|
|
@@ -85,14 +91,14 @@ class ModelManager(DataSource):
|
|
|
85
91
|
self.append_data(cf, f"{self.model}{self.cycle}_cf")
|
|
86
92
|
self.keys[self._product] = f"{self.model}{self.cycle}_cf"
|
|
87
93
|
|
|
88
|
-
def _get_iwc(self):
|
|
89
|
-
iwc = self.
|
|
94
|
+
def _get_iwc(self) -> None:
|
|
95
|
+
iwc = self.get_water_content("iwc")
|
|
90
96
|
iwc[iwc < 1e-7] = ma.masked
|
|
91
97
|
self.append_data(iwc, f"{self.model}{self.cycle}_iwc")
|
|
92
98
|
self.keys[self._product] = f"{self.model}{self.cycle}_iwc"
|
|
93
99
|
|
|
94
|
-
def _get_lwc(self):
|
|
95
|
-
lwc = self.
|
|
100
|
+
def _get_lwc(self) -> None:
|
|
101
|
+
lwc = self.get_water_content("lwc")
|
|
96
102
|
lwc[lwc < 1e-5] = ma.masked
|
|
97
103
|
self.append_data(lwc, f"{self.model}{self.cycle}_lwc")
|
|
98
104
|
self.keys[self._product] = f"{self.model}{self.cycle}_lwc"
|
|
@@ -104,7 +110,7 @@ class ModelManager(DataSource):
|
|
|
104
110
|
var.append(VARIABLES[arg].long_name)
|
|
105
111
|
return var
|
|
106
112
|
|
|
107
|
-
def
|
|
113
|
+
def get_water_content(self, var: str) -> npt.NDArray:
|
|
108
114
|
p_name = self.get_model_var_names(("p",))[0]
|
|
109
115
|
t_name = self.get_model_var_names(("T",))[0]
|
|
110
116
|
lwc_name = self.get_model_var_names((var,))[0]
|
|
@@ -117,16 +123,20 @@ class ModelManager(DataSource):
|
|
|
117
123
|
return wc
|
|
118
124
|
|
|
119
125
|
@staticmethod
|
|
120
|
-
def _calc_water_content(
|
|
126
|
+
def _calc_water_content(
|
|
127
|
+
q: npt.NDArray, p: npt.NDArray, t: npt.NDArray
|
|
128
|
+
) -> npt.NDArray:
|
|
121
129
|
return q * p / (287 * t)
|
|
122
130
|
|
|
123
|
-
def _add_variables(self):
|
|
124
|
-
"""Add basic variables off model and cycle"""
|
|
131
|
+
def _add_variables(self) -> None:
|
|
132
|
+
"""Add basic variables off model and cycle."""
|
|
125
133
|
|
|
126
|
-
def _add_common_variables():
|
|
127
|
-
"""Model variables that are always the same within cycles"""
|
|
134
|
+
def _add_common_variables() -> None:
|
|
135
|
+
"""Model variables that are always the same within cycles."""
|
|
128
136
|
wanted_vars = self.model_vars.common_var
|
|
129
|
-
|
|
137
|
+
if wanted_vars is None:
|
|
138
|
+
msg = f"Model {self.model} has no common variables"
|
|
139
|
+
raise ValueError(msg)
|
|
130
140
|
wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
|
|
131
141
|
for var in wanted_vars_split:
|
|
132
142
|
if var in self.dataset.variables:
|
|
@@ -135,10 +145,12 @@ class ModelManager(DataSource):
|
|
|
135
145
|
data = self.cut_off_extra_levels(self.dataset.variables[var][:])
|
|
136
146
|
self.append_data(data, f"{var}")
|
|
137
147
|
|
|
138
|
-
def _add_cycle_variables():
|
|
139
|
-
"""Add cycle depending variables"""
|
|
148
|
+
def _add_cycle_variables() -> None:
|
|
149
|
+
"""Add cycle depending variables."""
|
|
140
150
|
wanted_vars = self.model_vars.cycle_var
|
|
141
|
-
|
|
151
|
+
if wanted_vars is None:
|
|
152
|
+
msg = f"Model {self.model} has no cycle variables"
|
|
153
|
+
raise ValueError(msg)
|
|
142
154
|
wanted_vars_split = [x.strip() for x in wanted_vars.split(",")]
|
|
143
155
|
for var in wanted_vars_split:
|
|
144
156
|
if var in self.dataset.variables:
|
|
@@ -153,21 +165,17 @@ class ModelManager(DataSource):
|
|
|
153
165
|
_add_common_variables()
|
|
154
166
|
_add_cycle_variables()
|
|
155
167
|
|
|
156
|
-
def cut_off_extra_levels(self, data:
|
|
157
|
-
"""Remove unused levels (over 22km) from model data"""
|
|
168
|
+
def cut_off_extra_levels(self, data: npt.NDArray) -> npt.NDArray:
|
|
169
|
+
"""Remove unused levels (over 22km) from model data."""
|
|
158
170
|
try:
|
|
159
171
|
level = self.model_info.level
|
|
160
172
|
except KeyError:
|
|
161
173
|
return data
|
|
162
174
|
|
|
163
|
-
if data.ndim > 1:
|
|
164
|
-
data = data[:, :level]
|
|
165
|
-
else:
|
|
166
|
-
data = data[:level]
|
|
167
|
-
return data
|
|
175
|
+
return data[:, :level] if data.ndim > 1 else data[:level]
|
|
168
176
|
|
|
169
|
-
def _calculate_wind_speed(self) ->
|
|
170
|
-
"""Real wind from x- and y-components"""
|
|
177
|
+
def _calculate_wind_speed(self) -> npt.NDArray:
|
|
178
|
+
"""Real wind from x- and y-components."""
|
|
171
179
|
u = self.getvar("uwind")
|
|
172
180
|
v = self.getvar("vwind")
|
|
173
181
|
u = self.cut_off_extra_levels(u)
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from datetime import datetime
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from os import PathLike
|
|
3
4
|
|
|
4
5
|
import numpy as np
|
|
6
|
+
import numpy.typing as npt
|
|
5
7
|
from numpy import ma
|
|
6
8
|
|
|
7
9
|
from cloudnetpy import utils
|
|
8
10
|
from cloudnetpy.datasource import DataSource
|
|
9
|
-
from cloudnetpy.products.product_tools import CategorizeBits
|
|
11
|
+
from cloudnetpy.products.product_tools import CategorizeBits, CategoryBits
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
class ObservationManager(DataSource):
|
|
@@ -24,7 +26,7 @@ class ObservationManager(DataSource):
|
|
|
24
26
|
should be processed using CloudnetPy for this class to work properly.
|
|
25
27
|
"""
|
|
26
28
|
|
|
27
|
-
def __init__(self, obs: str, obs_file: str):
|
|
29
|
+
def __init__(self, obs: str, obs_file: str | PathLike) -> None:
|
|
28
30
|
super().__init__(obs_file)
|
|
29
31
|
self.obs = obs
|
|
30
32
|
self._file = obs_file
|
|
@@ -42,22 +44,23 @@ class ObservationManager(DataSource):
|
|
|
42
44
|
0,
|
|
43
45
|
0,
|
|
44
46
|
0,
|
|
47
|
+
tzinfo=timezone.utc,
|
|
45
48
|
)
|
|
46
49
|
|
|
47
|
-
def _get_radar_frequency(self) ->
|
|
50
|
+
def _get_radar_frequency(self) -> npt.NDArray | None:
|
|
48
51
|
try:
|
|
49
52
|
return self.getvar("radar_frequency")
|
|
50
53
|
except (KeyError, RuntimeError):
|
|
51
54
|
return None
|
|
52
55
|
|
|
53
|
-
def _get_z_sensitivity(self) ->
|
|
56
|
+
def _get_z_sensitivity(self) -> npt.NDArray | None:
|
|
54
57
|
try:
|
|
55
58
|
return self.getvar("Z_sensitivity")
|
|
56
59
|
except (KeyError, RuntimeError):
|
|
57
60
|
return None
|
|
58
61
|
|
|
59
|
-
def _generate_product(self):
|
|
60
|
-
"""Process needed data of observation to a ObservationManager object"""
|
|
62
|
+
def _generate_product(self) -> None:
|
|
63
|
+
"""Process needed data of observation to a ObservationManager object."""
|
|
61
64
|
try:
|
|
62
65
|
if self.obs == "cf":
|
|
63
66
|
self.append_data(self._generate_cf(), "cf")
|
|
@@ -66,31 +69,31 @@ class ObservationManager(DataSource):
|
|
|
66
69
|
if self.obs == "iwc":
|
|
67
70
|
self._generate_iwc_masks()
|
|
68
71
|
self.append_data(self.getvar("height"), "height")
|
|
69
|
-
except (KeyError, RuntimeError)
|
|
70
|
-
|
|
72
|
+
except (KeyError, RuntimeError):
|
|
73
|
+
msg = f"Failed to read {self.obs} from {self._file}"
|
|
74
|
+
logging.exception(msg)
|
|
71
75
|
raise
|
|
72
76
|
|
|
73
|
-
def _generate_cf(self) ->
|
|
74
|
-
"""Generates cloud fractions using categorize bits and masking conditions"""
|
|
77
|
+
def _generate_cf(self) -> npt.NDArray:
|
|
78
|
+
"""Generates cloud fractions using categorize bits and masking conditions."""
|
|
75
79
|
categorize_bits = CategorizeBits(self._file)
|
|
76
80
|
cloud_mask = self._classify_basic_mask(categorize_bits.category_bits)
|
|
77
|
-
|
|
78
|
-
return cloud_mask
|
|
81
|
+
return self._mask_cloud_bits(cloud_mask)
|
|
79
82
|
|
|
80
83
|
@staticmethod
|
|
81
|
-
def _classify_basic_mask(bits:
|
|
82
|
-
cloud_mask = bits
|
|
83
|
-
cloud_mask[bits
|
|
84
|
-
cloud_mask[bits
|
|
84
|
+
def _classify_basic_mask(bits: CategoryBits) -> npt.NDArray:
|
|
85
|
+
cloud_mask = bits.droplet + bits.falling * 2
|
|
86
|
+
cloud_mask[bits.falling & bits.freezing] = (
|
|
87
|
+
cloud_mask[bits.falling & bits.freezing] + 2
|
|
85
88
|
)
|
|
86
|
-
cloud_mask[bits
|
|
87
|
-
cloud_mask[bits
|
|
88
|
-
cloud_mask[bits
|
|
89
|
+
cloud_mask[bits.aerosol] = 6
|
|
90
|
+
cloud_mask[bits.insect] = 7
|
|
91
|
+
cloud_mask[bits.aerosol & bits.insect] = 8
|
|
89
92
|
return cloud_mask
|
|
90
93
|
|
|
91
94
|
@staticmethod
|
|
92
|
-
def _mask_cloud_bits(cloud_mask:
|
|
93
|
-
"""Creates cloud fraction"""
|
|
95
|
+
def _mask_cloud_bits(cloud_mask: npt.NDArray) -> npt.NDArray:
|
|
96
|
+
"""Creates cloud fraction."""
|
|
94
97
|
for i in [1, 3, 4, 5]:
|
|
95
98
|
cloud_mask[cloud_mask == i] = 1
|
|
96
99
|
for i in [2, 6, 7, 8]:
|
|
@@ -98,28 +101,30 @@ class ObservationManager(DataSource):
|
|
|
98
101
|
return cloud_mask
|
|
99
102
|
|
|
100
103
|
def _check_rainrate(self) -> bool:
|
|
101
|
-
"""Check if rainrate in file"""
|
|
104
|
+
"""Check if rainrate in file."""
|
|
102
105
|
try:
|
|
103
106
|
self.getvar("rainrate")
|
|
104
|
-
|
|
105
|
-
except RuntimeError:
|
|
107
|
+
except KeyError:
|
|
106
108
|
return False
|
|
109
|
+
return True
|
|
107
110
|
|
|
108
111
|
def _get_rainrate_threshold(self) -> int:
|
|
109
|
-
|
|
112
|
+
if self.radar_freq is None:
|
|
113
|
+
msg = "Radar frequency not found from file"
|
|
114
|
+
raise RuntimeError(msg)
|
|
110
115
|
wband = utils.get_wl_band(float(self.radar_freq))
|
|
111
116
|
rainrate_threshold = 8
|
|
112
|
-
if
|
|
117
|
+
if wband == "W":
|
|
113
118
|
rainrate_threshold = 2
|
|
114
119
|
return rainrate_threshold
|
|
115
120
|
|
|
116
|
-
def _rain_index(self) ->
|
|
121
|
+
def _rain_index(self) -> npt.NDArray:
|
|
117
122
|
rainrate = self.getvar("rainrate")
|
|
118
123
|
rainrate_threshold = self._get_rainrate_threshold()
|
|
119
124
|
return rainrate > rainrate_threshold
|
|
120
125
|
|
|
121
126
|
def _generate_iwc_masks(self) -> None:
|
|
122
|
-
"""Generates ice water content variables with different masks"""
|
|
127
|
+
"""Generates ice water content variables with different masks."""
|
|
123
128
|
# TODO: Differences with CloudnetPy (status=2) and Legacy data (status=3)
|
|
124
129
|
iwc = self.getvar(self.obs)
|
|
125
130
|
iwc_status = self.getvar("iwc_retrieval_status")
|
|
@@ -127,21 +132,22 @@ class ObservationManager(DataSource):
|
|
|
127
132
|
self._get_rain_iwc(iwc_status)
|
|
128
133
|
self._mask_iwc(iwc, iwc_status)
|
|
129
134
|
|
|
130
|
-
def _mask_iwc(self, iwc:
|
|
131
|
-
"""Leaves only reliable data and corrected liquid attenuation"""
|
|
135
|
+
def _mask_iwc(self, iwc: npt.NDArray, iwc_status: npt.NDArray) -> None:
|
|
136
|
+
"""Leaves only reliable data and corrected liquid attenuation."""
|
|
132
137
|
iwc_mask = ma.copy(iwc)
|
|
133
138
|
iwc_mask[np.bitwise_and(iwc_status != 1, iwc_status != 2)] = ma.masked
|
|
134
139
|
self.append_data(iwc_mask, "iwc")
|
|
135
140
|
|
|
136
|
-
def _mask_iwc_att(self, iwc:
|
|
141
|
+
def _mask_iwc_att(self, iwc: npt.NDArray, iwc_status: npt.NDArray) -> None:
|
|
137
142
|
"""Leaves only where reliable data, corrected liquid attenuation
|
|
138
|
-
and uncorrected liquid attenuation
|
|
143
|
+
and uncorrected liquid attenuation.
|
|
144
|
+
"""
|
|
139
145
|
iwc_att = ma.copy(iwc)
|
|
140
146
|
iwc_att[iwc_status > 3] = ma.masked
|
|
141
147
|
self.append_data(iwc_att, "iwc_att")
|
|
142
148
|
|
|
143
|
-
def _get_rain_iwc(self, iwc_status:
|
|
144
|
-
"""Finds columns where is rain, return boolean of x-axis shape"""
|
|
149
|
+
def _get_rain_iwc(self, iwc_status: npt.NDArray) -> None:
|
|
150
|
+
"""Finds columns where is rain, return boolean of x-axis shape."""
|
|
145
151
|
iwc_rain = np.zeros(iwc_status.shape, dtype=bool)
|
|
146
152
|
iwc_rain[iwc_status == 5] = 1
|
|
147
153
|
iwc_rain = np.any(iwc_rain, axis=1)
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from os import PathLike
|
|
3
|
+
from uuid import UUID
|
|
2
4
|
|
|
3
5
|
import cloudnetpy.model_evaluation.products.tools as tl
|
|
4
6
|
from cloudnetpy.model_evaluation.file_handler import (
|
|
@@ -12,22 +14,25 @@ from cloudnetpy.model_evaluation.products.grid_methods import ProductGrid
|
|
|
12
14
|
from cloudnetpy.model_evaluation.products.model_products import ModelManager
|
|
13
15
|
from cloudnetpy.model_evaluation.products.observation_products import ObservationManager
|
|
14
16
|
from cloudnetpy.model_evaluation.utils import file_exists
|
|
17
|
+
from cloudnetpy.utils import get_uuid
|
|
15
18
|
|
|
16
19
|
|
|
17
20
|
def process_L3_day_product(
|
|
18
21
|
model: str,
|
|
19
22
|
obs: str,
|
|
20
|
-
model_files: list,
|
|
21
|
-
product_file: str,
|
|
22
|
-
output_file: str,
|
|
23
|
-
uuid: str | None = None,
|
|
23
|
+
model_files: list[str | PathLike],
|
|
24
|
+
product_file: str | PathLike,
|
|
25
|
+
output_file: str | PathLike,
|
|
26
|
+
uuid: str | UUID | None = None,
|
|
27
|
+
*,
|
|
24
28
|
overwrite: bool = False,
|
|
25
|
-
):
|
|
29
|
+
) -> UUID:
|
|
26
30
|
"""Main function to generate downsample of observations to match model grid.
|
|
27
31
|
|
|
28
32
|
This function will generate a L3 product nc-file. It includes the information of
|
|
29
33
|
downsampled observation products for each model cycles and model products
|
|
30
34
|
and other variables of each cycles.
|
|
35
|
+
|
|
31
36
|
Args:
|
|
32
37
|
model (str): Name of model
|
|
33
38
|
obs (str): Name of product to generate
|
|
@@ -61,11 +66,16 @@ def process_L3_day_product(
|
|
|
61
66
|
>>> process_L3_day_product(model, product, [model_file], input_file,
|
|
62
67
|
output_file)
|
|
63
68
|
"""
|
|
69
|
+
uuid = get_uuid(uuid)
|
|
64
70
|
product_obj = ObservationManager(obs, product_file)
|
|
65
71
|
tl.check_model_file_list(model, model_files)
|
|
66
72
|
for m_file in model_files:
|
|
67
73
|
model_obj = ModelManager(
|
|
68
|
-
m_file,
|
|
74
|
+
m_file,
|
|
75
|
+
model,
|
|
76
|
+
output_file,
|
|
77
|
+
obs,
|
|
78
|
+
check_file=not overwrite,
|
|
69
79
|
)
|
|
70
80
|
try:
|
|
71
81
|
AdvanceProductMethods(model_obj, m_file, product_obj)
|
|
@@ -76,7 +86,7 @@ def process_L3_day_product(
|
|
|
76
86
|
update_attributes(model_obj.data, attributes)
|
|
77
87
|
if not file_exists(output_file) or overwrite:
|
|
78
88
|
tl.add_date(model_obj, product_obj)
|
|
79
|
-
|
|
89
|
+
save_downsampled_file(
|
|
80
90
|
f"{obs}_{model}",
|
|
81
91
|
output_file,
|
|
82
92
|
(model_obj, product_obj),
|
|
@@ -1,27 +1,32 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import logging
|
|
3
|
+
import os.path
|
|
4
|
+
from collections.abc import Sequence
|
|
3
5
|
from datetime import timedelta
|
|
6
|
+
from os import PathLike
|
|
4
7
|
|
|
5
8
|
import numpy as np
|
|
9
|
+
import numpy.typing as npt
|
|
6
10
|
from numpy import ma
|
|
7
11
|
|
|
8
12
|
from cloudnetpy.model_evaluation.products.model_products import ModelManager
|
|
9
13
|
from cloudnetpy.model_evaluation.products.observation_products import ObservationManager
|
|
10
14
|
|
|
11
15
|
|
|
12
|
-
def check_model_file_list(name: str, models:
|
|
13
|
-
"""Check that files in models are from same model and date"""
|
|
16
|
+
def check_model_file_list(name: str, models: Sequence[str | PathLike]) -> None:
|
|
17
|
+
"""Check that files in models are from same model and date."""
|
|
14
18
|
for m in models:
|
|
15
|
-
if name not in m:
|
|
19
|
+
if name not in os.path.basename(m):
|
|
16
20
|
logging.error("Invalid model file set")
|
|
17
|
-
|
|
21
|
+
msg = f"{m} not from {name}"
|
|
22
|
+
raise AttributeError(msg)
|
|
18
23
|
|
|
19
24
|
|
|
20
|
-
def time2datetime(time:
|
|
25
|
+
def time2datetime(time: npt.NDArray, date: datetime.datetime) -> npt.NDArray:
|
|
21
26
|
return np.asarray([date + timedelta(hours=float(t)) for t in time])
|
|
22
27
|
|
|
23
28
|
|
|
24
|
-
def rebin_edges(arr:
|
|
29
|
+
def rebin_edges(arr: npt.NDArray) -> npt.NDArray:
|
|
25
30
|
"""Rebins array bins by half and adds boundaries."""
|
|
26
31
|
new_arr = [(arr[i] + arr[i + 1]) / 2 for i in range(len(arr) - 1)]
|
|
27
32
|
new_arr.insert(0, arr[0] - ((arr[0] + arr[1]) / 2))
|
|
@@ -30,9 +35,11 @@ def rebin_edges(arr: np.ndarray) -> np.ndarray:
|
|
|
30
35
|
|
|
31
36
|
|
|
32
37
|
def calculate_advection_time(
|
|
33
|
-
resolution: int,
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
resolution: int,
|
|
39
|
+
wind: ma.MaskedArray,
|
|
40
|
+
sampling: int,
|
|
41
|
+
) -> npt.NDArray:
|
|
42
|
+
"""Calculates time which variable takes to go through the time window.
|
|
36
43
|
|
|
37
44
|
Notes:
|
|
38
45
|
Wind speed is stronger in upper levels, so advection time is more
|
|
@@ -40,8 +47,6 @@ def calculate_advection_time(
|
|
|
40
47
|
but visible in a tropics.
|
|
41
48
|
|
|
42
49
|
sampling = 1 -> hour, sampling 1/6 -> 10min
|
|
43
|
-
|
|
44
|
-
References:
|
|
45
50
|
"""
|
|
46
51
|
t_adv = resolution * 1000 / wind / 60**2
|
|
47
52
|
t_adv[t_adv.mask] = 0
|
|
@@ -49,8 +54,12 @@ def calculate_advection_time(
|
|
|
49
54
|
return np.asarray([[timedelta(hours=float(t)) for t in time] for time in t_adv])
|
|
50
55
|
|
|
51
56
|
|
|
52
|
-
def get_1d_indices(
|
|
53
|
-
|
|
57
|
+
def get_1d_indices(
|
|
58
|
+
window: tuple,
|
|
59
|
+
data: npt.NDArray,
|
|
60
|
+
mask: npt.NDArray | None = None,
|
|
61
|
+
) -> npt.NDArray:
|
|
62
|
+
indices: npt.NDArray = np.array((window[0] <= data) & (data < window[-1]))
|
|
54
63
|
if mask is not None:
|
|
55
64
|
indices[mask] = ma.masked
|
|
56
65
|
return indices
|
|
@@ -59,17 +68,17 @@ def get_1d_indices(window: tuple, data: np.ndarray, mask: np.ndarray | None = No
|
|
|
59
68
|
def get_adv_indices(
|
|
60
69
|
model_t: int,
|
|
61
70
|
adv_t: float,
|
|
62
|
-
data:
|
|
63
|
-
mask:
|
|
64
|
-
):
|
|
71
|
+
data: npt.NDArray,
|
|
72
|
+
mask: npt.NDArray | None = None,
|
|
73
|
+
) -> npt.NDArray:
|
|
65
74
|
adv_indices = ((model_t - adv_t / 2) <= data) & (data < (model_t + adv_t / 2))
|
|
66
75
|
if mask is not None:
|
|
67
76
|
adv_indices[mask] = ma.masked
|
|
68
77
|
return adv_indices
|
|
69
78
|
|
|
70
79
|
|
|
71
|
-
def get_obs_window_size(ind_x:
|
|
72
|
-
"""Returns shape (tuple) of window area, where values are True"""
|
|
80
|
+
def get_obs_window_size(ind_x: npt.NDArray, ind_y: npt.NDArray) -> tuple | None:
|
|
81
|
+
"""Returns shape (tuple) of window area, where values are True."""
|
|
73
82
|
x = np.where(ind_x)[0]
|
|
74
83
|
y = np.where(ind_y)[0]
|
|
75
84
|
if np.any(x) and np.any(y):
|
|
@@ -82,6 +91,6 @@ def add_date(model_obj: ModelManager, obs_obj: ObservationManager) -> None:
|
|
|
82
91
|
model_obj.date.append(getattr(obs_obj.dataset, a))
|
|
83
92
|
|
|
84
93
|
|
|
85
|
-
def average_column_sum(data:
|
|
86
|
-
"""Returns average sum of columns which have any data"""
|
|
94
|
+
def average_column_sum(data: npt.NDArray) -> npt.NDArray:
|
|
95
|
+
"""Returns average sum of columns which have any data."""
|
|
87
96
|
return np.nanmean(np.nansum(data, 1) > 0)
|
|
@@ -3,13 +3,14 @@ import os
|
|
|
3
3
|
import sys
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
|
+
import numpy.typing as npt
|
|
6
7
|
from numpy import ma
|
|
7
8
|
|
|
8
9
|
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
class DayStatistics:
|
|
12
|
-
"""Class for calculating statistical analysis of day scale products
|
|
13
|
+
"""Class for calculating statistical analysis of day scale products.
|
|
13
14
|
|
|
14
15
|
Class generates one statistical method at the time with given model data
|
|
15
16
|
and observation data of wanted product.
|
|
@@ -20,8 +21,8 @@ class DayStatistics:
|
|
|
20
21
|
done with. A list includes observed product name (str), model variable (str)
|
|
21
22
|
name and a name of observation variable (str). Example: ['cf', 'ECMWF',
|
|
22
23
|
'Cloud fraction by volume']
|
|
23
|
-
model (
|
|
24
|
-
observation (
|
|
24
|
+
model (npt.NDArray): Ndarray of model simulation of product
|
|
25
|
+
observation (npt.NDArray): Ndrray of Downsampled observation of product
|
|
25
26
|
|
|
26
27
|
Raises:
|
|
27
28
|
RuntimeError: A function of given method not found
|
|
@@ -44,9 +45,9 @@ class DayStatistics:
|
|
|
44
45
|
self,
|
|
45
46
|
method: str,
|
|
46
47
|
product_info: list,
|
|
47
|
-
model:
|
|
48
|
-
observation:
|
|
49
|
-
):
|
|
48
|
+
model: npt.NDArray,
|
|
49
|
+
observation: npt.NDArray,
|
|
50
|
+
) -> None:
|
|
50
51
|
self.method = method
|
|
51
52
|
self.product = product_info
|
|
52
53
|
self.model_data = model
|
|
@@ -72,20 +73,22 @@ class DayStatistics:
|
|
|
72
73
|
full_name = "vertical_profile"
|
|
73
74
|
return full_name, params
|
|
74
75
|
|
|
75
|
-
def _generate_day_statistics(self):
|
|
76
|
+
def _generate_day_statistics(self) -> None:
|
|
76
77
|
full_name, params = self._get_method_attr()
|
|
77
78
|
cls = __import__("statistical_methods")
|
|
78
79
|
try:
|
|
79
80
|
self.model_stat, self.observation_stat = getattr(cls, f"{full_name}")(
|
|
80
|
-
*params
|
|
81
|
+
*params,
|
|
81
82
|
)
|
|
82
|
-
self.title =
|
|
83
|
-
except RuntimeError
|
|
84
|
-
|
|
83
|
+
self.title = cls.day_stat_title(self.method, self.product)
|
|
84
|
+
except RuntimeError:
|
|
85
|
+
msg = f"Failed to calculate {self.method} of {self.product[0]}"
|
|
86
|
+
logging.exception(msg)
|
|
85
87
|
|
|
86
88
|
|
|
87
89
|
def relative_error(
|
|
88
|
-
model: ma.MaskedArray,
|
|
90
|
+
model: ma.MaskedArray,
|
|
91
|
+
observation: ma.MaskedArray,
|
|
89
92
|
) -> tuple[float, str]:
|
|
90
93
|
model, observation = combine_masked_indices(model, observation)
|
|
91
94
|
error = ((model - observation) / observation) * 100
|
|
@@ -93,7 +96,8 @@ def relative_error(
|
|
|
93
96
|
|
|
94
97
|
|
|
95
98
|
def absolute_error(
|
|
96
|
-
model: ma.MaskedArray,
|
|
99
|
+
model: ma.MaskedArray,
|
|
100
|
+
observation: ma.MaskedArray,
|
|
97
101
|
) -> tuple[float, str]:
|
|
98
102
|
model, observation = combine_masked_indices(model, observation)
|
|
99
103
|
error = (observation - model) * 100
|
|
@@ -101,9 +105,10 @@ def absolute_error(
|
|
|
101
105
|
|
|
102
106
|
|
|
103
107
|
def combine_masked_indices(
|
|
104
|
-
model: ma.MaskedArray,
|
|
108
|
+
model: ma.MaskedArray,
|
|
109
|
+
observation: ma.MaskedArray,
|
|
105
110
|
) -> tuple[ma.MaskedArray, ma.MaskedArray]:
|
|
106
|
-
"""Connects two array masked indices to one and add in two array same mask"""
|
|
111
|
+
"""Connects two array masked indices to one and add in two array same mask."""
|
|
107
112
|
observation[np.where(np.isnan(observation))] = ma.masked
|
|
108
113
|
model[model < np.min(observation)] = ma.masked
|
|
109
114
|
combine_mask = model.mask + observation.mask
|
|
@@ -113,9 +118,10 @@ def combine_masked_indices(
|
|
|
113
118
|
|
|
114
119
|
|
|
115
120
|
def calc_common_area_sum(
|
|
116
|
-
model: ma.MaskedArray,
|
|
121
|
+
model: ma.MaskedArray,
|
|
122
|
+
observation: ma.MaskedArray,
|
|
117
123
|
) -> tuple[float, str]:
|
|
118
|
-
def _indices_of_mask_sum():
|
|
124
|
+
def _indices_of_mask_sum() -> float:
|
|
119
125
|
# Calculate percentage value of common area of indices from two arrays.
|
|
120
126
|
# Results is total number of common indices with value
|
|
121
127
|
observation[np.where(np.isnan(observation))] = ma.masked
|
|
@@ -131,17 +137,21 @@ def calc_common_area_sum(
|
|
|
131
137
|
|
|
132
138
|
|
|
133
139
|
def histogram(
|
|
134
|
-
product: list,
|
|
140
|
+
product: list,
|
|
141
|
+
model: ma.MaskedArray,
|
|
142
|
+
observation: ma.MaskedArray,
|
|
135
143
|
) -> tuple:
|
|
136
144
|
if "cf" in product:
|
|
137
145
|
model = ma.round(model[~model.mask].data, decimals=1).flatten()
|
|
138
146
|
observation = ma.round(
|
|
139
|
-
observation[~observation.mask].data,
|
|
147
|
+
observation[~observation.mask].data,
|
|
148
|
+
decimals=1,
|
|
140
149
|
).flatten()
|
|
141
150
|
else:
|
|
142
151
|
model = ma.round(model[~model.mask].data, decimals=6).flatten()
|
|
143
152
|
observation = ma.round(
|
|
144
|
-
observation[~observation.mask].data,
|
|
153
|
+
observation[~observation.mask].data,
|
|
154
|
+
decimals=6,
|
|
145
155
|
).flatten()
|
|
146
156
|
observation = observation[~np.isnan(observation)]
|
|
147
157
|
hist_bins = np.histogram(observation, density=True)[-1]
|
|
@@ -3,11 +3,11 @@ import pytest
|
|
|
3
3
|
args = ["site", "date", "input", "output", "full_path"]
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def pytest_addoption(parser):
|
|
6
|
+
def pytest_addoption(parser) -> None:
|
|
7
7
|
for arg in args:
|
|
8
8
|
parser.addoption(f"--{arg}", action="store")
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
@pytest.fixture
|
|
12
|
-
def params(request):
|
|
11
|
+
@pytest.fixture()
|
|
12
|
+
def params(request) -> dict:
|
|
13
13
|
return {arg: request.config.getoption(f"--{arg}") for arg in args}
|