cloudnetpy 1.80.8__py3-none-any.whl → 1.81.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudnetpy/categorize/__init__.py +1 -1
- cloudnetpy/categorize/atmos_utils.py +31 -27
- cloudnetpy/categorize/attenuations/__init__.py +4 -4
- cloudnetpy/categorize/attenuations/liquid_attenuation.py +7 -5
- cloudnetpy/categorize/attenuations/melting_attenuation.py +3 -3
- cloudnetpy/categorize/attenuations/rain_attenuation.py +4 -4
- cloudnetpy/categorize/categorize.py +25 -11
- cloudnetpy/categorize/classify.py +9 -8
- cloudnetpy/categorize/containers.py +13 -10
- cloudnetpy/categorize/disdrometer.py +5 -3
- cloudnetpy/categorize/droplet.py +12 -9
- cloudnetpy/categorize/falling.py +9 -8
- cloudnetpy/categorize/freezing.py +10 -7
- cloudnetpy/categorize/insects.py +18 -17
- cloudnetpy/categorize/lidar.py +7 -3
- cloudnetpy/categorize/melting.py +16 -15
- cloudnetpy/categorize/model.py +17 -10
- cloudnetpy/categorize/mwr.py +5 -3
- cloudnetpy/categorize/radar.py +15 -13
- cloudnetpy/cli.py +10 -8
- cloudnetpy/cloudnetarray.py +8 -7
- cloudnetpy/concat_lib.py +29 -20
- cloudnetpy/datasource.py +26 -21
- cloudnetpy/exceptions.py +12 -10
- cloudnetpy/instruments/basta.py +19 -9
- cloudnetpy/instruments/bowtie.py +18 -11
- cloudnetpy/instruments/ceilo.py +22 -10
- cloudnetpy/instruments/ceilometer.py +33 -34
- cloudnetpy/instruments/cl61d.py +5 -3
- cloudnetpy/instruments/cloudnet_instrument.py +7 -7
- cloudnetpy/instruments/copernicus.py +16 -7
- cloudnetpy/instruments/disdrometer/common.py +5 -4
- cloudnetpy/instruments/disdrometer/parsivel.py +14 -9
- cloudnetpy/instruments/disdrometer/thies.py +11 -7
- cloudnetpy/instruments/fd12p.py +7 -6
- cloudnetpy/instruments/galileo.py +16 -7
- cloudnetpy/instruments/hatpro.py +33 -24
- cloudnetpy/instruments/lufft.py +6 -4
- cloudnetpy/instruments/mira.py +33 -19
- cloudnetpy/instruments/mrr.py +12 -12
- cloudnetpy/instruments/nc_lidar.py +1 -1
- cloudnetpy/instruments/nc_radar.py +8 -8
- cloudnetpy/instruments/pollyxt.py +19 -12
- cloudnetpy/instruments/radiometrics.py +17 -10
- cloudnetpy/instruments/rain_e_h3.py +9 -5
- cloudnetpy/instruments/rpg.py +32 -21
- cloudnetpy/instruments/rpg_reader.py +15 -12
- cloudnetpy/instruments/vaisala.py +32 -24
- cloudnetpy/instruments/weather_station.py +28 -21
- cloudnetpy/model_evaluation/file_handler.py +27 -29
- cloudnetpy/model_evaluation/plotting/plot_tools.py +7 -5
- cloudnetpy/model_evaluation/plotting/plotting.py +41 -32
- cloudnetpy/model_evaluation/products/advance_methods.py +38 -34
- cloudnetpy/model_evaluation/products/grid_methods.py +10 -9
- cloudnetpy/model_evaluation/products/model_products.py +15 -9
- cloudnetpy/model_evaluation/products/observation_products.py +12 -10
- cloudnetpy/model_evaluation/products/product_resampling.py +11 -7
- cloudnetpy/model_evaluation/products/tools.py +18 -14
- cloudnetpy/model_evaluation/statistics/statistical_methods.py +6 -5
- cloudnetpy/model_evaluation/tests/unit/test_plotting.py +18 -25
- cloudnetpy/model_evaluation/utils.py +3 -3
- cloudnetpy/output.py +15 -32
- cloudnetpy/plotting/plotting.py +22 -12
- cloudnetpy/products/classification.py +15 -9
- cloudnetpy/products/der.py +24 -19
- cloudnetpy/products/drizzle.py +21 -13
- cloudnetpy/products/drizzle_error.py +8 -7
- cloudnetpy/products/drizzle_tools.py +27 -23
- cloudnetpy/products/epsilon.py +6 -5
- cloudnetpy/products/ier.py +11 -5
- cloudnetpy/products/iwc.py +18 -9
- cloudnetpy/products/lwc.py +41 -31
- cloudnetpy/products/mwr_tools.py +30 -19
- cloudnetpy/products/product_tools.py +23 -19
- cloudnetpy/utils.py +84 -98
- cloudnetpy/version.py +2 -2
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/METADATA +3 -2
- cloudnetpy-1.81.1.dist-info/RECORD +126 -0
- cloudnetpy-1.80.8.dist-info/RECORD +0 -126
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/WHEEL +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/entry_points.txt +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/licenses/LICENSE +0 -0
- {cloudnetpy-1.80.8.dist-info → cloudnetpy-1.81.1.dist-info}/top_level.txt +0 -0
cloudnetpy/instruments/hatpro.py
CHANGED
@@ -3,9 +3,12 @@
|
|
3
3
|
import datetime
|
4
4
|
import logging
|
5
5
|
from collections import defaultdict
|
6
|
+
from os import PathLike
|
6
7
|
from pathlib import Path
|
7
8
|
from typing import Literal
|
9
|
+
from uuid import UUID
|
8
10
|
|
11
|
+
import mwrpy.rpg_mwr
|
9
12
|
import netCDF4
|
10
13
|
import numpy as np
|
11
14
|
from mwrpy.exceptions import MissingInputData
|
@@ -35,13 +38,13 @@ ITYPE_MAP: dict[IType, Instrument] = {
|
|
35
38
|
|
36
39
|
|
37
40
|
def hatpro2l1c(
|
38
|
-
mwr_dir: str,
|
39
|
-
output_file: str,
|
41
|
+
mwr_dir: str | PathLike,
|
42
|
+
output_file: str | PathLike,
|
40
43
|
site_meta: dict,
|
41
44
|
instrument_type: IType = "hatpro",
|
42
|
-
uuid: str | None = None,
|
43
|
-
date: datetime.date |
|
44
|
-
) ->
|
45
|
+
uuid: str | UUID | None = None,
|
46
|
+
date: str | datetime.date | None = None,
|
47
|
+
) -> UUID:
|
45
48
|
"""Converts RPG HATPRO microwave radiometer data into Cloudnet Level 1c netCDF file.
|
46
49
|
|
47
50
|
Args:
|
@@ -57,6 +60,7 @@ def hatpro2l1c(
|
|
57
60
|
"""
|
58
61
|
if isinstance(date, str):
|
59
62
|
date = datetime.date.fromisoformat(date)
|
63
|
+
uuid = utils.get_uuid(uuid)
|
60
64
|
|
61
65
|
coeff_files = site_meta.get("coefficientFiles")
|
62
66
|
time_offset = site_meta.get("time_offset")
|
@@ -64,9 +68,9 @@ def hatpro2l1c(
|
|
64
68
|
try:
|
65
69
|
hatpro_raw = lev1_to_nc(
|
66
70
|
"1C01",
|
67
|
-
mwr_dir,
|
71
|
+
str(mwr_dir),
|
68
72
|
instrument_type=instrument_type,
|
69
|
-
output_file=output_file,
|
73
|
+
output_file=str(output_file),
|
70
74
|
coeff_files=coeff_files,
|
71
75
|
instrument_config=site_meta,
|
72
76
|
date=date,
|
@@ -122,7 +126,7 @@ def hatpro2l1c(
|
|
122
126
|
attrs_copy = ATTRIBUTES_1B01.copy()
|
123
127
|
attributes = output.add_time_attribute(attrs_copy, hatpro.date)
|
124
128
|
output.update_attributes(hatpro.data, attributes)
|
125
|
-
|
129
|
+
output.save_level1b(hatpro, output_file, uuid)
|
126
130
|
with netCDF4.Dataset(output_file, "a") as nc:
|
127
131
|
nc.cloudnet_file_type = "mwr-l1c"
|
128
132
|
nc.title = nc.title.replace("radiometer", "radiometer Level 1c")
|
@@ -134,22 +138,24 @@ def hatpro2l1c(
|
|
134
138
|
|
135
139
|
|
136
140
|
class HatproL1c:
|
137
|
-
def __init__(
|
141
|
+
def __init__(
|
142
|
+
self, hatpro: mwrpy.rpg_mwr.Rpg, site_meta: dict, instrument: Instrument
|
143
|
+
) -> None:
|
138
144
|
self.raw_data = hatpro.raw_data
|
139
145
|
self.data = hatpro.data
|
140
|
-
self.date = hatpro.date
|
146
|
+
self.date = hatpro.date
|
141
147
|
self.site_meta = site_meta
|
142
148
|
self.instrument = instrument
|
143
149
|
|
144
150
|
|
145
151
|
def hatpro2nc(
|
146
|
-
path_to_files: str,
|
147
|
-
output_file: str,
|
152
|
+
path_to_files: str | PathLike,
|
153
|
+
output_file: str | PathLike,
|
148
154
|
site_meta: dict,
|
149
155
|
instrument_type: IType = "hatpro",
|
150
|
-
uuid: str | None = None,
|
151
|
-
date: str | None = None,
|
152
|
-
) -> tuple[
|
156
|
+
uuid: str | UUID | None = None,
|
157
|
+
date: str | datetime.date | None = None,
|
158
|
+
) -> tuple[UUID, list[Path]]:
|
153
159
|
"""Converts RPG HATPRO microwave radiometer data into Cloudnet Level 1b
|
154
160
|
netCDF file.
|
155
161
|
|
@@ -188,9 +194,12 @@ def hatpro2nc(
|
|
188
194
|
>>> hatpro2nc('/path/to/files/', 'hatpro.nc', site_meta)
|
189
195
|
|
190
196
|
"""
|
197
|
+
if isinstance(date, str):
|
198
|
+
date = datetime.date.fromisoformat(date)
|
199
|
+
uuid = utils.get_uuid(uuid)
|
191
200
|
hatpro_objects, valid_files = _get_hatpro_objects(Path(path_to_files), date)
|
192
|
-
is_lwp_files = any(f.
|
193
|
-
is_iwv_files = any(f.
|
201
|
+
is_lwp_files = any(f.suffix == ".LWP" for f in valid_files)
|
202
|
+
is_iwv_files = any(f.suffix == ".IWV" for f in valid_files)
|
194
203
|
if not is_lwp_files:
|
195
204
|
raise ValidTimeStampError
|
196
205
|
if is_iwv_files:
|
@@ -203,14 +212,14 @@ def hatpro2nc(
|
|
203
212
|
hatpro.remove_duplicate_timestamps()
|
204
213
|
attributes = output.add_time_attribute({}, hatpro.date)
|
205
214
|
output.update_attributes(hatpro.data, attributes)
|
206
|
-
|
215
|
+
output.save_level1b(hatpro, output_file, uuid)
|
207
216
|
return uuid, valid_files
|
208
217
|
|
209
218
|
|
210
219
|
def _get_hatpro_objects(
|
211
220
|
directory: Path,
|
212
|
-
expected_date:
|
213
|
-
) -> tuple[list[HatproBinCombined], list[
|
221
|
+
expected_date: datetime.date | None,
|
222
|
+
) -> tuple[list[HatproBinCombined], list[Path]]:
|
214
223
|
objects = defaultdict(list)
|
215
224
|
for filename in directory.iterdir():
|
216
225
|
try:
|
@@ -230,12 +239,12 @@ def _get_hatpro_objects(
|
|
230
239
|
logging.warning("Ignoring file '%s': %s", filename, err)
|
231
240
|
continue
|
232
241
|
|
233
|
-
valid_files: list[
|
242
|
+
valid_files: list[Path] = []
|
234
243
|
combined_objs = []
|
235
244
|
for _stem, objs in sorted(objects.items()):
|
236
245
|
try:
|
237
246
|
combined_objs.append(HatproBinCombined(objs))
|
238
|
-
valid_files.extend(
|
247
|
+
valid_files.extend(obj.filename for obj in objs)
|
239
248
|
except (TypeError, ValueError) as err:
|
240
249
|
files = "'" + "', '".join(str(obj.filename) for obj in objs) + "'"
|
241
250
|
logging.warning("Ignoring files %s: %s", files, err)
|
@@ -244,13 +253,13 @@ def _get_hatpro_objects(
|
|
244
253
|
return combined_objs, valid_files
|
245
254
|
|
246
255
|
|
247
|
-
def _validate_date(obj: HatproBin, expected_date:
|
256
|
+
def _validate_date(obj: HatproBin, expected_date: datetime.date) -> HatproBin:
|
248
257
|
if obj.header["_time_reference"] != 1:
|
249
258
|
msg = "Can not validate non-UTC dates"
|
250
259
|
raise ValueError(msg)
|
251
260
|
inds = []
|
252
261
|
for ind, timestamp in enumerate(obj.data["time"][:]):
|
253
|
-
date =
|
262
|
+
date = utils.seconds2date(timestamp).date()
|
254
263
|
if date == expected_date:
|
255
264
|
inds.append(ind)
|
256
265
|
if not inds:
|
cloudnetpy/instruments/lufft.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
"""Module with a class for Lufft chm15k ceilometer."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import logging
|
5
|
+
from os import PathLike
|
4
6
|
|
5
7
|
import netCDF4
|
6
8
|
import numpy as np
|
@@ -16,10 +18,10 @@ class LufftCeilo(NcLidar):
|
|
16
18
|
|
17
19
|
def __init__(
|
18
20
|
self,
|
19
|
-
file_name: str,
|
21
|
+
file_name: str | PathLike,
|
20
22
|
site_meta: dict,
|
21
|
-
expected_date:
|
22
|
-
):
|
23
|
+
expected_date: datetime.date | None = None,
|
24
|
+
) -> None:
|
23
25
|
super().__init__()
|
24
26
|
self.file_name = file_name
|
25
27
|
self.site_meta = site_meta
|
@@ -89,7 +91,7 @@ class LufftCeilo(NcLidar):
|
|
89
91
|
return 1
|
90
92
|
return step_factor ** (-(nn1 - reference) / scale)
|
91
93
|
|
92
|
-
def _getvar(self, *args) -> float | ma.MaskedArray:
|
94
|
+
def _getvar(self, *args: str) -> float | ma.MaskedArray:
|
93
95
|
if self.dataset is None:
|
94
96
|
msg = "No dataset found"
|
95
97
|
raise RuntimeError(msg)
|
cloudnetpy/instruments/mira.py
CHANGED
@@ -1,9 +1,13 @@
|
|
1
1
|
"""Module for reading raw cloud radar data."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import logging
|
4
5
|
import os
|
5
6
|
from collections import OrderedDict
|
7
|
+
from collections.abc import Sequence
|
8
|
+
from os import PathLike
|
6
9
|
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
10
|
+
from uuid import UUID
|
7
11
|
|
8
12
|
from numpy import ma
|
9
13
|
|
@@ -14,12 +18,12 @@ from cloudnetpy.metadata import MetaData
|
|
14
18
|
|
15
19
|
|
16
20
|
def mira2nc(
|
17
|
-
raw_mira: str |
|
18
|
-
output_file: str,
|
21
|
+
raw_mira: str | PathLike | Sequence[str | PathLike],
|
22
|
+
output_file: str | PathLike,
|
19
23
|
site_meta: dict,
|
20
|
-
uuid: str | None = None,
|
21
|
-
date: str | None = None,
|
22
|
-
) ->
|
24
|
+
uuid: str | UUID | None = None,
|
25
|
+
date: str | datetime.date | None = None,
|
26
|
+
) -> UUID:
|
23
27
|
"""Converts METEK MIRA-35 cloud radar data into Cloudnet Level 1b netCDF file.
|
24
28
|
|
25
29
|
This function converts raw MIRA file(s) into a much smaller file that
|
@@ -55,6 +59,10 @@ def mira2nc(
|
|
55
59
|
>>> mira2nc('/one/day/of/mira/znc/files/', 'radar.nc', site_meta)
|
56
60
|
|
57
61
|
"""
|
62
|
+
if isinstance(date, str):
|
63
|
+
date = datetime.date.fromisoformat(date)
|
64
|
+
uuid = utils.get_uuid(uuid)
|
65
|
+
|
58
66
|
with TemporaryDirectory() as temp_dir:
|
59
67
|
input_filename, keymap = _parse_input_files(raw_mira, temp_dir)
|
60
68
|
|
@@ -62,7 +70,7 @@ def mira2nc(
|
|
62
70
|
mira.init_data(keymap)
|
63
71
|
if date is not None:
|
64
72
|
mira.screen_by_date(date)
|
65
|
-
mira.date = date
|
73
|
+
mira.date = date
|
66
74
|
mira.sort_timestamps()
|
67
75
|
mira.remove_duplicate_timestamps()
|
68
76
|
mira.linear_to_db(("Zh", "ldr", "SNR"))
|
@@ -101,7 +109,8 @@ def mira2nc(
|
|
101
109
|
mira.test_if_all_masked()
|
102
110
|
attributes = output.add_time_attribute(ATTRIBUTES, mira.date)
|
103
111
|
output.update_attributes(mira.data, attributes)
|
104
|
-
|
112
|
+
output.save_level1b(mira, output_file, uuid)
|
113
|
+
return uuid
|
105
114
|
|
106
115
|
|
107
116
|
class Mira(NcRadar):
|
@@ -113,9 +122,9 @@ class Mira(NcRadar):
|
|
113
122
|
|
114
123
|
"""
|
115
124
|
|
116
|
-
epoch = (1970, 1, 1)
|
125
|
+
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
|
117
126
|
|
118
|
-
def __init__(self, full_path: str, site_meta: dict):
|
127
|
+
def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
|
119
128
|
super().__init__(full_path, site_meta)
|
120
129
|
self.date = self._init_mira_date()
|
121
130
|
if "model" not in site_meta or site_meta["model"] == "mira-35":
|
@@ -126,21 +135,21 @@ class Mira(NcRadar):
|
|
126
135
|
msg = f"Invalid model: {site_meta['model']}"
|
127
136
|
raise ValueError(msg)
|
128
137
|
|
129
|
-
def screen_by_date(self, expected_date:
|
138
|
+
def screen_by_date(self, expected_date: datetime.date) -> None:
|
130
139
|
"""Screens incorrect time stamps."""
|
131
140
|
time_stamps = self.getvar("time")
|
132
141
|
valid_indices = []
|
133
142
|
for ind, timestamp in enumerate(time_stamps):
|
134
143
|
if not timestamp:
|
135
144
|
continue
|
136
|
-
date =
|
145
|
+
date = utils.seconds2date(timestamp, self.epoch).date()
|
137
146
|
if date == expected_date:
|
138
147
|
valid_indices.append(ind)
|
139
148
|
self.screen_time_indices(valid_indices)
|
140
149
|
|
141
|
-
def _init_mira_date(self) ->
|
150
|
+
def _init_mira_date(self) -> datetime.date:
|
142
151
|
time_stamps = self.getvar("time")
|
143
|
-
return utils.seconds2date(float(time_stamps[0]), self.epoch)
|
152
|
+
return utils.seconds2date(float(time_stamps[0]), self.epoch).date()
|
144
153
|
|
145
154
|
def screen_invalid_ldr(self) -> None:
|
146
155
|
"""Masks LDR in MIRA STSR mode data.
|
@@ -169,16 +178,21 @@ class Mira(NcRadar):
|
|
169
178
|
array[array > (upper + margin)] = ma.masked
|
170
179
|
|
171
180
|
|
172
|
-
def _parse_input_files(
|
173
|
-
|
181
|
+
def _parse_input_files(
|
182
|
+
input_files: str | PathLike | Sequence[str | PathLike], temp_dir: str
|
183
|
+
) -> tuple[str | PathLike, dict[str, str]]:
|
184
|
+
input_filename: str | PathLike
|
185
|
+
if (
|
186
|
+
not isinstance(input_files, str) and isinstance(input_files, Sequence)
|
187
|
+
) or os.path.isdir(input_files):
|
174
188
|
with NamedTemporaryFile(
|
175
189
|
dir=temp_dir,
|
176
190
|
suffix=".nc",
|
177
191
|
delete=False,
|
178
192
|
) as temp_file:
|
179
193
|
input_filename = temp_file.name
|
180
|
-
if isinstance(input_files,
|
181
|
-
valid_files = sorted(input_files)
|
194
|
+
if not isinstance(input_files, str) and isinstance(input_files, Sequence):
|
195
|
+
valid_files = sorted(map(str, input_files))
|
182
196
|
else:
|
183
197
|
valid_files = utils.get_sorted_filenames(input_files, ".znc")
|
184
198
|
if not valid_files:
|
@@ -210,7 +224,7 @@ def _parse_input_files(input_files: str | list[str], temp_dir: str) -> tuple:
|
|
210
224
|
)
|
211
225
|
else:
|
212
226
|
input_filename = input_files
|
213
|
-
keymap = _get_keymap(input_filename.split(".")[-1])
|
227
|
+
keymap = _get_keymap(str(input_filename).split(".")[-1])
|
214
228
|
|
215
229
|
return input_filename, keymap
|
216
230
|
|
@@ -227,7 +241,7 @@ def _get_ignored_variables(filetype: str) -> list | None:
|
|
227
241
|
return keymaps.get(filetype.lower(), keymaps.get("mmclx"))
|
228
242
|
|
229
243
|
|
230
|
-
def _get_keymap(filetype: str) -> dict:
|
244
|
+
def _get_keymap(filetype: str) -> dict[str, str]:
|
231
245
|
"""Returns a dictionary mapping the variables in the raw data to the processed
|
232
246
|
Cloudnet file.
|
233
247
|
"""
|
cloudnetpy/instruments/mrr.py
CHANGED
@@ -19,9 +19,9 @@ def mrr2nc(
|
|
19
19
|
input_file: PathLike | str | Iterable[PathLike | str],
|
20
20
|
output_file: PathLike | str,
|
21
21
|
site_meta: dict,
|
22
|
-
uuid:
|
23
|
-
date: datetime.date |
|
24
|
-
) ->
|
22
|
+
uuid: str | UUID | None = None,
|
23
|
+
date: str | datetime.date | None = None,
|
24
|
+
) -> UUID:
|
25
25
|
"""Converts METEK MRR-PRO data into Cloudnet Level 1b netCDF file.
|
26
26
|
|
27
27
|
This function converts raw MRR file(s) into a much smaller file that
|
@@ -48,10 +48,9 @@ def mrr2nc(
|
|
48
48
|
>>> site_meta = {'name': 'LIM', 'latitude': 51.333, 'longitude': 12.389}
|
49
49
|
>>> mrr2nc('input.nc', 'output.nc', site_meta)
|
50
50
|
"""
|
51
|
-
if isinstance(uuid, str):
|
52
|
-
uuid = UUID(uuid)
|
53
51
|
if isinstance(date, str):
|
54
52
|
date = datetime.date.fromisoformat(date)
|
53
|
+
uuid = utils.get_uuid(uuid)
|
55
54
|
|
56
55
|
keymap = {
|
57
56
|
"RR": "rainfall_rate",
|
@@ -112,7 +111,8 @@ def mrr2nc(
|
|
112
111
|
mrr.sort_timestamps()
|
113
112
|
attributes = output.add_time_attribute(ATTRIBUTES, mrr.date)
|
114
113
|
output.update_attributes(mrr.data, attributes)
|
115
|
-
|
114
|
+
output.save_level1b(mrr, output_file, uuid)
|
115
|
+
return uuid
|
116
116
|
|
117
117
|
|
118
118
|
class MrrPro(NcRadar):
|
@@ -125,9 +125,9 @@ class MrrPro(NcRadar):
|
|
125
125
|
|
126
126
|
"""
|
127
127
|
|
128
|
-
epoch = (1970, 1, 1)
|
128
|
+
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
|
129
129
|
|
130
|
-
def __init__(self, full_path: PathLike | str, site_meta: dict):
|
130
|
+
def __init__(self, full_path: PathLike | str, site_meta: dict) -> None:
|
131
131
|
super().__init__(full_path, site_meta)
|
132
132
|
self.instrument = instruments.MRR_PRO
|
133
133
|
if m := re.search(
|
@@ -137,9 +137,9 @@ class MrrPro(NcRadar):
|
|
137
137
|
):
|
138
138
|
self.serial_number = m[1]
|
139
139
|
|
140
|
-
def init_date(self) ->
|
140
|
+
def init_date(self) -> datetime.date:
|
141
141
|
time_stamps = self.getvar("time")
|
142
|
-
return utils.seconds2date(time_stamps[0], (
|
142
|
+
return utils.seconds2date(time_stamps[0], self.epoch).date()
|
143
143
|
|
144
144
|
def fix_units(self) -> None:
|
145
145
|
self.data["v"].data *= -1 # towards -> away from instrument
|
@@ -156,8 +156,8 @@ class MrrPro(NcRadar):
|
|
156
156
|
time_stamps = self.getvar("time")
|
157
157
|
valid_indices = []
|
158
158
|
for ind, timestamp in enumerate(time_stamps):
|
159
|
-
date =
|
160
|
-
if date == expected_date
|
159
|
+
date = utils.seconds2date(timestamp, self.epoch).date()
|
160
|
+
if date == expected_date:
|
161
161
|
valid_indices.append(ind)
|
162
162
|
self.screen_time_indices(valid_indices)
|
163
163
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
"""Module for reading raw cloud radar data."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import logging
|
4
5
|
from os import PathLike
|
5
6
|
from typing import TYPE_CHECKING
|
@@ -28,10 +29,10 @@ class NcRadar(DataSource, CloudnetInstrument):
|
|
28
29
|
Used with BASTA, MIRA and Copernicus radars.
|
29
30
|
"""
|
30
31
|
|
31
|
-
def __init__(self, full_path: PathLike | str, site_meta: dict):
|
32
|
+
def __init__(self, full_path: PathLike | str, site_meta: dict) -> None:
|
32
33
|
super().__init__(full_path)
|
33
34
|
self.site_meta = site_meta
|
34
|
-
self.date:
|
35
|
+
self.date: datetime.date
|
35
36
|
self.instrument: Instrument | None = None
|
36
37
|
|
37
38
|
def init_data(self, keymap: dict) -> None:
|
@@ -171,7 +172,7 @@ class NcRadar(DataSource, CloudnetInstrument):
|
|
171
172
|
class ChilboltonRadar(NcRadar):
|
172
173
|
"""Class for Chilbolton cloud radars Galileo and Copernicus."""
|
173
174
|
|
174
|
-
def __init__(self, full_path: str, site_meta: dict) -> None:
|
175
|
+
def __init__(self, full_path: str | PathLike, site_meta: dict) -> None:
|
175
176
|
super().__init__(full_path, site_meta)
|
176
177
|
self.date = self._init_date()
|
177
178
|
|
@@ -181,10 +182,9 @@ class ChilboltonRadar(NcRadar):
|
|
181
182
|
folding_velocity = self.dataset.variables[key].folding_velocity
|
182
183
|
self.append_data(np.array(folding_velocity), "nyquist_velocity")
|
183
184
|
|
184
|
-
def check_date(self, date:
|
185
|
-
if self.date != date
|
185
|
+
def check_date(self, date: datetime.date) -> None:
|
186
|
+
if self.date != date:
|
186
187
|
raise ValidTimeStampError
|
187
188
|
|
188
|
-
def _init_date(self) ->
|
189
|
-
|
190
|
-
return [str(x).zfill(2) for x in epoch]
|
189
|
+
def _init_date(self) -> datetime.date:
|
190
|
+
return utils.get_epoch(self.dataset["time"].units).date()
|
@@ -1,11 +1,15 @@
|
|
1
1
|
"""Module for reading / converting pollyxt data."""
|
2
2
|
|
3
|
+
import datetime
|
3
4
|
import glob
|
4
5
|
import logging
|
5
6
|
from collections import Counter
|
7
|
+
from os import PathLike
|
8
|
+
from uuid import UUID
|
6
9
|
|
7
10
|
import netCDF4
|
8
11
|
import numpy as np
|
12
|
+
import numpy.typing as npt
|
9
13
|
from numpy import ma
|
10
14
|
from numpy.testing import assert_almost_equal
|
11
15
|
|
@@ -14,16 +18,15 @@ from cloudnetpy.exceptions import InconsistentDataError, ValidTimeStampError
|
|
14
18
|
from cloudnetpy.instruments import instruments
|
15
19
|
from cloudnetpy.instruments.ceilometer import Ceilometer
|
16
20
|
from cloudnetpy.metadata import COMMON_ATTRIBUTES, MetaData
|
17
|
-
from cloudnetpy.utils import Epoch
|
18
21
|
|
19
22
|
|
20
23
|
def pollyxt2nc(
|
21
|
-
input_folder: str,
|
22
|
-
output_file: str,
|
24
|
+
input_folder: str | PathLike,
|
25
|
+
output_file: str | PathLike,
|
23
26
|
site_meta: dict,
|
24
|
-
uuid: str | None = None,
|
25
|
-
date: str | None = None,
|
26
|
-
) ->
|
27
|
+
uuid: str | UUID | None = None,
|
28
|
+
date: str | datetime.date | None = None,
|
29
|
+
) -> UUID:
|
27
30
|
"""Converts PollyXT Raman lidar data into Cloudnet Level 1b netCDF file.
|
28
31
|
|
29
32
|
Args:
|
@@ -50,6 +53,9 @@ def pollyxt2nc(
|
|
50
53
|
>>> pollyxt2nc('/path/to/files/', 'pollyxt.nc', site_meta)
|
51
54
|
|
52
55
|
"""
|
56
|
+
if isinstance(date, str):
|
57
|
+
date = datetime.date.fromisoformat(date)
|
58
|
+
uuid = utils.get_uuid(uuid)
|
53
59
|
snr_limit = site_meta.get("snr_limit", 2)
|
54
60
|
polly = PollyXt(site_meta, date)
|
55
61
|
epoch = polly.fetch_data(input_folder)
|
@@ -64,11 +70,12 @@ def pollyxt2nc(
|
|
64
70
|
attributes = output.add_time_attribute(ATTRIBUTES, polly.date)
|
65
71
|
output.update_attributes(polly.data, attributes)
|
66
72
|
polly.add_snr_info("beta", snr_limit)
|
67
|
-
|
73
|
+
output.save_level1b(polly, output_file, uuid)
|
74
|
+
return uuid
|
68
75
|
|
69
76
|
|
70
77
|
class PollyXt(Ceilometer):
|
71
|
-
def __init__(self, site_meta: dict, expected_date:
|
78
|
+
def __init__(self, site_meta: dict, expected_date: datetime.date | None) -> None:
|
72
79
|
super().__init__()
|
73
80
|
self.site_meta = site_meta
|
74
81
|
self.expected_date = expected_date
|
@@ -95,7 +102,7 @@ class PollyXt(Ceilometer):
|
|
95
102
|
default = 5
|
96
103
|
self.data["zenith_angle"] = float(self.metadata.get("zenith_angle", default))
|
97
104
|
|
98
|
-
def fetch_data(self, input_folder: str) ->
|
105
|
+
def fetch_data(self, input_folder: str | PathLike) -> datetime.datetime:
|
99
106
|
"""Read input data."""
|
100
107
|
bsc_files = glob.glob(f"{input_folder}/*[0-9]_att*.nc")
|
101
108
|
depol_files = glob.glob(f"{input_folder}/*[0-9]_vol*.nc")
|
@@ -114,7 +121,7 @@ class PollyXt(Ceilometer):
|
|
114
121
|
self._fetch_attributes(bsc_files[0])
|
115
122
|
with netCDF4.Dataset(bsc_files[0], "r") as nc:
|
116
123
|
self.data["range"] = nc.variables["height"][:]
|
117
|
-
calibration_factors:
|
124
|
+
calibration_factors: npt.NDArray = np.array([])
|
118
125
|
beta_channel = self._get_valid_beta_channel(bsc_files)
|
119
126
|
bsc_key = f"attenuated_backscatter_{beta_channel}nm"
|
120
127
|
for bsc_file, depol_file in zip(bsc_files, depol_files, strict=True):
|
@@ -197,7 +204,7 @@ class PollyXt(Ceilometer):
|
|
197
204
|
def _fetch_files_with_same_range(
|
198
205
|
bsc_files: list[str], depol_files: list[str]
|
199
206
|
) -> tuple[list[str], list[str]]:
|
200
|
-
def get_sum(file: str):
|
207
|
+
def get_sum(file: str) -> float:
|
201
208
|
with netCDF4.Dataset(file, "r") as nc:
|
202
209
|
return np.sum(np.round(nc.variables["height"][:]))
|
203
210
|
|
@@ -229,7 +236,7 @@ def _read_array_from_file_pair(
|
|
229
236
|
nc_file1: netCDF4.Dataset,
|
230
237
|
nc_file2: netCDF4.Dataset,
|
231
238
|
key: str,
|
232
|
-
) ->
|
239
|
+
) -> npt.NDArray:
|
233
240
|
array1 = nc_file1.variables[key][:]
|
234
241
|
array2 = nc_file2.variables[key][:]
|
235
242
|
assert_almost_equal(
|
@@ -7,10 +7,13 @@ import math
|
|
7
7
|
import os
|
8
8
|
import re
|
9
9
|
from operator import attrgetter
|
10
|
+
from os import PathLike
|
10
11
|
from pathlib import Path
|
11
12
|
from typing import Any, NamedTuple
|
13
|
+
from uuid import UUID
|
12
14
|
|
13
15
|
import numpy as np
|
16
|
+
import numpy.typing as npt
|
14
17
|
from numpy import ma
|
15
18
|
|
16
19
|
from cloudnetpy import output, utils
|
@@ -21,12 +24,12 @@ from cloudnetpy.metadata import MetaData
|
|
21
24
|
|
22
25
|
|
23
26
|
def radiometrics2nc(
|
24
|
-
full_path: str |
|
25
|
-
output_file: str |
|
27
|
+
full_path: str | PathLike,
|
28
|
+
output_file: str | PathLike,
|
26
29
|
site_meta: dict,
|
27
|
-
uuid: str | None = None,
|
30
|
+
uuid: str | UUID | None = None,
|
28
31
|
date: str | datetime.date | None = None,
|
29
|
-
) ->
|
32
|
+
) -> UUID:
|
30
33
|
"""Converts Radiometrics .csv file into Cloudnet Level 1b netCDF file.
|
31
34
|
|
32
35
|
Args:
|
@@ -49,6 +52,7 @@ def radiometrics2nc(
|
|
49
52
|
"""
|
50
53
|
if isinstance(date, str):
|
51
54
|
date = datetime.date.fromisoformat(date)
|
55
|
+
uuid = utils.get_uuid(uuid)
|
52
56
|
if os.path.isdir(full_path):
|
53
57
|
valid_filenames = list(Path(full_path).iterdir())
|
54
58
|
else:
|
@@ -65,7 +69,8 @@ def radiometrics2nc(
|
|
65
69
|
raise ValidTimeStampError(msg)
|
66
70
|
attributes = output.add_time_attribute(ATTRIBUTES, radiometrics.date)
|
67
71
|
output.update_attributes(radiometrics.data, attributes)
|
68
|
-
|
72
|
+
output.save_level1b(radiometrics, output_file, uuid)
|
73
|
+
return uuid
|
69
74
|
|
70
75
|
|
71
76
|
class Record(NamedTuple):
|
@@ -85,7 +90,7 @@ class RadiometricsMP:
|
|
85
90
|
MP-1500A, MP-183A.
|
86
91
|
"""
|
87
92
|
|
88
|
-
def __init__(self, filename: Path):
|
93
|
+
def __init__(self, filename: Path) -> None:
|
89
94
|
self.filename = filename
|
90
95
|
self.raw_data: list[Record] = []
|
91
96
|
self.data: dict = {}
|
@@ -248,7 +253,7 @@ class RadiometricsWVR:
|
|
248
253
|
radiometer.
|
249
254
|
"""
|
250
255
|
|
251
|
-
def __init__(self, filename: Path):
|
256
|
+
def __init__(self, filename: Path) -> None:
|
252
257
|
self.filename = filename
|
253
258
|
self.raw_data: dict = {}
|
254
259
|
self.data: dict = {}
|
@@ -307,7 +312,9 @@ class RadiometricsCombined:
|
|
307
312
|
date: datetime.date | None
|
308
313
|
instrument: instruments.Instrument
|
309
314
|
|
310
|
-
def __init__(
|
315
|
+
def __init__(
|
316
|
+
self, objs: list[RadiometricsMP | RadiometricsWVR], site_meta: dict
|
317
|
+
) -> None:
|
311
318
|
self.site_meta = site_meta
|
312
319
|
self.data = {}
|
313
320
|
self.date = None
|
@@ -337,7 +344,7 @@ class RadiometricsCombined:
|
|
337
344
|
continue
|
338
345
|
self.data[key] = self.data[key][valid_mask]
|
339
346
|
|
340
|
-
def sort_timestamps(self):
|
347
|
+
def sort_timestamps(self) -> None:
|
341
348
|
ind = np.argsort(self.data["time"])
|
342
349
|
for key in self.data:
|
343
350
|
if key in ("range", "height"):
|
@@ -413,7 +420,7 @@ def _parse_datetime(text: str) -> datetime.datetime:
|
|
413
420
|
)
|
414
421
|
|
415
422
|
|
416
|
-
def _find_closest(x:
|
423
|
+
def _find_closest(x: npt.NDArray, y: npt.NDArray, x_new: npt.NDArray) -> npt.NDArray:
|
417
424
|
return y[np.argmin(np.abs(x_new - x[:, np.newaxis]), axis=0)]
|
418
425
|
|
419
426
|
|