obsarray 1.0.2__py3-none-any.whl → 1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- obsarray/__init__.py +9 -3
- obsarray/conventions.py +84 -0
- obsarray/err_corr.py +3 -7
- obsarray/flag_accessor.py +27 -25
- obsarray/options.py +78 -0
- obsarray/templater/dataset_util.py +21 -14
- obsarray/templater/dstemplater.py +1 -1
- obsarray/templater/template_util.py +5 -2
- obsarray/templater/tests/test_dataset_util.py +3 -1
- obsarray/test/test_conventions.py +127 -0
- obsarray/test/test_flag_accessor.py +0 -16
- obsarray/test/test_options.py +51 -0
- obsarray/test/test_unc_accessor.py +87 -28
- obsarray/test/test_utils.py +161 -0
- obsarray/unc_accessor.py +120 -26
- obsarray/utils.py +40 -3
- obsarray-1.1.dist-info/METADATA +130 -0
- obsarray-1.1.dist-info/RECORD +28 -0
- {obsarray-1.0.2.dist-info → obsarray-1.1.dist-info}/WHEEL +1 -1
- obsarray-1.1.dist-info/top_level.txt +1 -0
- docs/conf.py +0 -181
- notebook/test_register_err_corr_form.py +0 -45
- obsarray/_version.py +0 -21
- obsarray-1.0.2.dist-info/METADATA +0 -57
- obsarray-1.0.2.dist-info/RECORD +0 -26
- obsarray-1.0.2.dist-info/top_level.txt +0 -3
- {obsarray-1.0.2.dist-info → obsarray-1.1.dist-info}/licenses/LICENSE +0 -0
obsarray/__init__.py
CHANGED
|
@@ -3,12 +3,18 @@
|
|
|
3
3
|
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
4
4
|
__all__ = []
|
|
5
5
|
|
|
6
|
-
from .
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
7
8
|
from obsarray import unc_accessor, flag_accessor
|
|
8
9
|
from obsarray.err_corr import err_corr_forms
|
|
9
10
|
from obsarray.templater.template_util import create_ds
|
|
10
11
|
from obsarray.templater.dstemplater import DSTemplater
|
|
11
12
|
from obsarray.templater.dswriter import DSWriter
|
|
13
|
+
from obsarray.utils import append_names
|
|
14
|
+
from obsarray.options import set_options, get_options
|
|
15
|
+
from obsarray.conventions import UNCConventionsWarning
|
|
12
16
|
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
try:
|
|
18
|
+
__version__ = version("obsarray")
|
|
19
|
+
except PackageNotFoundError:
|
|
20
|
+
__version__ = "0.0.0+unknown"
|
obsarray/conventions.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""conventions - handling of the UNC metadata conventions version obsarray follows"""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import os
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
from obsarray.options import OPTIONS
|
|
8
|
+
|
|
9
|
+
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
10
|
+
__all__ = ["UNCConventionsWarning"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
UNC_CONVENTIONS_DOCS_URL = (
|
|
14
|
+
"https://obsarray.readthedocs.io/en/latest/content/user/unc_conventions.html"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
_UNC_CONVENTIONS_MESSAGE = (
|
|
18
|
+
"obsarray currently writes uncertainty metadata following v0.1 of the UNC "
|
|
19
|
+
"specification. A future release will default to UNC draft v0.2, which is proposed "
|
|
20
|
+
"for inclusion into the CF conventions and is not backwards compatible with v0.1 (for "
|
|
21
|
+
"example, 'unc_comps' becomes 'uncertainty_variables' and 'err_corr_*' becomes "
|
|
22
|
+
"'error_correlation'). See " + UNC_CONVENTIONS_DOCS_URL + " for details. To keep "
|
|
23
|
+
"the current behaviour and silence this warning, call "
|
|
24
|
+
'obsarray.set_options(unc_conventions="v0.1").'
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# set once the warning has been issued, so it is only shown once per session
|
|
28
|
+
_warned = False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class UNCConventionsWarning(FutureWarning):
|
|
32
|
+
"""
|
|
33
|
+
Warning that obsarray's default UNC conventions version will change from v0.1 to draft v0.2
|
|
34
|
+
|
|
35
|
+
Issued once per session, the first time obsarray writes uncertainty metadata, unless a conventions
|
|
36
|
+
version has been chosen explicitly with :py:class:`~obsarray.options.set_options`. It can also be
|
|
37
|
+
filtered like any other warning:
|
|
38
|
+
|
|
39
|
+
.. code-block:: python
|
|
40
|
+
|
|
41
|
+
import warnings
|
|
42
|
+
import obsarray
|
|
43
|
+
|
|
44
|
+
warnings.filterwarnings("ignore", category=obsarray.UNCConventionsWarning)
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def warn_unc_conventions() -> None:
|
|
49
|
+
"""
|
|
50
|
+
Warns, once per session, that the default UNC conventions version will change - unless a version has
|
|
51
|
+
been chosen explicitly with :py:class:`~obsarray.options.set_options`
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
global _warned
|
|
55
|
+
|
|
56
|
+
if _warned or OPTIONS["unc_conventions"] is not None:
|
|
57
|
+
return
|
|
58
|
+
|
|
59
|
+
_warned = True
|
|
60
|
+
warnings.warn(
|
|
61
|
+
_UNC_CONVENTIONS_MESSAGE, UNCConventionsWarning, stacklevel=_find_stack_level()
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _find_stack_level() -> int:
|
|
66
|
+
"""
|
|
67
|
+
Returns the ``stacklevel`` of the first frame outside obsarray, so warnings point at the calling code
|
|
68
|
+
|
|
69
|
+
:return: stack level to pass to :py:func:`warnings.warn` from the function calling this one
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
pkg_dir = os.path.dirname(os.path.abspath(__file__)) + os.sep
|
|
73
|
+
|
|
74
|
+
frame = inspect.currentframe()
|
|
75
|
+
level = 0
|
|
76
|
+
while frame is not None and frame.f_code.co_filename.startswith(pkg_dir):
|
|
77
|
+
frame = frame.f_back
|
|
78
|
+
level += 1
|
|
79
|
+
|
|
80
|
+
return level
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
if __name__ == "__main__":
|
|
84
|
+
pass
|
obsarray/err_corr.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""err_corr_forms - module for the defintion of error-correlation parameterisation forms"""
|
|
2
2
|
|
|
3
3
|
import abc
|
|
4
|
-
from typing import Callable, Type
|
|
4
|
+
from typing import Callable, Type
|
|
5
5
|
import numpy as np
|
|
6
6
|
from comet_maths.linear_algebra.matrix_conversion import expand_errcorr_dims
|
|
7
7
|
|
|
@@ -150,7 +150,7 @@ class BaseErrCorrForm(abc.ABC):
|
|
|
150
150
|
)
|
|
151
151
|
|
|
152
152
|
@abc.abstractmethod
|
|
153
|
-
def build_matrix(self, sli:
|
|
153
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
154
154
|
"""
|
|
155
155
|
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
156
156
|
in this parameterisation
|
|
@@ -161,7 +161,7 @@ class BaseErrCorrForm(abc.ABC):
|
|
|
161
161
|
"""
|
|
162
162
|
pass
|
|
163
163
|
|
|
164
|
-
def build_dot_matrix(self, sli:
|
|
164
|
+
def build_dot_matrix(self, sli: tuple) -> np.ndarray:
|
|
165
165
|
"""
|
|
166
166
|
Returns expanded error correlation matrix for use in dot product with error correlation
|
|
167
167
|
in other dimensions.
|
|
@@ -201,7 +201,6 @@ def register_err_corr_form(form_name: str) -> Callable:
|
|
|
201
201
|
|
|
202
202
|
@register_err_corr_form("random")
|
|
203
203
|
class RandomCorrelation(BaseErrCorrForm):
|
|
204
|
-
|
|
205
204
|
form = "random"
|
|
206
205
|
is_random = True
|
|
207
206
|
|
|
@@ -226,7 +225,6 @@ class RandomCorrelation(BaseErrCorrForm):
|
|
|
226
225
|
|
|
227
226
|
@register_err_corr_form("systematic")
|
|
228
227
|
class SystematicCorrelation(BaseErrCorrForm):
|
|
229
|
-
|
|
230
228
|
form = "systematic"
|
|
231
229
|
is_systematic = True
|
|
232
230
|
|
|
@@ -251,7 +249,6 @@ class SystematicCorrelation(BaseErrCorrForm):
|
|
|
251
249
|
|
|
252
250
|
@register_err_corr_form("err_corr_matrix")
|
|
253
251
|
class ErrCorrMatrixCorrelation(BaseErrCorrForm):
|
|
254
|
-
|
|
255
252
|
form = "err_corr_matrix"
|
|
256
253
|
|
|
257
254
|
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
@@ -278,7 +275,6 @@ class ErrCorrMatrixCorrelation(BaseErrCorrForm):
|
|
|
278
275
|
|
|
279
276
|
@register_err_corr_form("ensemble")
|
|
280
277
|
class EnsembleCorrelation(BaseErrCorrForm):
|
|
281
|
-
|
|
282
278
|
form = "ensemble"
|
|
283
279
|
|
|
284
280
|
def build_matrix(self, sli: tuple) -> np.ndarray:
|
obsarray/flag_accessor.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
import xarray as xr
|
|
5
|
-
from typing import List, Optional, Union, Tuple
|
|
5
|
+
from typing import Hashable, List, Optional, Union, Tuple
|
|
6
6
|
from obsarray.templater.template_util import create_var
|
|
7
7
|
from obsarray.templater.dataset_util import DatasetUtil
|
|
8
8
|
|
|
@@ -45,7 +45,7 @@ class Flag:
|
|
|
45
45
|
"""Custom __repr__"""
|
|
46
46
|
return str(self)
|
|
47
47
|
|
|
48
|
-
def __getitem__(self, sli: tuple):
|
|
48
|
+
def __getitem__(self, sli: Union[tuple, slice, int]):
|
|
49
49
|
"""
|
|
50
50
|
Defines variable slice
|
|
51
51
|
|
|
@@ -58,7 +58,7 @@ class Flag:
|
|
|
58
58
|
|
|
59
59
|
return self
|
|
60
60
|
|
|
61
|
-
def __setitem__(self, sli: tuple, flag_value):
|
|
61
|
+
def __setitem__(self, sli: Union[tuple, slice, int], flag_value):
|
|
62
62
|
"""
|
|
63
63
|
Sets flag values
|
|
64
64
|
|
|
@@ -83,26 +83,28 @@ class Flag:
|
|
|
83
83
|
)
|
|
84
84
|
else:
|
|
85
85
|
self._obj[self._flag_var_name].values[self._sli] = (
|
|
86
|
-
self._obj[self._flag_var_name][self._sli].values
|
|
87
|
-
|
|
86
|
+
self._obj[self._flag_var_name][self._sli].values
|
|
87
|
+
& np.array(~flag_mask)
|
|
88
|
+
).astype(self._obj[self._flag_var_name].values[self._sli].dtype)
|
|
88
89
|
|
|
89
90
|
# else apply mask
|
|
90
91
|
else:
|
|
91
92
|
if flag_value.dtype is not np.dtype(bool):
|
|
92
93
|
TypeError("Flag mask must of boolean type")
|
|
93
94
|
|
|
94
|
-
i_true = np.where(flag_value == True)
|
|
95
|
-
i_false = np.where(flag_value == False)
|
|
95
|
+
i_true = np.where(flag_value == True) # noqa: E712 (elementwise)
|
|
96
|
+
i_false = np.where(flag_value == False) # noqa: E712 (elementwise)
|
|
96
97
|
|
|
97
98
|
self._obj[self._flag_var_name][self._sli].values[i_true] = np.array(
|
|
98
99
|
self._obj[self._flag_var_name][self._sli].values[i_true] | flag_mask
|
|
99
100
|
)
|
|
100
101
|
|
|
101
102
|
self._obj[self._flag_var_name][self._sli].values[i_false] = np.array(
|
|
102
|
-
self._obj[self._flag_var_name][self._sli].values[i_false]
|
|
103
|
-
|
|
103
|
+
self._obj[self._flag_var_name][self._sli].values[i_false]
|
|
104
|
+
& np.array(~flag_mask)
|
|
105
|
+
).astype(self._obj[self._flag_var_name].values[self._sli].dtype)
|
|
104
106
|
|
|
105
|
-
def _expand_sli(self, sli: Optional[tuple] = None) -> tuple:
|
|
107
|
+
def _expand_sli(self, sli: Optional[Union[tuple, slice, int]] = None) -> tuple:
|
|
106
108
|
"""
|
|
107
109
|
Function to expand the provided sli so that it always has the right number of dimensions
|
|
108
110
|
|
|
@@ -118,12 +120,14 @@ class Flag:
|
|
|
118
120
|
# E.g. if providing [0] for a variable with 3 dimensions, this becomes
|
|
119
121
|
# [0,slice(None),slice(None)]
|
|
120
122
|
else:
|
|
121
|
-
|
|
123
|
+
out_sli_list: list = list(
|
|
124
|
+
[slice(None)] * self._obj[self._flag_var_name].ndim
|
|
125
|
+
)
|
|
122
126
|
sli_list = list(sli) if isinstance(sli, tuple) else [sli]
|
|
123
127
|
for i in range(len(sli_list)):
|
|
124
128
|
if not sli_list[i] == ":":
|
|
125
|
-
|
|
126
|
-
out_sli = tuple(
|
|
129
|
+
out_sli_list[i] = sli_list[i]
|
|
130
|
+
out_sli = tuple(out_sli_list)
|
|
127
131
|
|
|
128
132
|
return out_sli
|
|
129
133
|
|
|
@@ -194,12 +198,12 @@ class FlagVariable:
|
|
|
194
198
|
)
|
|
195
199
|
|
|
196
200
|
if flag_meaning not in flag_meanings:
|
|
197
|
-
self._obj[
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
201
|
+
self._obj[
|
|
202
|
+
self._flag_var_name
|
|
203
|
+
].attrs = DatasetUtil.add_flag_meaning_to_attrs(
|
|
204
|
+
self._obj[self._flag_var_name].attrs,
|
|
205
|
+
flag_meaning,
|
|
206
|
+
self._obj[self._flag_var_name].dtype,
|
|
203
207
|
)
|
|
204
208
|
|
|
205
209
|
self[flag_meaning][:] = flag_value
|
|
@@ -318,8 +322,7 @@ class FlagAccessor(object):
|
|
|
318
322
|
"""
|
|
319
323
|
|
|
320
324
|
# add uncertainty variable
|
|
321
|
-
if type(flag_def)
|
|
322
|
-
|
|
325
|
+
if type(flag_def) is xr.DataArray:
|
|
323
326
|
# add necessary flag metadata if missing from provided data array
|
|
324
327
|
flag_meanings, flag_masks = DatasetUtil.unpack_flag_attrs(flag_def.attrs)
|
|
325
328
|
flag_def.attrs.update(DatasetUtil.pack_flag_attrs(flag_meanings))
|
|
@@ -327,8 +330,7 @@ class FlagAccessor(object):
|
|
|
327
330
|
self._obj[flag_var_name] = flag_def
|
|
328
331
|
|
|
329
332
|
# use templater functionality if var defined with tuple
|
|
330
|
-
elif type(flag_def)
|
|
331
|
-
|
|
333
|
+
elif type(flag_def) is tuple:
|
|
332
334
|
attrs = flag_def[1]
|
|
333
335
|
|
|
334
336
|
if "flag_meanings" not in attrs:
|
|
@@ -415,7 +417,7 @@ class FlagAccessor(object):
|
|
|
415
417
|
|
|
416
418
|
return self._obj[list(flag_var_names)].data_vars
|
|
417
419
|
|
|
418
|
-
def _is_data_var(self, var_name:
|
|
420
|
+
def _is_data_var(self, var_name: Hashable) -> bool:
|
|
419
421
|
"""
|
|
420
422
|
Returns true if named dataset variable is an data variable (i.e. not a flag variable)
|
|
421
423
|
|
|
@@ -426,7 +428,7 @@ class FlagAccessor(object):
|
|
|
426
428
|
return True
|
|
427
429
|
return False
|
|
428
430
|
|
|
429
|
-
def _is_flag_var(self, var_name:
|
|
431
|
+
def _is_flag_var(self, var_name: Hashable) -> bool:
|
|
430
432
|
"""
|
|
431
433
|
Returns true if named dataset variable is an flag variable
|
|
432
434
|
|
obsarray/options.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""options - global obsarray options"""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
6
|
+
__all__ = ["set_options", "get_options"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# None means "not chosen explicitly" - obsarray then uses its default behaviour
|
|
10
|
+
OPTIONS: Dict[str, Any] = {"unc_conventions": None}
|
|
11
|
+
|
|
12
|
+
_VALID_VALUES = {"unc_conventions": ("v0.1",)}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class set_options:
|
|
16
|
+
"""
|
|
17
|
+
Sets obsarray global options, either for the rest of the session or within a ``with`` block
|
|
18
|
+
|
|
19
|
+
Available options:
|
|
20
|
+
|
|
21
|
+
* ``unc_conventions`` - version of the `UNC specification
|
|
22
|
+
<https://comet-toolkit.github.io/unc_website/>`_ obsarray follows when writing uncertainty metadata.
|
|
23
|
+
Currently the only supported value is ``"v0.1"`` (obsarray's current behaviour). Setting it explicitly
|
|
24
|
+
also silences the :py:class:`~obsarray.conventions.UNCConventionsWarning` about the upcoming change of
|
|
25
|
+
default to draft v0.2.
|
|
26
|
+
|
|
27
|
+
.. code-block:: python
|
|
28
|
+
|
|
29
|
+
import obsarray
|
|
30
|
+
|
|
31
|
+
# for the rest of the session
|
|
32
|
+
obsarray.set_options(unc_conventions="v0.1")
|
|
33
|
+
|
|
34
|
+
# or only within a block
|
|
35
|
+
with obsarray.set_options(unc_conventions="v0.1"):
|
|
36
|
+
ds.unc["temperature"]["u_noise"] = (["x"], u_noise, {"units": "K"})
|
|
37
|
+
|
|
38
|
+
:param kwargs: option names and their new values
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, **kwargs: Any):
|
|
42
|
+
self._old: Dict[str, Any] = {}
|
|
43
|
+
|
|
44
|
+
for name, value in kwargs.items():
|
|
45
|
+
if name not in OPTIONS:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"{name!r} is not an obsarray option, must be one of {sorted(OPTIONS)}"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
if value not in _VALID_VALUES[name]:
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"{value!r} is not a valid value for {name!r}, must be one of "
|
|
53
|
+
f"{list(_VALID_VALUES[name])}"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
self._old[name] = OPTIONS[name]
|
|
57
|
+
|
|
58
|
+
OPTIONS.update(kwargs)
|
|
59
|
+
|
|
60
|
+
def __enter__(self) -> None:
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
def __exit__(self, *exc_info: Any) -> None:
|
|
64
|
+
OPTIONS.update(self._old)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_options() -> Dict[str, Any]:
|
|
68
|
+
"""
|
|
69
|
+
Returns a copy of the current obsarray global options
|
|
70
|
+
|
|
71
|
+
:return: option names and their values (``None`` if not set explicitly)
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
return dict(OPTIONS)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
pass
|
|
@@ -6,7 +6,9 @@ import string
|
|
|
6
6
|
from copy import deepcopy
|
|
7
7
|
import xarray
|
|
8
8
|
import numpy
|
|
9
|
+
from numpy.typing import DTypeLike
|
|
9
10
|
from typing import Union, Optional, List, Dict, Tuple
|
|
11
|
+
from obsarray.conventions import warn_unc_conventions
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
@@ -34,9 +36,9 @@ class DatasetUtil:
|
|
|
34
36
|
@staticmethod
|
|
35
37
|
def create_default_array(
|
|
36
38
|
dim_sizes: List[int],
|
|
37
|
-
dtype:
|
|
39
|
+
dtype: DTypeLike,
|
|
38
40
|
dim_names: Optional[List[str]] = None,
|
|
39
|
-
fill_value: Optional[Union[int, float]] = None,
|
|
41
|
+
fill_value: Optional[Union[int, float, numpy.number]] = None,
|
|
40
42
|
) -> xarray.DataArray:
|
|
41
43
|
"""
|
|
42
44
|
Return default empty xarray DataArray
|
|
@@ -66,10 +68,10 @@ class DatasetUtil:
|
|
|
66
68
|
@staticmethod
|
|
67
69
|
def create_variable(
|
|
68
70
|
dim_sizes: List[int],
|
|
69
|
-
dtype:
|
|
71
|
+
dtype: DTypeLike,
|
|
70
72
|
dim_names: Optional[List[str]] = None,
|
|
71
|
-
attributes: Dict = None,
|
|
72
|
-
fill_value: Optional[Union[int, float]] = None,
|
|
73
|
+
attributes: Optional[Dict] = None,
|
|
74
|
+
fill_value: Optional[Union[int, float, numpy.number]] = None,
|
|
73
75
|
) -> xarray.Variable:
|
|
74
76
|
"""
|
|
75
77
|
Return default empty xarray Variable
|
|
@@ -107,7 +109,7 @@ class DatasetUtil:
|
|
|
107
109
|
@staticmethod
|
|
108
110
|
def create_unc_variable(
|
|
109
111
|
dim_sizes: List[int],
|
|
110
|
-
dtype:
|
|
112
|
+
dtype: DTypeLike,
|
|
111
113
|
dim_names: List[str],
|
|
112
114
|
attributes: Optional[Dict] = None,
|
|
113
115
|
pdf_shape: str = "gaussian",
|
|
@@ -145,6 +147,8 @@ class DatasetUtil:
|
|
|
145
147
|
populated to the effect of this assumption.
|
|
146
148
|
"""
|
|
147
149
|
|
|
150
|
+
warn_unc_conventions()
|
|
151
|
+
|
|
148
152
|
# define uncertainty variable attributes, based on FIDUCEO Full FCDR definition (if required)
|
|
149
153
|
attributes = {} if attributes is None else attributes
|
|
150
154
|
|
|
@@ -315,7 +319,7 @@ class DatasetUtil:
|
|
|
315
319
|
|
|
316
320
|
@staticmethod
|
|
317
321
|
def add_flag_meaning_to_attrs(
|
|
318
|
-
attrs: dict, flag_meaning: str, dtype:
|
|
322
|
+
attrs: dict, flag_meaning: str, dtype: DTypeLike
|
|
319
323
|
) -> dict:
|
|
320
324
|
"""
|
|
321
325
|
Add new meaning to flag variable attributes
|
|
@@ -384,7 +388,7 @@ class DatasetUtil:
|
|
|
384
388
|
return updated_attrs
|
|
385
389
|
|
|
386
390
|
@staticmethod
|
|
387
|
-
def return_flags_dtype(n_masks: int) ->
|
|
391
|
+
def return_flags_dtype(n_masks: int) -> DTypeLike:
|
|
388
392
|
"""
|
|
389
393
|
Return required flags array data type
|
|
390
394
|
|
|
@@ -404,10 +408,10 @@ class DatasetUtil:
|
|
|
404
408
|
@staticmethod
|
|
405
409
|
def add_encoding(
|
|
406
410
|
variable: xarray.Variable,
|
|
407
|
-
dtype:
|
|
411
|
+
dtype: DTypeLike,
|
|
408
412
|
scale_factor: Optional[float] = 1.0,
|
|
409
413
|
offset: Optional[float] = 0.0,
|
|
410
|
-
fill_value: Optional[Union[int, float]] = None,
|
|
414
|
+
fill_value: Optional[Union[int, float, numpy.number]] = None,
|
|
411
415
|
chunksizes: Optional[float] = None,
|
|
412
416
|
):
|
|
413
417
|
"""
|
|
@@ -442,7 +446,9 @@ class DatasetUtil:
|
|
|
442
446
|
variable.encoding = encoding_dict
|
|
443
447
|
|
|
444
448
|
@staticmethod
|
|
445
|
-
def get_default_fill_value(
|
|
449
|
+
def get_default_fill_value(
|
|
450
|
+
dtype: DTypeLike,
|
|
451
|
+
) -> Optional[Union[int, float, numpy.number]]:
|
|
446
452
|
"""
|
|
447
453
|
Returns default fill_value for given data type
|
|
448
454
|
|
|
@@ -469,6 +475,7 @@ class DatasetUtil:
|
|
|
469
475
|
return numpy.float32(9.96921e36)
|
|
470
476
|
elif dtype == numpy.float64:
|
|
471
477
|
return numpy.float64(9.969209968386869e36)
|
|
478
|
+
return None
|
|
472
479
|
|
|
473
480
|
@staticmethod
|
|
474
481
|
def _get_flag_encoding(da):
|
|
@@ -507,7 +514,7 @@ class DatasetUtil:
|
|
|
507
514
|
ds[flag_meaning] = DatasetUtil.create_variable(
|
|
508
515
|
list(da.shape), bool, dim_names=list(da.dims)
|
|
509
516
|
)
|
|
510
|
-
ds[flag_meaning] = (da & flag_mask).astype(bool)
|
|
517
|
+
ds[flag_meaning] = (da.astype(int) & flag_mask).astype(bool)
|
|
511
518
|
|
|
512
519
|
return ds
|
|
513
520
|
|
|
@@ -569,7 +576,7 @@ class DatasetUtil:
|
|
|
569
576
|
|
|
570
577
|
set_flags = DatasetUtil.unpack_flags(da)[flag_name]
|
|
571
578
|
|
|
572
|
-
if numpy.any(set_flags == True) and error_if_set:
|
|
579
|
+
if numpy.any(set_flags == True) and error_if_set: # noqa: E712
|
|
573
580
|
raise ValueError(
|
|
574
581
|
"Flag " + flag_name + " already set for variable " + da.name
|
|
575
582
|
)
|
|
@@ -597,7 +604,7 @@ class DatasetUtil:
|
|
|
597
604
|
|
|
598
605
|
set_flags = DatasetUtil.unpack_flags(da)[flag_name]
|
|
599
606
|
|
|
600
|
-
if numpy.any(set_flags == False) and error_if_unset:
|
|
607
|
+
if numpy.any(set_flags == False) and error_if_unset: # noqa: E712
|
|
601
608
|
raise ValueError(
|
|
602
609
|
"Flag " + flag_name + " already set for variable " + da.name
|
|
603
610
|
)
|
|
@@ -91,7 +91,6 @@ class TemplateUtil:
|
|
|
91
91
|
"""
|
|
92
92
|
|
|
93
93
|
for var_name in template.keys():
|
|
94
|
-
|
|
95
94
|
var = TemplateUtil._create_var(var_name, template[var_name], size)
|
|
96
95
|
|
|
97
96
|
ds[var_name] = var
|
|
@@ -131,6 +130,10 @@ class TemplateUtil:
|
|
|
131
130
|
# Create variable and add to dataset
|
|
132
131
|
if isinstance(dtype, str):
|
|
133
132
|
if dtype == "flag":
|
|
133
|
+
if attributes is None:
|
|
134
|
+
raise ValueError(
|
|
135
|
+
"Flag variable " + var_name + " must define flag_meanings"
|
|
136
|
+
)
|
|
134
137
|
flag_meanings = attributes.pop("flag_meanings")
|
|
135
138
|
variable = du.create_flags_variable(
|
|
136
139
|
dim_sizes,
|
|
@@ -173,7 +176,7 @@ class TemplateUtil:
|
|
|
173
176
|
"""
|
|
174
177
|
|
|
175
178
|
# Variable name must be type str
|
|
176
|
-
if type(variable_name)
|
|
179
|
+
if type(variable_name) is not str:
|
|
177
180
|
raise TypeError(
|
|
178
181
|
"Invalid variable name: " + str(variable_name) + " (must be string)"
|
|
179
182
|
)
|
|
@@ -471,7 +471,9 @@ class TestDatasetUtil(unittest.TestCase):
|
|
|
471
471
|
|
|
472
472
|
self.assertEqual(-127, DatasetUtil.get_default_fill_value(np.int8))
|
|
473
473
|
self.assertEqual(-32767, DatasetUtil.get_default_fill_value(np.int16))
|
|
474
|
-
self.assertEqual(
|
|
474
|
+
self.assertEqual(
|
|
475
|
+
np.array(-1).astype("uint16"), DatasetUtil.get_default_fill_value(np.uint16)
|
|
476
|
+
)
|
|
475
477
|
self.assertEqual(-2147483647, DatasetUtil.get_default_fill_value(np.int32))
|
|
476
478
|
self.assertEqual(
|
|
477
479
|
-9223372036854775806, DatasetUtil.get_default_fill_value(np.int64)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""test_conventions - tests for obsarray.conventions"""
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
import warnings
|
|
5
|
+
import numpy as np
|
|
6
|
+
import xarray as xr
|
|
7
|
+
from obsarray import UNCConventionsWarning, create_ds, get_options, set_options
|
|
8
|
+
from obsarray import conventions
|
|
9
|
+
from obsarray.options import OPTIONS
|
|
10
|
+
|
|
11
|
+
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
12
|
+
__all__ = []
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def create_obs_ds():
|
|
16
|
+
return xr.Dataset({"temperature": (["x"], np.ones(3), {"units": "K"})})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TestUNCConventionsWarning(unittest.TestCase):
|
|
20
|
+
def setUp(self):
|
|
21
|
+
# each test starts as a fresh session that hasn't warned yet
|
|
22
|
+
self.original_warned = conventions._warned
|
|
23
|
+
self.original_options = get_options()
|
|
24
|
+
conventions._warned = False
|
|
25
|
+
|
|
26
|
+
self.ds = create_obs_ds()
|
|
27
|
+
|
|
28
|
+
def tearDown(self):
|
|
29
|
+
conventions._warned = self.original_warned
|
|
30
|
+
OPTIONS.update(self.original_options)
|
|
31
|
+
|
|
32
|
+
def add_unc_var(self, unc_var="u_noise"):
|
|
33
|
+
self.ds.unc["temperature"][unc_var] = (["x"], np.ones(3) * 0.1, {"units": "K"})
|
|
34
|
+
|
|
35
|
+
def assertNoUNCConventionsWarning(self, func, *args, **kwargs):
|
|
36
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
37
|
+
warnings.simplefilter("always")
|
|
38
|
+
func(*args, **kwargs)
|
|
39
|
+
|
|
40
|
+
self.assertFalse(
|
|
41
|
+
[w for w in caught if issubclass(w.category, UNCConventionsWarning)]
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def test_is_future_warning(self):
|
|
45
|
+
# FutureWarning is shown to end users by default, unlike DeprecationWarning
|
|
46
|
+
self.assertTrue(issubclass(UNCConventionsWarning, FutureWarning))
|
|
47
|
+
|
|
48
|
+
def test_warns_on_adding_unc_var(self):
|
|
49
|
+
with self.assertWarns(UNCConventionsWarning):
|
|
50
|
+
self.add_unc_var()
|
|
51
|
+
|
|
52
|
+
def test_warns_on_adding_unc_var_dataarray(self):
|
|
53
|
+
unc = xr.DataArray(np.ones(3) * 0.1, dims=["x"], attrs={"units": "K"})
|
|
54
|
+
|
|
55
|
+
with self.assertWarns(UNCConventionsWarning):
|
|
56
|
+
self.ds.unc["temperature"]["u_noise"] = unc
|
|
57
|
+
|
|
58
|
+
def test_warns_on_template_unc_var(self):
|
|
59
|
+
template = {
|
|
60
|
+
"temperature": {
|
|
61
|
+
"dim": ["x"],
|
|
62
|
+
"dtype": np.float32,
|
|
63
|
+
"attributes": {"units": "K", "unc_comps": ["u_temperature"]},
|
|
64
|
+
},
|
|
65
|
+
"u_temperature": {
|
|
66
|
+
"dim": ["x"],
|
|
67
|
+
"dtype": np.float32,
|
|
68
|
+
"attributes": {
|
|
69
|
+
"units": "K",
|
|
70
|
+
"err_corr": [{"dim": "x", "form": "random", "params": []}],
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
with self.assertWarns(UNCConventionsWarning):
|
|
76
|
+
create_ds(template, {"x": 3})
|
|
77
|
+
|
|
78
|
+
def test_warns_once_per_session(self):
|
|
79
|
+
with self.assertWarns(UNCConventionsWarning):
|
|
80
|
+
self.add_unc_var("u_noise")
|
|
81
|
+
|
|
82
|
+
self.assertNoUNCConventionsWarning(self.add_unc_var, "u_calibration")
|
|
83
|
+
|
|
84
|
+
def test_no_warning_when_conventions_set(self):
|
|
85
|
+
set_options(unc_conventions="v0.1")
|
|
86
|
+
|
|
87
|
+
self.assertNoUNCConventionsWarning(self.add_unc_var)
|
|
88
|
+
|
|
89
|
+
def test_no_warning_within_set_options_block(self):
|
|
90
|
+
with set_options(unc_conventions="v0.1"):
|
|
91
|
+
self.assertNoUNCConventionsWarning(self.add_unc_var, "u_noise")
|
|
92
|
+
|
|
93
|
+
# not silenced for good - the next write outside the block still warns
|
|
94
|
+
with self.assertWarns(UNCConventionsWarning):
|
|
95
|
+
self.add_unc_var("u_calibration")
|
|
96
|
+
|
|
97
|
+
def test_no_warning_on_read(self):
|
|
98
|
+
with set_options(unc_conventions="v0.1"):
|
|
99
|
+
self.add_unc_var()
|
|
100
|
+
|
|
101
|
+
self.assertNoUNCConventionsWarning(
|
|
102
|
+
lambda: self.ds.unc["temperature"]["u_noise"].value
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def test_warning_points_at_calling_code(self):
|
|
106
|
+
user_code = compile(
|
|
107
|
+
'ds.unc["temperature"]["u_noise"] = (["x"], u, {"units": "K"})',
|
|
108
|
+
"user_script.py",
|
|
109
|
+
"exec",
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
with self.assertWarns(UNCConventionsWarning) as cm:
|
|
113
|
+
exec(user_code, {"ds": self.ds, "u": np.ones(3) * 0.1})
|
|
114
|
+
|
|
115
|
+
self.assertEqual(cm.filename, "user_script.py")
|
|
116
|
+
|
|
117
|
+
def test_message_explains_how_to_silence(self):
|
|
118
|
+
with self.assertWarns(UNCConventionsWarning) as cm:
|
|
119
|
+
self.add_unc_var()
|
|
120
|
+
|
|
121
|
+
message = str(cm.warning)
|
|
122
|
+
self.assertIn('set_options(unc_conventions="v0.1")', message)
|
|
123
|
+
self.assertIn(conventions.UNC_CONVENTIONS_DOCS_URL, message)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
unittest.main()
|