obsarray 0.2.5__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {obsarray-0.2.5 → obsarray-0.2.7}/.gitignore +1 -1
- {obsarray-0.2.5/obsarray.egg-info → obsarray-0.2.7}/PKG-INFO +1 -1
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/__init__.py +1 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/_version.py +3 -3
- obsarray-0.2.7/obsarray/err_corr.py +298 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/flag_accessor.py +6 -6
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dataset_util.py +12 -8
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dswriter.py +2 -1
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/template_util.py +1 -0
- obsarray-0.2.7/obsarray/test/test_err_corr_forms.py +251 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/test_flag_accessor.py +7 -5
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/test_unc_accessor.py +36 -18
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/unc_accessor.py +22 -27
- {obsarray-0.2.5 → obsarray-0.2.7/obsarray.egg-info}/PKG-INFO +1 -1
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/SOURCES.txt +0 -2
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/requires.txt +1 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/setup.py +2 -1
- obsarray-0.2.5/obsarray/err_corr.py +0 -218
- obsarray-0.2.5/obsarray/templater/dsdoc.py +0 -302
- obsarray-0.2.5/obsarray/templater/tests/test_dsdoc.py +0 -312
- obsarray-0.2.5/obsarray/test/test_err_corr_forms.py +0 -156
- {obsarray-0.2.5 → obsarray-0.2.7}/.pre-commit-config.yaml +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/LICENSE +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/MANIFEST.in +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/README.md +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/docs/Makefile +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/docs/conf.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/docs/index.rst +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/docs/make.bat +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dstemplater.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/template_util_jon.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_dataset_util.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_dswriter.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_template_util.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/utils.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/dependency_links.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/top_level.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/.gitignore +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/QualityDocumentation.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/CookiecutterMacros.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/acronyms.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/history.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/metadata.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/pythoninputstyle.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sil2.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sil3.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sources.bib +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sources.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/countpdfpages.sh +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/footnotehyper-sphinx.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/obsarray_QF-59.docm +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/obsarray_requirements.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16a.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16b.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16c.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16d.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16e.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16f.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/softwareRequirements.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinx.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxcyrillic.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxhighlight.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxhowto.cls +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmanual.cls +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmessages.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmulticell.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/uml/copy your design png file into this folder.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/setup.cfg +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/tox.ini +0 -0
- {obsarray-0.2.5 → obsarray-0.2.7}/versioneer.py +0 -0
|
@@ -8,6 +8,7 @@ from obsarray import unc_accessor, flag_accessor
|
|
|
8
8
|
from obsarray.err_corr import err_corr_forms
|
|
9
9
|
from obsarray.templater.template_util import create_ds
|
|
10
10
|
from obsarray.templater.dstemplater import DSTemplater
|
|
11
|
+
from obsarray.templater.dswriter import DSWriter
|
|
11
12
|
|
|
12
13
|
__version__ = get_versions()["version"]
|
|
13
14
|
del get_versions
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2024-
|
|
11
|
+
"date": "2024-04-02T11:59:05+0100",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "v0.2.
|
|
14
|
+
"full-revisionid": "4f32df4b9da131ab66376b319a8036975d891233",
|
|
15
|
+
"version": "v0.2.7"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""err_corr_forms - module for the defintion of error-correlation parameterisation forms"""
|
|
2
|
+
|
|
3
|
+
import abc
|
|
4
|
+
from typing import Callable, Type, Union, List
|
|
5
|
+
import numpy as np
|
|
6
|
+
from comet_maths.linear_algebra.matrix_conversion import expand_errcorr_dims
|
|
7
|
+
|
|
8
|
+
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
9
|
+
__all__ = ["err_corr_forms", "register_err_corr_form", "BaseErrCorrForm"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ErrCorrForms:
|
|
13
|
+
"""
|
|
14
|
+
Container for error-correlation parameterisation form definition objects
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
self._forms = {}
|
|
19
|
+
|
|
20
|
+
def __setitem__(self, form_name, form_cls):
|
|
21
|
+
if not issubclass(form_cls, BaseErrCorrForm):
|
|
22
|
+
raise TypeError(
|
|
23
|
+
"form must be subclass of " + str(BaseErrCorrForm.__class__)
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
self._forms[form_name] = form_cls
|
|
27
|
+
|
|
28
|
+
def __getitem__(self, name):
|
|
29
|
+
return self._forms[name]
|
|
30
|
+
|
|
31
|
+
def __delitem__(self, name):
|
|
32
|
+
del self._forms[name]
|
|
33
|
+
|
|
34
|
+
def keys(self):
|
|
35
|
+
return self._forms.keys()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
err_corr_forms = ErrCorrForms()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# placeholder currently
|
|
42
|
+
class BaseErrCorrForm(abc.ABC):
|
|
43
|
+
"""
|
|
44
|
+
Base class for error-correlation parameterisation form defintions
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
is_random = False
|
|
48
|
+
is_systematic = False
|
|
49
|
+
|
|
50
|
+
def __init__(self, xarray_obj, unc_var_name, dims, params, units):
|
|
51
|
+
self._obj = xarray_obj
|
|
52
|
+
self._unc_var_name = unc_var_name
|
|
53
|
+
self.dims = dims if isinstance(dims, list) else [dims]
|
|
54
|
+
self.params = params if isinstance(params, list) else [params]
|
|
55
|
+
self.units = units
|
|
56
|
+
|
|
57
|
+
def __str__(self):
|
|
58
|
+
"""Custom __str__"""
|
|
59
|
+
|
|
60
|
+
return self.form + " " + str(self.params)
|
|
61
|
+
|
|
62
|
+
def __repr__(self):
|
|
63
|
+
"""Custom __repr__"""
|
|
64
|
+
return str(self)
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
@abc.abstractmethod
|
|
68
|
+
def form(self) -> str:
|
|
69
|
+
"""Form name"""
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
def get_varshape_errcorr(self):
|
|
73
|
+
"""
|
|
74
|
+
return shape of uncertainty variable, including only dimensions which are included in the current error correlation form.
|
|
75
|
+
|
|
76
|
+
:return: shape of included dimensions
|
|
77
|
+
"""
|
|
78
|
+
all_dims = self._obj[self._unc_var_name].dims
|
|
79
|
+
all_dims_sizes = self._obj.sizes
|
|
80
|
+
|
|
81
|
+
return tuple([all_dims_sizes[dim] for dim in all_dims if dim in self.dims])
|
|
82
|
+
|
|
83
|
+
def get_sliced_dim_sizes_uncvar(self, sli: tuple) -> dict:
|
|
84
|
+
"""
|
|
85
|
+
Return dictionary with sizes of sliced dimensions of unc variable, including all dimensions.
|
|
86
|
+
|
|
87
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
88
|
+
:return: dictionary with shape of included sliced dimensions
|
|
89
|
+
"""
|
|
90
|
+
uncvar_dims = self._obj[self._unc_var_name][sli].dims
|
|
91
|
+
uncvar_shape = self._obj[self._unc_var_name][sli].shape
|
|
92
|
+
return {
|
|
93
|
+
uncvar_dims[idim]: uncvar_shape[idim] for idim in range(len(uncvar_dims))
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
def get_sliced_dim_sizes_errcorr(self, sli: tuple) -> dict:
|
|
97
|
+
"""
|
|
98
|
+
Return dictionary with sizes of sliced dimensions of unc variable, including only dimensions which are
|
|
99
|
+
included in the current error correlation form.
|
|
100
|
+
|
|
101
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
102
|
+
:return: dictionary with shape of included sliced dimensions
|
|
103
|
+
"""
|
|
104
|
+
uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
|
|
105
|
+
sliced_dims = self.get_sliced_dims_errcorr(sli)
|
|
106
|
+
|
|
107
|
+
return {dim: uncvar_sizes[dim] for dim in sliced_dims}
|
|
108
|
+
|
|
109
|
+
def get_sliced_dims_errcorr(self, sli: tuple) -> list:
|
|
110
|
+
"""
|
|
111
|
+
Return dimensions which are within the slice and included in the current error correlation form.
|
|
112
|
+
|
|
113
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
114
|
+
:return: list with sliced dimensions
|
|
115
|
+
"""
|
|
116
|
+
all_dims = self._obj[self._unc_var_name].dims
|
|
117
|
+
return [
|
|
118
|
+
all_dims[idim]
|
|
119
|
+
for idim in range(len(all_dims))
|
|
120
|
+
if (isinstance(sli[idim], slice) and all_dims[idim] in self.dims)
|
|
121
|
+
]
|
|
122
|
+
|
|
123
|
+
def get_sliced_shape_errcorr(self, sli: tuple) -> tuple:
|
|
124
|
+
"""
|
|
125
|
+
return shape of sliced uncertainty variable, including only dimensions which are included in the current error correlation form.
|
|
126
|
+
|
|
127
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
128
|
+
:return: shape of included sliced dimensions
|
|
129
|
+
"""
|
|
130
|
+
uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
|
|
131
|
+
sliced_dims = self.get_sliced_dims_errcorr(sli)
|
|
132
|
+
|
|
133
|
+
return tuple([uncvar_sizes[dim] for dim in sliced_dims])
|
|
134
|
+
|
|
135
|
+
def slice_errcorr_matrix(self, err_corr_matrix, variable_shape, sli) -> np.ndarray:
|
|
136
|
+
"""
|
|
137
|
+
Slice the provided error correlation matrix (typically the error correlation matrix of the
|
|
138
|
+
BaseErrCorrForm) using the
|
|
139
|
+
|
|
140
|
+
:param err_corr_matrix: error correlation matrix to be sliced
|
|
141
|
+
:param variable_shape: tuple with the length of the dimensions in the error correlation matrix (in correct order for flattening)
|
|
142
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
143
|
+
:return: sliced error correlation matrix
|
|
144
|
+
"""
|
|
145
|
+
mask_array = np.ones(variable_shape, dtype=bool)
|
|
146
|
+
mask_array[sli] = False
|
|
147
|
+
|
|
148
|
+
return np.delete(
|
|
149
|
+
np.delete(err_corr_matrix, mask_array.ravel(), 0), mask_array.ravel(), 1
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
@abc.abstractmethod
|
|
153
|
+
def build_matrix(self, sli: Union[np.ndarray, tuple]) -> np.ndarray:
|
|
154
|
+
"""
|
|
155
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
156
|
+
in this parameterisation
|
|
157
|
+
|
|
158
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
159
|
+
|
|
160
|
+
:return: populated error-correlation matrix
|
|
161
|
+
"""
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
def build_dot_matrix(self, sli: Union[np.ndarray, tuple]) -> np.ndarray:
|
|
165
|
+
"""
|
|
166
|
+
Returns expanded error correlation matrix for use in dot product with error correlation
|
|
167
|
+
in other dimensions.
|
|
168
|
+
|
|
169
|
+
The (sliced) error correlation matrix for this BaseErrCorrForm is expanded from its current
|
|
170
|
+
(sliced) dimensions (which often don't include all dimensions of the associated uncertainty
|
|
171
|
+
variable) to the dimensions of the full (sliced) error correlation (i.e. all dimensions of
|
|
172
|
+
the uncertainty).
|
|
173
|
+
The returned matrix is not meaningfull unless combined in a dot product with the expanded
|
|
174
|
+
matrices of other error correlation matrices (together spanning all uncertainty dimensions).
|
|
175
|
+
|
|
176
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
177
|
+
:return: expanded matrix for use in dot product with error correlation in other dimensions.
|
|
178
|
+
"""
|
|
179
|
+
return expand_errcorr_dims(
|
|
180
|
+
in_corr=self.build_matrix(sli),
|
|
181
|
+
in_dim=self.get_sliced_dims_errcorr(sli),
|
|
182
|
+
out_dim=list(self._obj[self._unc_var_name][sli].dims),
|
|
183
|
+
dim_sizes=self.get_sliced_dim_sizes_uncvar(sli),
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def register_err_corr_form(form_name: str) -> Callable:
|
|
188
|
+
"""
|
|
189
|
+
Decorator for registering error-correlation parmaterisation form definition to package definition container
|
|
190
|
+
|
|
191
|
+
:param form_name: name of error-correlation parmaterisation form e.g. "random"
|
|
192
|
+
:return: decorator function
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
def decorator(form_cls: Type) -> Type:
|
|
196
|
+
err_corr_forms[form_name] = form_cls
|
|
197
|
+
return form_cls
|
|
198
|
+
|
|
199
|
+
return decorator
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
@register_err_corr_form("random")
|
|
203
|
+
class RandomCorrelation(BaseErrCorrForm):
|
|
204
|
+
|
|
205
|
+
form = "random"
|
|
206
|
+
is_random = True
|
|
207
|
+
|
|
208
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
209
|
+
"""
|
|
210
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
211
|
+
in this parameterisation
|
|
212
|
+
|
|
213
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
214
|
+
|
|
215
|
+
:return: populated error-correlation matrix
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
# evaluate correlation over matrices in form defintion
|
|
219
|
+
dim_lens = self.get_sliced_shape_errcorr(sli)
|
|
220
|
+
n_elems = int(np.prod(dim_lens))
|
|
221
|
+
|
|
222
|
+
submatrix = np.eye(n_elems)
|
|
223
|
+
|
|
224
|
+
return submatrix
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@register_err_corr_form("systematic")
|
|
228
|
+
class SystematicCorrelation(BaseErrCorrForm):
|
|
229
|
+
|
|
230
|
+
form = "systematic"
|
|
231
|
+
is_systematic = True
|
|
232
|
+
|
|
233
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
234
|
+
"""
|
|
235
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
236
|
+
in this parameterisation
|
|
237
|
+
|
|
238
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
239
|
+
|
|
240
|
+
:return: populated error-correlation matrix
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
# evaluate correlation over matrices in form defintion
|
|
244
|
+
dim_lens = self.get_sliced_shape_errcorr(sli)
|
|
245
|
+
n_elems = int(np.prod(dim_lens))
|
|
246
|
+
|
|
247
|
+
submatrix = np.ones((n_elems, n_elems))
|
|
248
|
+
|
|
249
|
+
return submatrix
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@register_err_corr_form("err_corr_matrix")
|
|
253
|
+
class ErrCorrMatrixCorrelation(BaseErrCorrForm):
|
|
254
|
+
|
|
255
|
+
form = "err_corr_matrix"
|
|
256
|
+
|
|
257
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
258
|
+
"""
|
|
259
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
260
|
+
in this parameterisation
|
|
261
|
+
|
|
262
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
263
|
+
|
|
264
|
+
:return: populated error-correlation matrix
|
|
265
|
+
"""
|
|
266
|
+
all_dims = self._obj[self._unc_var_name].dims
|
|
267
|
+
|
|
268
|
+
sli_submatrix = tuple(
|
|
269
|
+
[sli[i] for i in range(len(all_dims)) if all_dims[i] in self.dims]
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
submatrix = self.slice_errcorr_matrix(
|
|
273
|
+
self._obj[self.params[0]], self.get_varshape_errcorr(), sli_submatrix
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
return submatrix
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@register_err_corr_form("ensemble")
|
|
280
|
+
class EnsembleCorrelation(BaseErrCorrForm):
|
|
281
|
+
|
|
282
|
+
form = "ensemble"
|
|
283
|
+
|
|
284
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
285
|
+
"""
|
|
286
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
287
|
+
in this parameterisation
|
|
288
|
+
|
|
289
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
290
|
+
|
|
291
|
+
:return: populated error-correlation matrix
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
raise NotImplementedError
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
if __name__ == "__main__":
|
|
298
|
+
pass
|
|
@@ -194,12 +194,12 @@ class FlagVariable:
|
|
|
194
194
|
)
|
|
195
195
|
|
|
196
196
|
if flag_meaning not in flag_meanings:
|
|
197
|
-
self._obj[
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
197
|
+
self._obj[self._flag_var_name].attrs = (
|
|
198
|
+
DatasetUtil.add_flag_meaning_to_attrs(
|
|
199
|
+
self._obj[self._flag_var_name].attrs,
|
|
200
|
+
flag_meaning,
|
|
201
|
+
self._obj[self._flag_var_name].dtype,
|
|
202
|
+
)
|
|
203
203
|
)
|
|
204
204
|
|
|
205
205
|
self[flag_meaning][:] = flag_value
|
|
@@ -259,8 +259,8 @@ class DatasetUtil:
|
|
|
259
259
|
attributes=attributes,
|
|
260
260
|
)
|
|
261
261
|
|
|
262
|
-
#initialise flags to zero (instead of fillvalue)
|
|
263
|
-
variable.values=0*variable.values
|
|
262
|
+
# initialise flags to zero (instead of fillvalue)
|
|
263
|
+
variable.values = 0 * variable.values
|
|
264
264
|
|
|
265
265
|
# add flag attributes
|
|
266
266
|
variable.attrs.update(DatasetUtil.pack_flag_attrs(meanings))
|
|
@@ -290,7 +290,7 @@ class DatasetUtil:
|
|
|
290
290
|
)
|
|
291
291
|
|
|
292
292
|
if flag_masks is None:
|
|
293
|
-
flag_masks = [2
|
|
293
|
+
flag_masks = [2**i for i in range(0, n_masks)]
|
|
294
294
|
|
|
295
295
|
flag_attrs["flag_masks"] = str(flag_masks)[1:-1]
|
|
296
296
|
|
|
@@ -333,7 +333,7 @@ class DatasetUtil:
|
|
|
333
333
|
|
|
334
334
|
# check if variable for available flags
|
|
335
335
|
max_n_flags = numpy.iinfo(dtype).bits
|
|
336
|
-
all_flag_masks = [2
|
|
336
|
+
all_flag_masks = [2**i for i in range(0, max_n_flags)]
|
|
337
337
|
available_flag_masks = list(set(all_flag_masks) - set(flag_masks))
|
|
338
338
|
|
|
339
339
|
if not available_flag_masks:
|
|
@@ -434,6 +434,10 @@ class DatasetUtil:
|
|
|
434
434
|
|
|
435
435
|
if fill_value is not None:
|
|
436
436
|
encoding_dict.update({"_FillValue": fill_value})
|
|
437
|
+
else:
|
|
438
|
+
encoding_dict.update(
|
|
439
|
+
{"_FillValue": DatasetUtil.get_default_fill_value(dtype)}
|
|
440
|
+
)
|
|
437
441
|
|
|
438
442
|
variable.encoding = encoding_dict
|
|
439
443
|
|
|
@@ -524,8 +528,8 @@ class DatasetUtil:
|
|
|
524
528
|
|
|
525
529
|
flags_ds = DatasetUtil.unpack_flags(da)
|
|
526
530
|
|
|
527
|
-
flags = flags if flags is not None else flags_ds.
|
|
528
|
-
mask_flags = [flags_ds[flag].values for flag in flags]
|
|
531
|
+
flags = flags if flags is not None else flags_ds.data_vars
|
|
532
|
+
mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
|
|
529
533
|
|
|
530
534
|
return numpy.logical_or.reduce(mask_flags)
|
|
531
535
|
|
|
@@ -546,8 +550,8 @@ class DatasetUtil:
|
|
|
546
550
|
|
|
547
551
|
flags_ds = DatasetUtil.unpack_flags(da)
|
|
548
552
|
|
|
549
|
-
flags = flags if flags is not None else flags_ds.
|
|
550
|
-
mask_flags = [flags_ds[flag].values for flag in flags]
|
|
553
|
+
flags = flags if flags is not None else flags_ds.data_vars
|
|
554
|
+
mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
|
|
551
555
|
|
|
552
556
|
return numpy.logical_and.reduce(mask_flags)
|
|
553
557
|
|
|
@@ -81,7 +81,8 @@ class DSWriter:
|
|
|
81
81
|
)
|
|
82
82
|
}
|
|
83
83
|
)
|
|
84
|
-
ds[var_name].attrs
|
|
84
|
+
if "_FillValue" in ds[var_name].attrs:
|
|
85
|
+
ds[var_name].attrs.pop("_FillValue")
|
|
85
86
|
encoding.update({var_name: var_encoding})
|
|
86
87
|
|
|
87
88
|
ds.to_netcdf(path, format="netCDF4", engine="netcdf4", encoding=encoding)
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""test_err_corr_forms - tests for obsarray.err_corr_forms"""
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
import numpy as np
|
|
5
|
+
from obsarray.err_corr import (
|
|
6
|
+
BaseErrCorrForm,
|
|
7
|
+
ErrCorrForms,
|
|
8
|
+
RandomCorrelation,
|
|
9
|
+
SystematicCorrelation,
|
|
10
|
+
ErrCorrMatrixCorrelation,
|
|
11
|
+
)
|
|
12
|
+
from obsarray.test.test_unc_accessor import create_ds
|
|
13
|
+
|
|
14
|
+
__author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
|
|
15
|
+
__all__ = []
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TestErrCorrForms(unittest.TestCase):
|
|
19
|
+
def setUp(self):
|
|
20
|
+
class ErrCorrFormDef(BaseErrCorrForm):
|
|
21
|
+
def form(self):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
self.ErrCorrFormDef = ErrCorrFormDef
|
|
25
|
+
|
|
26
|
+
def test___setitem__(self):
|
|
27
|
+
err_corr_forms = ErrCorrForms()
|
|
28
|
+
err_corr_forms["test"] = self.ErrCorrFormDef
|
|
29
|
+
|
|
30
|
+
self.assertCountEqual(list(err_corr_forms._forms.keys()), ["test"])
|
|
31
|
+
|
|
32
|
+
def test___setitem___not_valid_cls(self):
|
|
33
|
+
class InvalidErrCorrFormDef:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
err_corr_forms = ErrCorrForms()
|
|
37
|
+
self.assertRaises(
|
|
38
|
+
TypeError, err_corr_forms.__setitem__, "test", InvalidErrCorrFormDef
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def test___getitem__(self):
|
|
42
|
+
err_corr_forms = ErrCorrForms()
|
|
43
|
+
err_corr_forms._forms["test"] = "test"
|
|
44
|
+
self.assertEqual("test", err_corr_forms["test"])
|
|
45
|
+
|
|
46
|
+
def test___delitem__(self):
|
|
47
|
+
err_corr_forms = ErrCorrForms()
|
|
48
|
+
err_corr_forms._forms["test"] = "test"
|
|
49
|
+
|
|
50
|
+
del err_corr_forms._forms["test"]
|
|
51
|
+
|
|
52
|
+
self.assertTrue("test" not in err_corr_forms._forms)
|
|
53
|
+
|
|
54
|
+
def test_keys(self):
|
|
55
|
+
err_corr_forms = ErrCorrForms()
|
|
56
|
+
err_corr_forms._forms["test"] = "test"
|
|
57
|
+
|
|
58
|
+
self.assertCountEqual(["test"], err_corr_forms.keys())
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class TestBaseErrCorrForm(unittest.TestCase):
|
|
62
|
+
def setUp(self) -> None:
|
|
63
|
+
self.ds = create_ds()
|
|
64
|
+
|
|
65
|
+
class BasicErrCorrForm(BaseErrCorrForm):
|
|
66
|
+
form = "basic"
|
|
67
|
+
|
|
68
|
+
def build_matrix(self, sli):
|
|
69
|
+
full_matrix = np.arange(144).reshape((12, 12))
|
|
70
|
+
return self.slice_errcorr_matrix(full_matrix, (2, 2, 3), sli)
|
|
71
|
+
|
|
72
|
+
self.BasicErrCorrForm = BasicErrCorrForm
|
|
73
|
+
|
|
74
|
+
def test_get_varshape_errcorr(self):
|
|
75
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
76
|
+
self.ds, "u_ran_temperature", ["x", "y", "time"], [], []
|
|
77
|
+
)
|
|
78
|
+
shape = basicerrcorr.get_varshape_errcorr()
|
|
79
|
+
np.testing.assert_equal(shape, (2, 2, 3))
|
|
80
|
+
|
|
81
|
+
def test_slice_full_cov_full(self):
|
|
82
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
83
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
full_matrix = np.arange(144).reshape((12, 12))
|
|
87
|
+
slice_matrix = basicerrcorr.build_matrix(
|
|
88
|
+
(slice(None), slice(None), slice(None))
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
np.testing.assert_equal(full_matrix, slice_matrix)
|
|
92
|
+
|
|
93
|
+
def test_get_sliced_dim_sizes_uncvar(self):
|
|
94
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
95
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
96
|
+
)
|
|
97
|
+
dim_sizes = basicerrcorr.get_sliced_dim_sizes_uncvar(
|
|
98
|
+
(slice(None), 0, slice(0, 2, 1))
|
|
99
|
+
)
|
|
100
|
+
assert dim_sizes == {"x": 2, "time": 2}
|
|
101
|
+
|
|
102
|
+
def test_get_sliced_dim_sizes_errcorr(self):
|
|
103
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
104
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
105
|
+
)
|
|
106
|
+
dim_sizes = basicerrcorr.get_sliced_dim_sizes_errcorr(
|
|
107
|
+
(slice(None), 0, slice(0, 2, 1))
|
|
108
|
+
)
|
|
109
|
+
assert dim_sizes == {"x": 2}
|
|
110
|
+
|
|
111
|
+
def test_get_sliced_dims_errcorr(self):
|
|
112
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
113
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
114
|
+
)
|
|
115
|
+
dims = basicerrcorr.get_sliced_dims_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
116
|
+
assert dims == ["x"]
|
|
117
|
+
|
|
118
|
+
def test_get_sliced_shape_errcorr(self):
|
|
119
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
120
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
121
|
+
)
|
|
122
|
+
shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
123
|
+
assert shape == (2,)
|
|
124
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
125
|
+
self.ds, "u_ran_temperature", ["x", "time"], [], []
|
|
126
|
+
)
|
|
127
|
+
shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
128
|
+
assert shape == (2, 2)
|
|
129
|
+
|
|
130
|
+
def test_slice_errcorr_matrix(self):
|
|
131
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
132
|
+
self.ds, "u_ran_temperature", ["x", "y", "z"], [], []
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
full_matrix = np.arange(144).reshape((12, 12))
|
|
136
|
+
slice_matrix = basicerrcorr.slice_errcorr_matrix(
|
|
137
|
+
full_matrix, (2, 2, 3), (slice(None), slice(None), 0)
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
exp_slice_matrix = np.array(
|
|
141
|
+
[[0, 3, 6, 9], [36, 39, 42, 45], [72, 75, 78, 81], [108, 111, 114, 117]]
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
np.testing.assert_equal(slice_matrix, exp_slice_matrix)
|
|
145
|
+
|
|
146
|
+
def test_slice_full_cov_slice(self):
|
|
147
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
148
|
+
self.ds, "u_ran_temperature", ["x", "y", "z"], [], []
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
slice_matrix = basicerrcorr.build_dot_matrix((slice(None), slice(None), 0))
|
|
152
|
+
|
|
153
|
+
exp_slice_matrix = np.array(
|
|
154
|
+
[[0, 3, 6, 9], [36, 39, 42, 45], [72, 75, 78, 81], [108, 111, 114, 117]]
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
np.testing.assert_equal(slice_matrix, exp_slice_matrix)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class TestRandomUnc(unittest.TestCase):
|
|
161
|
+
def setUp(self) -> None:
|
|
162
|
+
self.ds = create_ds()
|
|
163
|
+
|
|
164
|
+
def test_build_matrix_1stdim(self):
|
|
165
|
+
rc = RandomCorrelation(self.ds, "u_ran_temperature", ["x"], [], [])
|
|
166
|
+
|
|
167
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
168
|
+
|
|
169
|
+
np.testing.assert_equal(ecrm, np.eye(12))
|
|
170
|
+
|
|
171
|
+
def test_build_matrix_2nddim(self):
|
|
172
|
+
rc = RandomCorrelation(self.ds, "u_ran_temperature", ["y"], [], [])
|
|
173
|
+
|
|
174
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
175
|
+
|
|
176
|
+
np.testing.assert_equal(ecrm, np.eye(12))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class TestSystematicUnc(unittest.TestCase):
|
|
180
|
+
def setUp(self) -> None:
|
|
181
|
+
self.ds = create_ds()
|
|
182
|
+
|
|
183
|
+
def build_matrix_1stdim(self):
|
|
184
|
+
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["x"], [], [])
|
|
185
|
+
|
|
186
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
187
|
+
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
188
|
+
|
|
189
|
+
return ecrm
|
|
190
|
+
|
|
191
|
+
def build_matrix_2nddim(self):
|
|
192
|
+
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["y"], [], [])
|
|
193
|
+
|
|
194
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
195
|
+
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
196
|
+
|
|
197
|
+
return ecrm
|
|
198
|
+
|
|
199
|
+
def build_matrix_3ddim(self):
|
|
200
|
+
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["time"], [], [])
|
|
201
|
+
|
|
202
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
203
|
+
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
204
|
+
|
|
205
|
+
return ecrm
|
|
206
|
+
|
|
207
|
+
def test_build_dot_matrix(self):
|
|
208
|
+
x = self.build_matrix_1stdim()
|
|
209
|
+
y = self.build_matrix_2nddim()
|
|
210
|
+
time = self.build_matrix_3ddim()
|
|
211
|
+
print(x.dot(y), x, y)
|
|
212
|
+
np.testing.assert_equal((x.dot(y)).dot(time), np.ones((12, 12)))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class TestErrCorrMatrixCorrelation(unittest.TestCase):
|
|
216
|
+
def setUp(self) -> None:
|
|
217
|
+
self.ds = create_ds()
|
|
218
|
+
|
|
219
|
+
def test_build_matrix_full(self):
|
|
220
|
+
ec = ErrCorrMatrixCorrelation(
|
|
221
|
+
self.ds,
|
|
222
|
+
"u_str_temperature",
|
|
223
|
+
["x", "time"],
|
|
224
|
+
["err_corr_str_temperature"],
|
|
225
|
+
[],
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
ecrm = ec.build_matrix((slice(None), slice(None), slice(None)))
|
|
229
|
+
np.testing.assert_equal(ecrm, np.ones((6, 6)))
|
|
230
|
+
|
|
231
|
+
def test_build_matrix_sliced(self):
|
|
232
|
+
ec = ErrCorrMatrixCorrelation(
|
|
233
|
+
self.ds,
|
|
234
|
+
"u_str_temperature",
|
|
235
|
+
["x", "time"],
|
|
236
|
+
["err_corr_str_temperature"],
|
|
237
|
+
[],
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
ecrm = ec.build_matrix((0, slice(None), slice(None)))
|
|
241
|
+
np.testing.assert_equal(ecrm, np.ones((3, 3)))
|
|
242
|
+
|
|
243
|
+
ecrm = ec.build_matrix((slice(None), 0, slice(None)))
|
|
244
|
+
np.testing.assert_equal(ecrm, np.ones((6, 6)))
|
|
245
|
+
|
|
246
|
+
ecrm = ec.build_matrix((slice(None), slice(None), 0))
|
|
247
|
+
np.testing.assert_equal(ecrm, np.ones((2, 2)))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
if __name__ == "main":
|
|
251
|
+
unittest.main()
|