obsarray 0.2.5__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {obsarray-0.2.5 → obsarray-0.2.6}/.gitignore +1 -1
- {obsarray-0.2.5/obsarray.egg-info → obsarray-0.2.6}/PKG-INFO +1 -1
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/__init__.py +1 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/_version.py +3 -3
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/err_corr.py +109 -32
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/flag_accessor.py +6 -6
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/dataset_util.py +12 -8
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/dswriter.py +2 -1
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/template_util.py +1 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/test/test_err_corr_forms.py +60 -6
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/test/test_flag_accessor.py +7 -5
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/test/test_unc_accessor.py +30 -12
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/unc_accessor.py +21 -26
- {obsarray-0.2.5 → obsarray-0.2.6/obsarray.egg-info}/PKG-INFO +1 -1
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray.egg-info/requires.txt +1 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/setup.py +2 -1
- {obsarray-0.2.5 → obsarray-0.2.6}/.pre-commit-config.yaml +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/LICENSE +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/MANIFEST.in +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/README.md +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/docs/Makefile +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/docs/conf.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/docs/index.rst +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/docs/make.bat +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/dsdoc.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/dstemplater.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/template_util_jon.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/tests/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/tests/test_dataset_util.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/tests/test_dsdoc.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/tests/test_dswriter.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/templater/tests/test_template_util.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/test/__init__.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray/utils.py +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray.egg-info/SOURCES.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray.egg-info/dependency_links.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/obsarray.egg-info/top_level.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/.gitignore +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/QualityDocumentation.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/CookiecutterMacros.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/acronyms.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/history.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/metadata.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/pythoninputstyle.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/sil2.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/sil3.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/sources.bib +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/base/sources.tex +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/countpdfpages.sh +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/footnotehyper-sphinx.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/obsarray_QF-59.docm +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/obsarray_requirements.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16a.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16b.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16c.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16d.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16e.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/review_checklists/qf-16f.docx +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/softwareRequirements.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinx.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxcyrillic.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxhighlight.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxhowto.cls +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxmanual.cls +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxmessages.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/sphinxmulticell.sty +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/quality_documentation/uml/copy your design png file into this folder.txt +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/setup.cfg +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/tox.ini +0 -0
- {obsarray-0.2.5 → obsarray-0.2.6}/versioneer.py +0 -0
|
@@ -8,6 +8,7 @@ from obsarray import unc_accessor, flag_accessor
|
|
|
8
8
|
from obsarray.err_corr import err_corr_forms
|
|
9
9
|
from obsarray.templater.template_util import create_ds
|
|
10
10
|
from obsarray.templater.dstemplater import DSTemplater
|
|
11
|
+
from obsarray.templater.dswriter import DSWriter
|
|
11
12
|
|
|
12
13
|
__version__ = get_versions()["version"]
|
|
13
14
|
del get_versions
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2024-
|
|
11
|
+
"date": "2024-03-26T17:59:27+0000",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "v0.2.
|
|
14
|
+
"full-revisionid": "3f90a44048e43a24a7ec1115d7b727944416ba9f",
|
|
15
|
+
"version": "v0.2.6"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""err_corr_forms - module for the defintion of error-correlation parameterisation forms"""
|
|
2
2
|
|
|
3
3
|
import abc
|
|
4
|
-
from typing import Callable, Type, Union
|
|
4
|
+
from typing import Callable, Type, Union, List
|
|
5
5
|
import numpy as np
|
|
6
6
|
from comet_maths.linear_algebra.matrix_conversion import expand_errcorr_dims
|
|
7
7
|
|
|
@@ -69,23 +69,79 @@ class BaseErrCorrForm(abc.ABC):
|
|
|
69
69
|
"""Form name"""
|
|
70
70
|
pass
|
|
71
71
|
|
|
72
|
-
def expand_dim_matrix(
|
|
72
|
+
def expand_dim_matrix(
|
|
73
|
+
self, submatrix: np.ndarray, submatrix_dim: Union[str, List[str]], sli: tuple
|
|
74
|
+
):
|
|
73
75
|
return expand_errcorr_dims(
|
|
74
76
|
in_corr=submatrix,
|
|
75
|
-
in_dim=
|
|
77
|
+
in_dim=submatrix_dim,
|
|
76
78
|
out_dim=list(self._obj[self._unc_var_name][sli].dims),
|
|
77
|
-
dim_sizes=
|
|
78
|
-
dim: self._obj.dims[dim]
|
|
79
|
-
for dim in self._obj[self._unc_var_name][sli].dims
|
|
80
|
-
},
|
|
79
|
+
dim_sizes=self.get_sliced_dim_sizes_uncvar(sli),
|
|
81
80
|
)
|
|
82
81
|
|
|
83
|
-
def
|
|
84
|
-
|
|
82
|
+
def get_sliced_dim_sizes_uncvar(self, sli: tuple) -> dict:
|
|
83
|
+
"""
|
|
84
|
+
Return dictionary with sizes of sliced dimensions of unc variable, including all dimensions.
|
|
85
|
+
|
|
86
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
87
|
+
:return: shape of included sliced dimensions
|
|
88
|
+
"""
|
|
89
|
+
uncvar_dims = self._obj[self._unc_var_name][sli].dims
|
|
90
|
+
uncvar_shape = self._obj[self._unc_var_name][sli].shape
|
|
91
|
+
return {
|
|
92
|
+
uncvar_dims[idim]: uncvar_shape[idim] for idim in range(len(uncvar_dims))
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
def get_sliced_dim_sizes_errcorr(self, sli: tuple) -> dict:
|
|
96
|
+
"""
|
|
97
|
+
Return dictionary with sizes of sliced dimensions of unc variable, including only dimensions which are
|
|
98
|
+
included in the current error correlation form.
|
|
99
|
+
|
|
100
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
101
|
+
:return: shape of included sliced dimensions
|
|
102
|
+
"""
|
|
103
|
+
uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
|
|
104
|
+
sliced_dims = self.get_sliced_dims_errcorr(sli)
|
|
105
|
+
|
|
106
|
+
return {dim: uncvar_sizes[dim] for dim in sliced_dims}
|
|
107
|
+
|
|
108
|
+
def get_sliced_dims_errcorr(self, sli: tuple) -> list:
|
|
109
|
+
"""
|
|
110
|
+
Return dimensions which are within the slice and included in the current error correlation form.
|
|
111
|
+
|
|
112
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
113
|
+
:return: list with sliced dimensions
|
|
114
|
+
"""
|
|
115
|
+
all_dims = self._obj[self._unc_var_name].dims
|
|
116
|
+
return [
|
|
117
|
+
all_dims[idim]
|
|
118
|
+
for idim in range(len(all_dims))
|
|
119
|
+
if (isinstance(sli[idim], slice) and all_dims[idim] in self.dims)
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
def get_sliced_shape_errcorr(self, sli: tuple) -> tuple:
|
|
123
|
+
"""
|
|
124
|
+
return shape of sliced uncertainty variable, including only dimensions which are included in the current error correlation form.
|
|
125
|
+
|
|
126
|
+
:param sli: slice (tuple with slice for each dimension)
|
|
127
|
+
:return: shape of included sliced dimensions
|
|
128
|
+
"""
|
|
129
|
+
uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
|
|
130
|
+
sliced_dims = self.get_sliced_dims_errcorr(sli)
|
|
131
|
+
|
|
132
|
+
return tuple([uncvar_sizes[dim] for dim in sliced_dims])
|
|
133
|
+
|
|
134
|
+
def slice_full_cov(self, full_matrix: np.ndarray, sli: tuple) -> np.ndarray:
|
|
135
|
+
return self.slice_flattened_matrix(
|
|
136
|
+
full_matrix, self._obj[self._unc_var_name].shape, sli
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def slice_flattened_matrix(self, flattened_matrix, variable_shape, sli):
|
|
140
|
+
mask_array = np.ones(variable_shape, dtype=bool)
|
|
85
141
|
mask_array[sli] = False
|
|
86
142
|
|
|
87
143
|
return np.delete(
|
|
88
|
-
np.delete(
|
|
144
|
+
np.delete(flattened_matrix, mask_array.ravel(), 0), mask_array.ravel(), 1
|
|
89
145
|
)
|
|
90
146
|
|
|
91
147
|
@abc.abstractmethod
|
|
@@ -100,6 +156,20 @@ class BaseErrCorrForm(abc.ABC):
|
|
|
100
156
|
"""
|
|
101
157
|
pass
|
|
102
158
|
|
|
159
|
+
def build_dot_matrix(self, sli: Union[np.ndarray, tuple]) -> np.ndarray:
|
|
160
|
+
"""
|
|
161
|
+
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
162
|
+
in this parameterisation
|
|
163
|
+
|
|
164
|
+
:param sli: slice of observation variable to return error-correlation matrix for
|
|
165
|
+
|
|
166
|
+
:return: populated error-correlation matrix
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
return self.expand_dim_matrix(
|
|
170
|
+
self.build_matrix(sli), self.get_sliced_dims_errcorr(sli), sli
|
|
171
|
+
)
|
|
172
|
+
|
|
103
173
|
|
|
104
174
|
def register_err_corr_form(form_name: str) -> Callable:
|
|
105
175
|
"""
|
|
@@ -122,7 +192,7 @@ class RandomCorrelation(BaseErrCorrForm):
|
|
|
122
192
|
form = "random"
|
|
123
193
|
is_random = True
|
|
124
194
|
|
|
125
|
-
def build_matrix(self, sli):
|
|
195
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
126
196
|
"""
|
|
127
197
|
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
128
198
|
in this parameterisation
|
|
@@ -133,16 +203,12 @@ class RandomCorrelation(BaseErrCorrForm):
|
|
|
133
203
|
"""
|
|
134
204
|
|
|
135
205
|
# evaluate correlation over matrices in form defintion
|
|
136
|
-
dim_lens =
|
|
206
|
+
dim_lens = self.get_sliced_shape_errcorr(sli)
|
|
137
207
|
n_elems = int(np.prod(dim_lens))
|
|
138
208
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
# expand to correlation matrix over all variable dims
|
|
142
|
-
return self.expand_dim_matrix(dims_matrix, sli)
|
|
209
|
+
submatrix = np.eye(n_elems)
|
|
143
210
|
|
|
144
|
-
|
|
145
|
-
# return self.slice_full_cov(full_matrix, sli)
|
|
211
|
+
return submatrix
|
|
146
212
|
|
|
147
213
|
|
|
148
214
|
@register_err_corr_form("systematic")
|
|
@@ -151,7 +217,7 @@ class SystematicCorrelation(BaseErrCorrForm):
|
|
|
151
217
|
form = "systematic"
|
|
152
218
|
is_systematic = True
|
|
153
219
|
|
|
154
|
-
def build_matrix(self, sli):
|
|
220
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
155
221
|
"""
|
|
156
222
|
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
157
223
|
in this parameterisation
|
|
@@ -162,16 +228,12 @@ class SystematicCorrelation(BaseErrCorrForm):
|
|
|
162
228
|
"""
|
|
163
229
|
|
|
164
230
|
# evaluate correlation over matrices in form defintion
|
|
165
|
-
dim_lens =
|
|
231
|
+
dim_lens = self.get_sliced_shape_errcorr(sli)
|
|
166
232
|
n_elems = int(np.prod(dim_lens))
|
|
167
233
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
# expand to correlation matrix over all variable dims
|
|
171
|
-
return self.expand_dim_matrix(dims_matrix, sli)
|
|
234
|
+
submatrix = np.ones((n_elems, n_elems))
|
|
172
235
|
|
|
173
|
-
|
|
174
|
-
# return self.slice_full_cov(full_matrix, sli)
|
|
236
|
+
return submatrix
|
|
175
237
|
|
|
176
238
|
|
|
177
239
|
@register_err_corr_form("err_corr_matrix")
|
|
@@ -179,7 +241,7 @@ class ErrCorrMatrixCorrelation(BaseErrCorrForm):
|
|
|
179
241
|
|
|
180
242
|
form = "err_corr_matrix"
|
|
181
243
|
|
|
182
|
-
def build_matrix(self, sli):
|
|
244
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
183
245
|
"""
|
|
184
246
|
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
185
247
|
in this parameterisation
|
|
@@ -189,11 +251,26 @@ class ErrCorrMatrixCorrelation(BaseErrCorrForm):
|
|
|
189
251
|
:return: populated error-correlation matrix
|
|
190
252
|
"""
|
|
191
253
|
|
|
192
|
-
|
|
193
|
-
|
|
254
|
+
all_dims = self._obj[self._unc_var_name].dims
|
|
255
|
+
all_dims_sizes = self._obj.sizes
|
|
256
|
+
|
|
257
|
+
sli_submatrix = tuple(
|
|
258
|
+
[sli[i] for i in range(len(all_dims)) if all_dims[i] in self.dims]
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
sliced_shape = tuple(
|
|
262
|
+
[
|
|
263
|
+
all_dims_sizes[all_dims[i]]
|
|
264
|
+
for i in range(len(all_dims))
|
|
265
|
+
if all_dims[i] in self.dims
|
|
266
|
+
]
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
submatrix = self.slice_flattened_matrix(
|
|
270
|
+
self._obj[self.params[0]], sliced_shape, sli_submatrix
|
|
271
|
+
)
|
|
194
272
|
|
|
195
|
-
|
|
196
|
-
# return self.slice_full_cov(full_matrix, sli)
|
|
273
|
+
return submatrix
|
|
197
274
|
|
|
198
275
|
|
|
199
276
|
@register_err_corr_form("ensemble")
|
|
@@ -201,7 +278,7 @@ class EnsembleCorrelation(BaseErrCorrForm):
|
|
|
201
278
|
|
|
202
279
|
form = "ensemble"
|
|
203
280
|
|
|
204
|
-
def build_matrix(self, sli):
|
|
281
|
+
def build_matrix(self, sli: tuple) -> np.ndarray:
|
|
205
282
|
"""
|
|
206
283
|
Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
|
|
207
284
|
in this parameterisation
|
|
@@ -194,12 +194,12 @@ class FlagVariable:
|
|
|
194
194
|
)
|
|
195
195
|
|
|
196
196
|
if flag_meaning not in flag_meanings:
|
|
197
|
-
self._obj[
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
197
|
+
self._obj[self._flag_var_name].attrs = (
|
|
198
|
+
DatasetUtil.add_flag_meaning_to_attrs(
|
|
199
|
+
self._obj[self._flag_var_name].attrs,
|
|
200
|
+
flag_meaning,
|
|
201
|
+
self._obj[self._flag_var_name].dtype,
|
|
202
|
+
)
|
|
203
203
|
)
|
|
204
204
|
|
|
205
205
|
self[flag_meaning][:] = flag_value
|
|
@@ -259,8 +259,8 @@ class DatasetUtil:
|
|
|
259
259
|
attributes=attributes,
|
|
260
260
|
)
|
|
261
261
|
|
|
262
|
-
#initialise flags to zero (instead of fillvalue)
|
|
263
|
-
variable.values=0*variable.values
|
|
262
|
+
# initialise flags to zero (instead of fillvalue)
|
|
263
|
+
variable.values = 0 * variable.values
|
|
264
264
|
|
|
265
265
|
# add flag attributes
|
|
266
266
|
variable.attrs.update(DatasetUtil.pack_flag_attrs(meanings))
|
|
@@ -290,7 +290,7 @@ class DatasetUtil:
|
|
|
290
290
|
)
|
|
291
291
|
|
|
292
292
|
if flag_masks is None:
|
|
293
|
-
flag_masks = [2
|
|
293
|
+
flag_masks = [2**i for i in range(0, n_masks)]
|
|
294
294
|
|
|
295
295
|
flag_attrs["flag_masks"] = str(flag_masks)[1:-1]
|
|
296
296
|
|
|
@@ -333,7 +333,7 @@ class DatasetUtil:
|
|
|
333
333
|
|
|
334
334
|
# check if variable for available flags
|
|
335
335
|
max_n_flags = numpy.iinfo(dtype).bits
|
|
336
|
-
all_flag_masks = [2
|
|
336
|
+
all_flag_masks = [2**i for i in range(0, max_n_flags)]
|
|
337
337
|
available_flag_masks = list(set(all_flag_masks) - set(flag_masks))
|
|
338
338
|
|
|
339
339
|
if not available_flag_masks:
|
|
@@ -434,6 +434,10 @@ class DatasetUtil:
|
|
|
434
434
|
|
|
435
435
|
if fill_value is not None:
|
|
436
436
|
encoding_dict.update({"_FillValue": fill_value})
|
|
437
|
+
else:
|
|
438
|
+
encoding_dict.update(
|
|
439
|
+
{"_FillValue": DatasetUtil.get_default_fill_value(dtype)}
|
|
440
|
+
)
|
|
437
441
|
|
|
438
442
|
variable.encoding = encoding_dict
|
|
439
443
|
|
|
@@ -524,8 +528,8 @@ class DatasetUtil:
|
|
|
524
528
|
|
|
525
529
|
flags_ds = DatasetUtil.unpack_flags(da)
|
|
526
530
|
|
|
527
|
-
flags = flags if flags is not None else flags_ds.
|
|
528
|
-
mask_flags = [flags_ds[flag].values for flag in flags]
|
|
531
|
+
flags = flags if flags is not None else flags_ds.data_vars
|
|
532
|
+
mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
|
|
529
533
|
|
|
530
534
|
return numpy.logical_or.reduce(mask_flags)
|
|
531
535
|
|
|
@@ -546,8 +550,8 @@ class DatasetUtil:
|
|
|
546
550
|
|
|
547
551
|
flags_ds = DatasetUtil.unpack_flags(da)
|
|
548
552
|
|
|
549
|
-
flags = flags if flags is not None else flags_ds.
|
|
550
|
-
mask_flags = [flags_ds[flag].values for flag in flags]
|
|
553
|
+
flags = flags if flags is not None else flags_ds.data_vars
|
|
554
|
+
mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
|
|
551
555
|
|
|
552
556
|
return numpy.logical_and.reduce(mask_flags)
|
|
553
557
|
|
|
@@ -81,7 +81,8 @@ class DSWriter:
|
|
|
81
81
|
)
|
|
82
82
|
}
|
|
83
83
|
)
|
|
84
|
-
ds[var_name].attrs
|
|
84
|
+
if "_FillValue" in ds[var_name].attrs:
|
|
85
|
+
ds[var_name].attrs.pop("_FillValue")
|
|
85
86
|
encoding.update({var_name: var_encoding})
|
|
86
87
|
|
|
87
88
|
ds.to_netcdf(path, format="netCDF4", engine="netcdf4", encoding=encoding)
|
|
@@ -81,6 +81,59 @@ class TestBaseErrCorrForm(unittest.TestCase):
|
|
|
81
81
|
|
|
82
82
|
np.testing.assert_equal(full_matrix, slice_matrix)
|
|
83
83
|
|
|
84
|
+
def test_get_sliced_dim_sizes_uncvar(self):
|
|
85
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
86
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
87
|
+
)
|
|
88
|
+
dim_sizes = basicerrcorr.get_sliced_dim_sizes_uncvar(
|
|
89
|
+
(slice(None), 0, slice(0, 2, 1))
|
|
90
|
+
)
|
|
91
|
+
assert dim_sizes == {"x": 2, "time": 2}
|
|
92
|
+
|
|
93
|
+
def test_get_sliced_dim_sizes_errcorr(self):
|
|
94
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
95
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
96
|
+
)
|
|
97
|
+
dim_sizes = basicerrcorr.get_sliced_dim_sizes_errcorr(
|
|
98
|
+
(slice(None), 0, slice(0, 2, 1))
|
|
99
|
+
)
|
|
100
|
+
assert dim_sizes == {"x": 2}
|
|
101
|
+
|
|
102
|
+
def test_get_sliced_dims_errcorr(self):
|
|
103
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
104
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
105
|
+
)
|
|
106
|
+
dims = basicerrcorr.get_sliced_dims_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
107
|
+
assert dims == ["x"]
|
|
108
|
+
|
|
109
|
+
def test_get_sliced_shape_errcorr(self):
|
|
110
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
111
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
112
|
+
)
|
|
113
|
+
shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
114
|
+
assert shape == (2,)
|
|
115
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
116
|
+
self.ds, "u_ran_temperature", ["x", "time"], [], []
|
|
117
|
+
)
|
|
118
|
+
shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
|
|
119
|
+
assert shape == (2, 2)
|
|
120
|
+
|
|
121
|
+
def test_slice_flattened_matrix(self):
|
|
122
|
+
basicerrcorr = self.BasicErrCorrForm(
|
|
123
|
+
self.ds, "u_ran_temperature", ["x"], [], []
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
full_matrix = np.arange(144).reshape((12, 12))
|
|
127
|
+
slice_matrix = basicerrcorr.slice_flattened_matrix(
|
|
128
|
+
full_matrix, (2, 2, 3), (slice(None), slice(None), 0)
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
exp_slice_matrix = np.array(
|
|
132
|
+
[[0, 3, 6, 9], [36, 39, 42, 45], [72, 75, 78, 81], [108, 111, 114, 117]]
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
np.testing.assert_equal(slice_matrix, exp_slice_matrix)
|
|
136
|
+
|
|
84
137
|
def test_slice_full_cov_slice(self):
|
|
85
138
|
basicerrcorr = self.BasicErrCorrForm(
|
|
86
139
|
self.ds, "u_ran_temperature", ["x"], [], []
|
|
@@ -105,14 +158,14 @@ class TestRandomUnc(unittest.TestCase):
|
|
|
105
158
|
def test_build_matrix_1stdim(self):
|
|
106
159
|
rc = RandomCorrelation(self.ds, "u_ran_temperature", ["x"], [], [])
|
|
107
160
|
|
|
108
|
-
ecrm = rc.
|
|
161
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
109
162
|
|
|
110
163
|
np.testing.assert_equal(ecrm, np.eye(12))
|
|
111
164
|
|
|
112
165
|
def test_build_matrix_2nddim(self):
|
|
113
166
|
rc = RandomCorrelation(self.ds, "u_ran_temperature", ["y"], [], [])
|
|
114
167
|
|
|
115
|
-
ecrm = rc.
|
|
168
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
116
169
|
|
|
117
170
|
np.testing.assert_equal(ecrm, np.eye(12))
|
|
118
171
|
|
|
@@ -124,7 +177,7 @@ class TestSystematicUnc(unittest.TestCase):
|
|
|
124
177
|
def build_matrix_1stdim(self):
|
|
125
178
|
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["x"], [], [])
|
|
126
179
|
|
|
127
|
-
ecrm = rc.
|
|
180
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
128
181
|
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
129
182
|
|
|
130
183
|
return ecrm
|
|
@@ -132,7 +185,7 @@ class TestSystematicUnc(unittest.TestCase):
|
|
|
132
185
|
def build_matrix_2nddim(self):
|
|
133
186
|
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["y"], [], [])
|
|
134
187
|
|
|
135
|
-
ecrm = rc.
|
|
188
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
136
189
|
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
137
190
|
|
|
138
191
|
return ecrm
|
|
@@ -140,15 +193,16 @@ class TestSystematicUnc(unittest.TestCase):
|
|
|
140
193
|
def build_matrix_3ddim(self):
|
|
141
194
|
rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["time"], [], [])
|
|
142
195
|
|
|
143
|
-
ecrm = rc.
|
|
196
|
+
ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
|
|
144
197
|
# np.testing.assert_equal(ecrm, np.ones((12, 12)))
|
|
145
198
|
|
|
146
199
|
return ecrm
|
|
147
200
|
|
|
148
|
-
def
|
|
201
|
+
def test_build_dot_matrix(self):
|
|
149
202
|
x = self.build_matrix_1stdim()
|
|
150
203
|
y = self.build_matrix_2nddim()
|
|
151
204
|
time = self.build_matrix_3ddim()
|
|
205
|
+
print(x.dot(y), x, y)
|
|
152
206
|
np.testing.assert_equal((x.dot(y)).dot(time), np.ones((12, 12)))
|
|
153
207
|
|
|
154
208
|
|
|
@@ -60,7 +60,7 @@ def create_ds():
|
|
|
60
60
|
)
|
|
61
61
|
|
|
62
62
|
ds["time_flags"] = create_var(
|
|
63
|
-
"
|
|
63
|
+
"time_flags",
|
|
64
64
|
{
|
|
65
65
|
"dtype": "flag",
|
|
66
66
|
"dim": ["time"],
|
|
@@ -83,7 +83,7 @@ class TestFlagAccessor(unittest.TestCase):
|
|
|
83
83
|
self.assertEqual(self.ds.flag["general_flag"]._flag_var_name, "general_flag")
|
|
84
84
|
|
|
85
85
|
def test___len__(self):
|
|
86
|
-
self.assertEqual(len(self.ds.flag),
|
|
86
|
+
self.assertEqual(len(self.ds.flag), 3)
|
|
87
87
|
|
|
88
88
|
def test___iter__(self):
|
|
89
89
|
|
|
@@ -92,11 +92,13 @@ class TestFlagAccessor(unittest.TestCase):
|
|
|
92
92
|
self.assertIsInstance(flag, obsarray.flag_accessor.FlagVariable)
|
|
93
93
|
var_names.append(flag._flag_var_name)
|
|
94
94
|
|
|
95
|
-
self.assertCountEqual(
|
|
95
|
+
self.assertCountEqual(
|
|
96
|
+
var_names, ["temperature_flags", "general_flags", "time_flags"]
|
|
97
|
+
)
|
|
96
98
|
|
|
97
99
|
def test_keys(self):
|
|
98
100
|
self.assertCountEqual(
|
|
99
|
-
self.ds.flag.keys(), ["temperature_flags", "general_flags"]
|
|
101
|
+
self.ds.flag.keys(), ["temperature_flags", "general_flags", "time_flags"]
|
|
100
102
|
)
|
|
101
103
|
|
|
102
104
|
def test__is_data_var_true(self):
|
|
@@ -117,7 +119,7 @@ class TestFlagAccessor(unittest.TestCase):
|
|
|
117
119
|
self.assertEqual(type(flag_vars), xr.core.dataset.DataVariables)
|
|
118
120
|
self.assertCountEqual(
|
|
119
121
|
list(flag_vars.variables),
|
|
120
|
-
["temperature_flags", "general_flags"],
|
|
122
|
+
["temperature_flags", "general_flags", "time_flags"],
|
|
121
123
|
)
|
|
122
124
|
|
|
123
125
|
def test__is_flag_var_true(self):
|
|
@@ -117,6 +117,11 @@ def create_ds():
|
|
|
117
117
|
},
|
|
118
118
|
)
|
|
119
119
|
|
|
120
|
+
ds["err_corr_str_temperature"] = (
|
|
121
|
+
["x", "x"],
|
|
122
|
+
np.eye(temperature.shape[0]),
|
|
123
|
+
)
|
|
124
|
+
|
|
120
125
|
return ds
|
|
121
126
|
|
|
122
127
|
|
|
@@ -474,13 +479,17 @@ class TestVariableUncertainty(unittest.TestCase):
|
|
|
474
479
|
self.ds.unc["temperature"][:, :, 0].systematic_unc()
|
|
475
480
|
mock.assert_called_once_with(["u_sys_temperature"])
|
|
476
481
|
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
482
|
+
def test_total_err_corr_matrix(
|
|
483
|
+
self,
|
|
484
|
+
):
|
|
485
|
+
tercm = self.ds.unc["temperature"].total_err_corr_matrix()
|
|
486
|
+
assert tercm.shape == (12, 12)
|
|
487
|
+
tercm = self.ds.unc["temperature"][:, :, 0].total_err_corr_matrix()
|
|
488
|
+
assert tercm.shape == (4, 4)
|
|
489
|
+
tercm = self.ds.unc["temperature"][:, 0, 0:2].total_err_corr_matrix()
|
|
490
|
+
assert tercm.shape == (4, 4)
|
|
491
|
+
tercm = self.ds.unc["temperature"][0, 0, :].total_err_corr_matrix()
|
|
492
|
+
assert tercm.shape == (3, 3)
|
|
484
493
|
|
|
485
494
|
def test_structured_err_corr_matrix(self):
|
|
486
495
|
pass
|
|
@@ -496,7 +505,7 @@ class TestUncertainty(unittest.TestCase):
|
|
|
496
505
|
def setUp(self):
|
|
497
506
|
self.ds = create_ds()
|
|
498
507
|
|
|
499
|
-
@patch("obsarray.unc_accessor.Uncertainty.
|
|
508
|
+
@patch("obsarray.unc_accessor.Uncertainty._expand_sli", return_value="slice")
|
|
500
509
|
def test___getitem__(self, m):
|
|
501
510
|
self.assertEqual(
|
|
502
511
|
self.ds.unc["temperature"]["u_ran_temperature"]["in_slice"]._sli, "slice"
|
|
@@ -513,21 +522,30 @@ class TestUncertainty(unittest.TestCase):
|
|
|
513
522
|
def test_expand_slice_1d_None(self):
|
|
514
523
|
self.ds["new"] = (["time"], np.ones(3), {})
|
|
515
524
|
self.ds.unc["new"]["u_new"] = (["time"], np.ones(3), {})
|
|
516
|
-
sli = self.ds.unc["
|
|
525
|
+
sli = self.ds.unc["new"]["u_new"]._expand_sli()
|
|
517
526
|
self.assertEqual((slice(None),), sli)
|
|
518
527
|
|
|
519
528
|
def test_expand_slice_full(self):
|
|
520
|
-
sli = self.ds.unc["temperature"]["u_ran_temperature"].
|
|
529
|
+
sli = self.ds.unc["temperature"]["u_ran_temperature"]._expand_sli((1, 1, 1))
|
|
521
530
|
self.assertEqual((1, 1, 1), sli)
|
|
522
531
|
|
|
523
532
|
def test_expand_slice_None(self):
|
|
524
|
-
sli = self.ds.unc["temperature"]["u_ran_temperature"].
|
|
533
|
+
sli = self.ds.unc["temperature"]["u_ran_temperature"]._expand_sli()
|
|
525
534
|
self.assertEqual((slice(None), slice(None), slice(None)), sli)
|
|
526
535
|
|
|
527
536
|
def test_expand_slice_first(self):
|
|
528
|
-
sli = self.ds.unc["temperature"]["u_ran_temperature"].
|
|
537
|
+
sli = self.ds.unc["temperature"]["u_ran_temperature"]._expand_sli((0,))
|
|
529
538
|
self.assertEqual((0, slice(None), slice(None)), sli)
|
|
530
539
|
|
|
540
|
+
def test_rename(self):
|
|
541
|
+
ds = self.ds.unc["temperature"]["u_ran_temperature"].rename("test")
|
|
542
|
+
self.assertTrue("test" in ds.keys())
|
|
543
|
+
self.assertTrue("u_ran_temperature" not in ds.keys())
|
|
544
|
+
self.assertCountEqual(
|
|
545
|
+
ds["temperature"].attrs["unc_comps"],
|
|
546
|
+
["test", "u_str_temperature", "u_sys_temperature"],
|
|
547
|
+
)
|
|
548
|
+
|
|
531
549
|
def test_err_corr(self):
|
|
532
550
|
|
|
533
551
|
expected_err_corr = [
|
|
@@ -92,6 +92,20 @@ class Uncertainty:
|
|
|
92
92
|
|
|
93
93
|
return out_sli
|
|
94
94
|
|
|
95
|
+
def rename(self, name) -> xr.Dataset:
|
|
96
|
+
"""
|
|
97
|
+
Renames uncertainty variable and safely updates variable name references in obs variable metadata
|
|
98
|
+
:param name: desired name
|
|
99
|
+
:returns: dataset with safely renamed variable
|
|
100
|
+
"""
|
|
101
|
+
self._obj = self._obj.rename({self._unc_var_name: name})
|
|
102
|
+
self._obj[self._var_name].attrs["unc_comps"] = [
|
|
103
|
+
v.replace(self._unc_var_name, name)
|
|
104
|
+
for v in self._obj[self._var_name].attrs["unc_comps"]
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
return self._obj
|
|
108
|
+
|
|
95
109
|
@property
|
|
96
110
|
def err_corr(self) -> List[Tuple[Union[str, List[str]], BaseErrCorrForm]]:
|
|
97
111
|
"""
|
|
@@ -105,7 +119,9 @@ class Uncertainty:
|
|
|
105
119
|
# Find dimensions in variable slice
|
|
106
120
|
sli_dims = [
|
|
107
121
|
dim
|
|
108
|
-
for dim, idx in zip(
|
|
122
|
+
for dim, idx in zip(
|
|
123
|
+
self._obj.dims.keys(), self._sli
|
|
124
|
+
) # Due to hit a FutureDeprecation warning
|
|
109
125
|
if not isinstance(idx, int)
|
|
110
126
|
]
|
|
111
127
|
|
|
@@ -305,22 +321,6 @@ class Uncertainty:
|
|
|
305
321
|
)
|
|
306
322
|
return err_corr_dict
|
|
307
323
|
|
|
308
|
-
def err_corr_dict_numdim(self) -> dict:
|
|
309
|
-
"""
|
|
310
|
-
Error-correlation dictionary for uncertainty effect, where the keys are the dimension index rather than dimension name.
|
|
311
|
-
|
|
312
|
-
:return: dictionary with error-correlation for each dimension
|
|
313
|
-
"""
|
|
314
|
-
# initialise error-correlation dictionary
|
|
315
|
-
err_corr_dict = self.err_corr_dict()
|
|
316
|
-
err_corr_dict_numdim = {}
|
|
317
|
-
|
|
318
|
-
for idim, dim in enumerate(self._obj.dims):
|
|
319
|
-
if dim in err_corr_dict.keys():
|
|
320
|
-
err_corr_dict_numdim[str(idim)] = err_corr_dict[dim]
|
|
321
|
-
|
|
322
|
-
return err_corr_dict_numdim
|
|
323
|
-
|
|
324
324
|
def err_corr_matrix(self) -> xr.DataArray:
|
|
325
325
|
"""
|
|
326
326
|
Error-correlation matrix for uncertainty effect.
|
|
@@ -336,14 +336,10 @@ class Uncertainty:
|
|
|
336
336
|
# populate with error-correlation matrices built be each error-correlation
|
|
337
337
|
# parameterisation object
|
|
338
338
|
for dim_err_corr in self.err_corr:
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
dim in self._obj[self._unc_var_name][self._sli].dims
|
|
342
|
-
for dim in dim_err_corr[1].dims
|
|
343
|
-
]
|
|
344
|
-
):
|
|
339
|
+
sliced_dims = dim_err_corr[1].get_sliced_dims_errcorr(self._sli)
|
|
340
|
+
if len(dim_err_corr[1].get_sliced_dims_errcorr(self._sli)) > 0:
|
|
345
341
|
err_corr_matrix.values = err_corr_matrix.values.dot(
|
|
346
|
-
dim_err_corr[1].
|
|
342
|
+
dim_err_corr[1].build_dot_matrix(self._sli)
|
|
347
343
|
)
|
|
348
344
|
|
|
349
345
|
return err_corr_matrix
|
|
@@ -522,7 +518,7 @@ class VariableUncertainty:
|
|
|
522
518
|
quadsum_unc += self[unc_var_name].abs_value ** 2.0
|
|
523
519
|
|
|
524
520
|
if quadsum_unc is not None:
|
|
525
|
-
quadsum_unc = quadsum_unc
|
|
521
|
+
quadsum_unc = quadsum_unc**0.5
|
|
526
522
|
|
|
527
523
|
return quadsum_unc
|
|
528
524
|
|
|
@@ -824,7 +820,6 @@ class UncAccessor(object):
|
|
|
824
820
|
"""
|
|
825
821
|
|
|
826
822
|
unc_var_names = self._var_unc_var_names(obs_var_name, unc_type=unc_type)
|
|
827
|
-
|
|
828
823
|
if len(unc_var_names) == 0:
|
|
829
824
|
return None
|
|
830
825
|
|
|
@@ -9,7 +9,7 @@ import versioneer
|
|
|
9
9
|
|
|
10
10
|
def read(filename):
|
|
11
11
|
filename = os.path.join(os.path.dirname(__file__), filename)
|
|
12
|
-
text_type = type(
|
|
12
|
+
text_type = type("")
|
|
13
13
|
with io.open(filename, mode="r", encoding="utf-8") as fd:
|
|
14
14
|
return re.sub(text_type(r":[a-z]+:`~?(.*?)`"), text_type(r"``\1``"), fd.read())
|
|
15
15
|
|
|
@@ -32,6 +32,7 @@ setup(
|
|
|
32
32
|
"tox",
|
|
33
33
|
"sphinx",
|
|
34
34
|
"sphinx_design",
|
|
35
|
+
"sphinx_book_theme",
|
|
35
36
|
"ipython",
|
|
36
37
|
"sphinx_autosummary_accessors",
|
|
37
38
|
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|