obsarray 0.2.5__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {obsarray-0.2.5 → obsarray-0.2.7}/.gitignore +1 -1
  2. {obsarray-0.2.5/obsarray.egg-info → obsarray-0.2.7}/PKG-INFO +1 -1
  3. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/__init__.py +1 -0
  4. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/_version.py +3 -3
  5. obsarray-0.2.7/obsarray/err_corr.py +298 -0
  6. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/flag_accessor.py +6 -6
  7. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dataset_util.py +12 -8
  8. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dswriter.py +2 -1
  9. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/template_util.py +1 -0
  10. obsarray-0.2.7/obsarray/test/test_err_corr_forms.py +251 -0
  11. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/test_flag_accessor.py +7 -5
  12. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/test_unc_accessor.py +36 -18
  13. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/unc_accessor.py +22 -27
  14. {obsarray-0.2.5 → obsarray-0.2.7/obsarray.egg-info}/PKG-INFO +1 -1
  15. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/SOURCES.txt +0 -2
  16. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/requires.txt +1 -0
  17. {obsarray-0.2.5 → obsarray-0.2.7}/setup.py +2 -1
  18. obsarray-0.2.5/obsarray/err_corr.py +0 -218
  19. obsarray-0.2.5/obsarray/templater/dsdoc.py +0 -302
  20. obsarray-0.2.5/obsarray/templater/tests/test_dsdoc.py +0 -312
  21. obsarray-0.2.5/obsarray/test/test_err_corr_forms.py +0 -156
  22. {obsarray-0.2.5 → obsarray-0.2.7}/.pre-commit-config.yaml +0 -0
  23. {obsarray-0.2.5 → obsarray-0.2.7}/LICENSE +0 -0
  24. {obsarray-0.2.5 → obsarray-0.2.7}/MANIFEST.in +0 -0
  25. {obsarray-0.2.5 → obsarray-0.2.7}/README.md +0 -0
  26. {obsarray-0.2.5 → obsarray-0.2.7}/docs/Makefile +0 -0
  27. {obsarray-0.2.5 → obsarray-0.2.7}/docs/conf.py +0 -0
  28. {obsarray-0.2.5 → obsarray-0.2.7}/docs/index.rst +0 -0
  29. {obsarray-0.2.5 → obsarray-0.2.7}/docs/make.bat +0 -0
  30. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/__init__.py +0 -0
  31. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/dstemplater.py +0 -0
  32. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/template_util_jon.py +0 -0
  33. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/__init__.py +0 -0
  34. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_dataset_util.py +0 -0
  35. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_dswriter.py +0 -0
  36. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/templater/tests/test_template_util.py +0 -0
  37. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/test/__init__.py +0 -0
  38. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray/utils.py +0 -0
  39. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/dependency_links.txt +0 -0
  40. {obsarray-0.2.5 → obsarray-0.2.7}/obsarray.egg-info/top_level.txt +0 -0
  41. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/.gitignore +0 -0
  42. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/QualityDocumentation.tex +0 -0
  43. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/CookiecutterMacros.tex +0 -0
  44. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/acronyms.tex +0 -0
  45. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/history.tex +0 -0
  46. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/metadata.tex +0 -0
  47. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/pythoninputstyle.tex +0 -0
  48. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sil2.tex +0 -0
  49. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sil3.tex +0 -0
  50. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sources.bib +0 -0
  51. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/base/sources.tex +0 -0
  52. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/countpdfpages.sh +0 -0
  53. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/footnotehyper-sphinx.sty +0 -0
  54. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/obsarray_QF-59.docm +0 -0
  55. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/obsarray_requirements.docx +0 -0
  56. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16a.docx +0 -0
  57. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16b.docx +0 -0
  58. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16c.docx +0 -0
  59. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16d.docx +0 -0
  60. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16e.docx +0 -0
  61. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/review_checklists/qf-16f.docx +0 -0
  62. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/softwareRequirements.sty +0 -0
  63. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinx.sty +0 -0
  64. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxcyrillic.sty +0 -0
  65. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxhighlight.sty +0 -0
  66. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxhowto.cls +0 -0
  67. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmanual.cls +0 -0
  68. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmessages.sty +0 -0
  69. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/sphinxmulticell.sty +0 -0
  70. {obsarray-0.2.5 → obsarray-0.2.7}/quality_documentation/uml/copy your design png file into this folder.txt +0 -0
  71. {obsarray-0.2.5 → obsarray-0.2.7}/setup.cfg +0 -0
  72. {obsarray-0.2.5 → obsarray-0.2.7}/tox.ini +0 -0
  73. {obsarray-0.2.5 → obsarray-0.2.7}/versioneer.py +0 -0
@@ -77,7 +77,7 @@ instance/
77
77
 
78
78
  # Sphinx documentation
79
79
  docs/_build/
80
- docs/content/generated/
80
+ docs/content/user/generated/
81
81
 
82
82
  # PyBuilder
83
83
  target/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: obsarray
3
- Version: 0.2.5
3
+ Version: 0.2.7
4
4
  Summary: Measurement data handling in Python
5
5
  Home-page: https://github.com/comet-toolkit/obsarray
6
6
  Author: CoMet Toolkit Team
@@ -8,6 +8,7 @@ from obsarray import unc_accessor, flag_accessor
8
8
  from obsarray.err_corr import err_corr_forms
9
9
  from obsarray.templater.template_util import create_ds
10
10
  from obsarray.templater.dstemplater import DSTemplater
11
+ from obsarray.templater.dswriter import DSWriter
11
12
 
12
13
  __version__ = get_versions()["version"]
13
14
  del get_versions
@@ -8,11 +8,11 @@ import json
8
8
 
9
9
  version_json = '''
10
10
  {
11
- "date": "2024-02-14T16:24:15+0000",
11
+ "date": "2024-04-02T11:59:05+0100",
12
12
  "dirty": false,
13
13
  "error": null,
14
- "full-revisionid": "a4c0efeceef26dd8201830bc88caab5689f25595",
15
- "version": "v0.2.5"
14
+ "full-revisionid": "4f32df4b9da131ab66376b319a8036975d891233",
15
+ "version": "v0.2.7"
16
16
  }
17
17
  ''' # END VERSION_JSON
18
18
 
@@ -0,0 +1,298 @@
1
+ """err_corr_forms - module for the defintion of error-correlation parameterisation forms"""
2
+
3
+ import abc
4
+ from typing import Callable, Type, Union, List
5
+ import numpy as np
6
+ from comet_maths.linear_algebra.matrix_conversion import expand_errcorr_dims
7
+
8
+ __author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
9
+ __all__ = ["err_corr_forms", "register_err_corr_form", "BaseErrCorrForm"]
10
+
11
+
12
+ class ErrCorrForms:
13
+ """
14
+ Container for error-correlation parameterisation form definition objects
15
+ """
16
+
17
+ def __init__(self):
18
+ self._forms = {}
19
+
20
+ def __setitem__(self, form_name, form_cls):
21
+ if not issubclass(form_cls, BaseErrCorrForm):
22
+ raise TypeError(
23
+ "form must be subclass of " + str(BaseErrCorrForm.__class__)
24
+ )
25
+
26
+ self._forms[form_name] = form_cls
27
+
28
+ def __getitem__(self, name):
29
+ return self._forms[name]
30
+
31
+ def __delitem__(self, name):
32
+ del self._forms[name]
33
+
34
+ def keys(self):
35
+ return self._forms.keys()
36
+
37
+
38
+ err_corr_forms = ErrCorrForms()
39
+
40
+
41
+ # placeholder currently
42
+ class BaseErrCorrForm(abc.ABC):
43
+ """
44
+ Base class for error-correlation parameterisation form defintions
45
+ """
46
+
47
+ is_random = False
48
+ is_systematic = False
49
+
50
+ def __init__(self, xarray_obj, unc_var_name, dims, params, units):
51
+ self._obj = xarray_obj
52
+ self._unc_var_name = unc_var_name
53
+ self.dims = dims if isinstance(dims, list) else [dims]
54
+ self.params = params if isinstance(params, list) else [params]
55
+ self.units = units
56
+
57
+ def __str__(self):
58
+ """Custom __str__"""
59
+
60
+ return self.form + " " + str(self.params)
61
+
62
+ def __repr__(self):
63
+ """Custom __repr__"""
64
+ return str(self)
65
+
66
+ @property
67
+ @abc.abstractmethod
68
+ def form(self) -> str:
69
+ """Form name"""
70
+ pass
71
+
72
+ def get_varshape_errcorr(self):
73
+ """
74
+ return shape of uncertainty variable, including only dimensions which are included in the current error correlation form.
75
+
76
+ :return: shape of included dimensions
77
+ """
78
+ all_dims = self._obj[self._unc_var_name].dims
79
+ all_dims_sizes = self._obj.sizes
80
+
81
+ return tuple([all_dims_sizes[dim] for dim in all_dims if dim in self.dims])
82
+
83
+ def get_sliced_dim_sizes_uncvar(self, sli: tuple) -> dict:
84
+ """
85
+ Return dictionary with sizes of sliced dimensions of unc variable, including all dimensions.
86
+
87
+ :param sli: slice (tuple with slice for each dimension)
88
+ :return: dictionary with shape of included sliced dimensions
89
+ """
90
+ uncvar_dims = self._obj[self._unc_var_name][sli].dims
91
+ uncvar_shape = self._obj[self._unc_var_name][sli].shape
92
+ return {
93
+ uncvar_dims[idim]: uncvar_shape[idim] for idim in range(len(uncvar_dims))
94
+ }
95
+
96
+ def get_sliced_dim_sizes_errcorr(self, sli: tuple) -> dict:
97
+ """
98
+ Return dictionary with sizes of sliced dimensions of unc variable, including only dimensions which are
99
+ included in the current error correlation form.
100
+
101
+ :param sli: slice (tuple with slice for each dimension)
102
+ :return: dictionary with shape of included sliced dimensions
103
+ """
104
+ uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
105
+ sliced_dims = self.get_sliced_dims_errcorr(sli)
106
+
107
+ return {dim: uncvar_sizes[dim] for dim in sliced_dims}
108
+
109
+ def get_sliced_dims_errcorr(self, sli: tuple) -> list:
110
+ """
111
+ Return dimensions which are within the slice and included in the current error correlation form.
112
+
113
+ :param sli: slice (tuple with slice for each dimension)
114
+ :return: list with sliced dimensions
115
+ """
116
+ all_dims = self._obj[self._unc_var_name].dims
117
+ return [
118
+ all_dims[idim]
119
+ for idim in range(len(all_dims))
120
+ if (isinstance(sli[idim], slice) and all_dims[idim] in self.dims)
121
+ ]
122
+
123
+ def get_sliced_shape_errcorr(self, sli: tuple) -> tuple:
124
+ """
125
+ return shape of sliced uncertainty variable, including only dimensions which are included in the current error correlation form.
126
+
127
+ :param sli: slice (tuple with slice for each dimension)
128
+ :return: shape of included sliced dimensions
129
+ """
130
+ uncvar_sizes = self.get_sliced_dim_sizes_uncvar(sli)
131
+ sliced_dims = self.get_sliced_dims_errcorr(sli)
132
+
133
+ return tuple([uncvar_sizes[dim] for dim in sliced_dims])
134
+
135
+ def slice_errcorr_matrix(self, err_corr_matrix, variable_shape, sli) -> np.ndarray:
136
+ """
137
+ Slice the provided error correlation matrix (typically the error correlation matrix of the
138
+ BaseErrCorrForm) using the
139
+
140
+ :param err_corr_matrix: error correlation matrix to be sliced
141
+ :param variable_shape: tuple with the length of the dimensions in the error correlation matrix (in correct order for flattening)
142
+ :param sli: slice of observation variable to return error-correlation matrix for
143
+ :return: sliced error correlation matrix
144
+ """
145
+ mask_array = np.ones(variable_shape, dtype=bool)
146
+ mask_array[sli] = False
147
+
148
+ return np.delete(
149
+ np.delete(err_corr_matrix, mask_array.ravel(), 0), mask_array.ravel(), 1
150
+ )
151
+
152
+ @abc.abstractmethod
153
+ def build_matrix(self, sli: Union[np.ndarray, tuple]) -> np.ndarray:
154
+ """
155
+ Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
156
+ in this parameterisation
157
+
158
+ :param sli: slice of observation variable to return error-correlation matrix for
159
+
160
+ :return: populated error-correlation matrix
161
+ """
162
+ pass
163
+
164
+ def build_dot_matrix(self, sli: Union[np.ndarray, tuple]) -> np.ndarray:
165
+ """
166
+ Returns expanded error correlation matrix for use in dot product with error correlation
167
+ in other dimensions.
168
+
169
+ The (sliced) error correlation matrix for this BaseErrCorrForm is expanded from its current
170
+ (sliced) dimensions (which often don't include all dimensions of the associated uncertainty
171
+ variable) to the dimensions of the full (sliced) error correlation (i.e. all dimensions of
172
+ the uncertainty).
173
+ The returned matrix is not meaningfull unless combined in a dot product with the expanded
174
+ matrices of other error correlation matrices (together spanning all uncertainty dimensions).
175
+
176
+ :param sli: slice of observation variable to return error-correlation matrix for
177
+ :return: expanded matrix for use in dot product with error correlation in other dimensions.
178
+ """
179
+ return expand_errcorr_dims(
180
+ in_corr=self.build_matrix(sli),
181
+ in_dim=self.get_sliced_dims_errcorr(sli),
182
+ out_dim=list(self._obj[self._unc_var_name][sli].dims),
183
+ dim_sizes=self.get_sliced_dim_sizes_uncvar(sli),
184
+ )
185
+
186
+
187
+ def register_err_corr_form(form_name: str) -> Callable:
188
+ """
189
+ Decorator for registering error-correlation parmaterisation form definition to package definition container
190
+
191
+ :param form_name: name of error-correlation parmaterisation form e.g. "random"
192
+ :return: decorator function
193
+ """
194
+
195
+ def decorator(form_cls: Type) -> Type:
196
+ err_corr_forms[form_name] = form_cls
197
+ return form_cls
198
+
199
+ return decorator
200
+
201
+
202
+ @register_err_corr_form("random")
203
+ class RandomCorrelation(BaseErrCorrForm):
204
+
205
+ form = "random"
206
+ is_random = True
207
+
208
+ def build_matrix(self, sli: tuple) -> np.ndarray:
209
+ """
210
+ Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
211
+ in this parameterisation
212
+
213
+ :param sli: slice of observation variable to return error-correlation matrix for
214
+
215
+ :return: populated error-correlation matrix
216
+ """
217
+
218
+ # evaluate correlation over matrices in form defintion
219
+ dim_lens = self.get_sliced_shape_errcorr(sli)
220
+ n_elems = int(np.prod(dim_lens))
221
+
222
+ submatrix = np.eye(n_elems)
223
+
224
+ return submatrix
225
+
226
+
227
+ @register_err_corr_form("systematic")
228
+ class SystematicCorrelation(BaseErrCorrForm):
229
+
230
+ form = "systematic"
231
+ is_systematic = True
232
+
233
+ def build_matrix(self, sli: tuple) -> np.ndarray:
234
+ """
235
+ Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
236
+ in this parameterisation
237
+
238
+ :param sli: slice of observation variable to return error-correlation matrix for
239
+
240
+ :return: populated error-correlation matrix
241
+ """
242
+
243
+ # evaluate correlation over matrices in form defintion
244
+ dim_lens = self.get_sliced_shape_errcorr(sli)
245
+ n_elems = int(np.prod(dim_lens))
246
+
247
+ submatrix = np.ones((n_elems, n_elems))
248
+
249
+ return submatrix
250
+
251
+
252
+ @register_err_corr_form("err_corr_matrix")
253
+ class ErrCorrMatrixCorrelation(BaseErrCorrForm):
254
+
255
+ form = "err_corr_matrix"
256
+
257
+ def build_matrix(self, sli: tuple) -> np.ndarray:
258
+ """
259
+ Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
260
+ in this parameterisation
261
+
262
+ :param sli: slice of observation variable to return error-correlation matrix for
263
+
264
+ :return: populated error-correlation matrix
265
+ """
266
+ all_dims = self._obj[self._unc_var_name].dims
267
+
268
+ sli_submatrix = tuple(
269
+ [sli[i] for i in range(len(all_dims)) if all_dims[i] in self.dims]
270
+ )
271
+
272
+ submatrix = self.slice_errcorr_matrix(
273
+ self._obj[self.params[0]], self.get_varshape_errcorr(), sli_submatrix
274
+ )
275
+
276
+ return submatrix
277
+
278
+
279
+ @register_err_corr_form("ensemble")
280
+ class EnsembleCorrelation(BaseErrCorrForm):
281
+
282
+ form = "ensemble"
283
+
284
+ def build_matrix(self, sli: tuple) -> np.ndarray:
285
+ """
286
+ Returns uncertainty effect error-correlation matrix, populated with error-correlation values defined
287
+ in this parameterisation
288
+
289
+ :param sli: slice of observation variable to return error-correlation matrix for
290
+
291
+ :return: populated error-correlation matrix
292
+ """
293
+
294
+ raise NotImplementedError
295
+
296
+
297
+ if __name__ == "__main__":
298
+ pass
@@ -194,12 +194,12 @@ class FlagVariable:
194
194
  )
195
195
 
196
196
  if flag_meaning not in flag_meanings:
197
- self._obj[
198
- self._flag_var_name
199
- ].attrs = DatasetUtil.add_flag_meaning_to_attrs(
200
- self._obj[self._flag_var_name].attrs,
201
- flag_meaning,
202
- self._obj[self._flag_var_name].dtype,
197
+ self._obj[self._flag_var_name].attrs = (
198
+ DatasetUtil.add_flag_meaning_to_attrs(
199
+ self._obj[self._flag_var_name].attrs,
200
+ flag_meaning,
201
+ self._obj[self._flag_var_name].dtype,
202
+ )
203
203
  )
204
204
 
205
205
  self[flag_meaning][:] = flag_value
@@ -259,8 +259,8 @@ class DatasetUtil:
259
259
  attributes=attributes,
260
260
  )
261
261
 
262
- #initialise flags to zero (instead of fillvalue)
263
- variable.values=0*variable.values
262
+ # initialise flags to zero (instead of fillvalue)
263
+ variable.values = 0 * variable.values
264
264
 
265
265
  # add flag attributes
266
266
  variable.attrs.update(DatasetUtil.pack_flag_attrs(meanings))
@@ -290,7 +290,7 @@ class DatasetUtil:
290
290
  )
291
291
 
292
292
  if flag_masks is None:
293
- flag_masks = [2 ** i for i in range(0, n_masks)]
293
+ flag_masks = [2**i for i in range(0, n_masks)]
294
294
 
295
295
  flag_attrs["flag_masks"] = str(flag_masks)[1:-1]
296
296
 
@@ -333,7 +333,7 @@ class DatasetUtil:
333
333
 
334
334
  # check if variable for available flags
335
335
  max_n_flags = numpy.iinfo(dtype).bits
336
- all_flag_masks = [2 ** i for i in range(0, max_n_flags)]
336
+ all_flag_masks = [2**i for i in range(0, max_n_flags)]
337
337
  available_flag_masks = list(set(all_flag_masks) - set(flag_masks))
338
338
 
339
339
  if not available_flag_masks:
@@ -434,6 +434,10 @@ class DatasetUtil:
434
434
 
435
435
  if fill_value is not None:
436
436
  encoding_dict.update({"_FillValue": fill_value})
437
+ else:
438
+ encoding_dict.update(
439
+ {"_FillValue": DatasetUtil.get_default_fill_value(dtype)}
440
+ )
437
441
 
438
442
  variable.encoding = encoding_dict
439
443
 
@@ -524,8 +528,8 @@ class DatasetUtil:
524
528
 
525
529
  flags_ds = DatasetUtil.unpack_flags(da)
526
530
 
527
- flags = flags if flags is not None else flags_ds.variables
528
- mask_flags = [flags_ds[flag].values for flag in flags]
531
+ flags = flags if flags is not None else flags_ds.data_vars
532
+ mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
529
533
 
530
534
  return numpy.logical_or.reduce(mask_flags)
531
535
 
@@ -546,8 +550,8 @@ class DatasetUtil:
546
550
 
547
551
  flags_ds = DatasetUtil.unpack_flags(da)
548
552
 
549
- flags = flags if flags is not None else flags_ds.variables
550
- mask_flags = [flags_ds[flag].values for flag in flags]
553
+ flags = flags if flags is not None else flags_ds.data_vars
554
+ mask_flags = numpy.array([flags_ds[flag].values for flag in flags])
551
555
 
552
556
  return numpy.logical_and.reduce(mask_flags)
553
557
 
@@ -81,7 +81,8 @@ class DSWriter:
81
81
  )
82
82
  }
83
83
  )
84
- ds[var_name].attrs.pop("_FillValue")
84
+ if "_FillValue" in ds[var_name].attrs:
85
+ ds[var_name].attrs.pop("_FillValue")
85
86
  encoding.update({var_name: var_encoding})
86
87
 
87
88
  ds.to_netcdf(path, format="netCDF4", engine="netcdf4", encoding=encoding)
@@ -159,6 +159,7 @@ class TemplateUtil:
159
159
 
160
160
  if "encoding" in var_attrs:
161
161
  du.add_encoding(variable, **var_attrs["encoding"])
162
+ variable.attrs.pop("_FillValue")
162
163
 
163
164
  return variable
164
165
 
@@ -0,0 +1,251 @@
1
+ """test_err_corr_forms - tests for obsarray.err_corr_forms"""
2
+
3
+ import unittest
4
+ import numpy as np
5
+ from obsarray.err_corr import (
6
+ BaseErrCorrForm,
7
+ ErrCorrForms,
8
+ RandomCorrelation,
9
+ SystematicCorrelation,
10
+ ErrCorrMatrixCorrelation,
11
+ )
12
+ from obsarray.test.test_unc_accessor import create_ds
13
+
14
+ __author__ = "Sam Hunt <sam.hunt@npl.co.uk>"
15
+ __all__ = []
16
+
17
+
18
+ class TestErrCorrForms(unittest.TestCase):
19
+ def setUp(self):
20
+ class ErrCorrFormDef(BaseErrCorrForm):
21
+ def form(self):
22
+ pass
23
+
24
+ self.ErrCorrFormDef = ErrCorrFormDef
25
+
26
+ def test___setitem__(self):
27
+ err_corr_forms = ErrCorrForms()
28
+ err_corr_forms["test"] = self.ErrCorrFormDef
29
+
30
+ self.assertCountEqual(list(err_corr_forms._forms.keys()), ["test"])
31
+
32
+ def test___setitem___not_valid_cls(self):
33
+ class InvalidErrCorrFormDef:
34
+ pass
35
+
36
+ err_corr_forms = ErrCorrForms()
37
+ self.assertRaises(
38
+ TypeError, err_corr_forms.__setitem__, "test", InvalidErrCorrFormDef
39
+ )
40
+
41
+ def test___getitem__(self):
42
+ err_corr_forms = ErrCorrForms()
43
+ err_corr_forms._forms["test"] = "test"
44
+ self.assertEqual("test", err_corr_forms["test"])
45
+
46
+ def test___delitem__(self):
47
+ err_corr_forms = ErrCorrForms()
48
+ err_corr_forms._forms["test"] = "test"
49
+
50
+ del err_corr_forms._forms["test"]
51
+
52
+ self.assertTrue("test" not in err_corr_forms._forms)
53
+
54
+ def test_keys(self):
55
+ err_corr_forms = ErrCorrForms()
56
+ err_corr_forms._forms["test"] = "test"
57
+
58
+ self.assertCountEqual(["test"], err_corr_forms.keys())
59
+
60
+
61
+ class TestBaseErrCorrForm(unittest.TestCase):
62
+ def setUp(self) -> None:
63
+ self.ds = create_ds()
64
+
65
+ class BasicErrCorrForm(BaseErrCorrForm):
66
+ form = "basic"
67
+
68
+ def build_matrix(self, sli):
69
+ full_matrix = np.arange(144).reshape((12, 12))
70
+ return self.slice_errcorr_matrix(full_matrix, (2, 2, 3), sli)
71
+
72
+ self.BasicErrCorrForm = BasicErrCorrForm
73
+
74
+ def test_get_varshape_errcorr(self):
75
+ basicerrcorr = self.BasicErrCorrForm(
76
+ self.ds, "u_ran_temperature", ["x", "y", "time"], [], []
77
+ )
78
+ shape = basicerrcorr.get_varshape_errcorr()
79
+ np.testing.assert_equal(shape, (2, 2, 3))
80
+
81
+ def test_slice_full_cov_full(self):
82
+ basicerrcorr = self.BasicErrCorrForm(
83
+ self.ds, "u_ran_temperature", ["x"], [], []
84
+ )
85
+
86
+ full_matrix = np.arange(144).reshape((12, 12))
87
+ slice_matrix = basicerrcorr.build_matrix(
88
+ (slice(None), slice(None), slice(None))
89
+ )
90
+
91
+ np.testing.assert_equal(full_matrix, slice_matrix)
92
+
93
+ def test_get_sliced_dim_sizes_uncvar(self):
94
+ basicerrcorr = self.BasicErrCorrForm(
95
+ self.ds, "u_ran_temperature", ["x"], [], []
96
+ )
97
+ dim_sizes = basicerrcorr.get_sliced_dim_sizes_uncvar(
98
+ (slice(None), 0, slice(0, 2, 1))
99
+ )
100
+ assert dim_sizes == {"x": 2, "time": 2}
101
+
102
+ def test_get_sliced_dim_sizes_errcorr(self):
103
+ basicerrcorr = self.BasicErrCorrForm(
104
+ self.ds, "u_ran_temperature", ["x"], [], []
105
+ )
106
+ dim_sizes = basicerrcorr.get_sliced_dim_sizes_errcorr(
107
+ (slice(None), 0, slice(0, 2, 1))
108
+ )
109
+ assert dim_sizes == {"x": 2}
110
+
111
+ def test_get_sliced_dims_errcorr(self):
112
+ basicerrcorr = self.BasicErrCorrForm(
113
+ self.ds, "u_ran_temperature", ["x"], [], []
114
+ )
115
+ dims = basicerrcorr.get_sliced_dims_errcorr((slice(None), 0, slice(0, 2, 1)))
116
+ assert dims == ["x"]
117
+
118
+ def test_get_sliced_shape_errcorr(self):
119
+ basicerrcorr = self.BasicErrCorrForm(
120
+ self.ds, "u_ran_temperature", ["x"], [], []
121
+ )
122
+ shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
123
+ assert shape == (2,)
124
+ basicerrcorr = self.BasicErrCorrForm(
125
+ self.ds, "u_ran_temperature", ["x", "time"], [], []
126
+ )
127
+ shape = basicerrcorr.get_sliced_shape_errcorr((slice(None), 0, slice(0, 2, 1)))
128
+ assert shape == (2, 2)
129
+
130
+ def test_slice_errcorr_matrix(self):
131
+ basicerrcorr = self.BasicErrCorrForm(
132
+ self.ds, "u_ran_temperature", ["x", "y", "z"], [], []
133
+ )
134
+
135
+ full_matrix = np.arange(144).reshape((12, 12))
136
+ slice_matrix = basicerrcorr.slice_errcorr_matrix(
137
+ full_matrix, (2, 2, 3), (slice(None), slice(None), 0)
138
+ )
139
+
140
+ exp_slice_matrix = np.array(
141
+ [[0, 3, 6, 9], [36, 39, 42, 45], [72, 75, 78, 81], [108, 111, 114, 117]]
142
+ )
143
+
144
+ np.testing.assert_equal(slice_matrix, exp_slice_matrix)
145
+
146
+ def test_slice_full_cov_slice(self):
147
+ basicerrcorr = self.BasicErrCorrForm(
148
+ self.ds, "u_ran_temperature", ["x", "y", "z"], [], []
149
+ )
150
+
151
+ slice_matrix = basicerrcorr.build_dot_matrix((slice(None), slice(None), 0))
152
+
153
+ exp_slice_matrix = np.array(
154
+ [[0, 3, 6, 9], [36, 39, 42, 45], [72, 75, 78, 81], [108, 111, 114, 117]]
155
+ )
156
+
157
+ np.testing.assert_equal(slice_matrix, exp_slice_matrix)
158
+
159
+
160
+ class TestRandomUnc(unittest.TestCase):
161
+ def setUp(self) -> None:
162
+ self.ds = create_ds()
163
+
164
+ def test_build_matrix_1stdim(self):
165
+ rc = RandomCorrelation(self.ds, "u_ran_temperature", ["x"], [], [])
166
+
167
+ ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
168
+
169
+ np.testing.assert_equal(ecrm, np.eye(12))
170
+
171
+ def test_build_matrix_2nddim(self):
172
+ rc = RandomCorrelation(self.ds, "u_ran_temperature", ["y"], [], [])
173
+
174
+ ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
175
+
176
+ np.testing.assert_equal(ecrm, np.eye(12))
177
+
178
+
179
+ class TestSystematicUnc(unittest.TestCase):
180
+ def setUp(self) -> None:
181
+ self.ds = create_ds()
182
+
183
+ def build_matrix_1stdim(self):
184
+ rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["x"], [], [])
185
+
186
+ ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
187
+ # np.testing.assert_equal(ecrm, np.ones((12, 12)))
188
+
189
+ return ecrm
190
+
191
+ def build_matrix_2nddim(self):
192
+ rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["y"], [], [])
193
+
194
+ ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
195
+ # np.testing.assert_equal(ecrm, np.ones((12, 12)))
196
+
197
+ return ecrm
198
+
199
+ def build_matrix_3ddim(self):
200
+ rc = SystematicCorrelation(self.ds, "u_sys_temperature", ["time"], [], [])
201
+
202
+ ecrm = rc.build_dot_matrix((slice(None), slice(None), slice(None)))
203
+ # np.testing.assert_equal(ecrm, np.ones((12, 12)))
204
+
205
+ return ecrm
206
+
207
+ def test_build_dot_matrix(self):
208
+ x = self.build_matrix_1stdim()
209
+ y = self.build_matrix_2nddim()
210
+ time = self.build_matrix_3ddim()
211
+ print(x.dot(y), x, y)
212
+ np.testing.assert_equal((x.dot(y)).dot(time), np.ones((12, 12)))
213
+
214
+
215
+ class TestErrCorrMatrixCorrelation(unittest.TestCase):
216
+ def setUp(self) -> None:
217
+ self.ds = create_ds()
218
+
219
+ def test_build_matrix_full(self):
220
+ ec = ErrCorrMatrixCorrelation(
221
+ self.ds,
222
+ "u_str_temperature",
223
+ ["x", "time"],
224
+ ["err_corr_str_temperature"],
225
+ [],
226
+ )
227
+
228
+ ecrm = ec.build_matrix((slice(None), slice(None), slice(None)))
229
+ np.testing.assert_equal(ecrm, np.ones((6, 6)))
230
+
231
+ def test_build_matrix_sliced(self):
232
+ ec = ErrCorrMatrixCorrelation(
233
+ self.ds,
234
+ "u_str_temperature",
235
+ ["x", "time"],
236
+ ["err_corr_str_temperature"],
237
+ [],
238
+ )
239
+
240
+ ecrm = ec.build_matrix((0, slice(None), slice(None)))
241
+ np.testing.assert_equal(ecrm, np.ones((3, 3)))
242
+
243
+ ecrm = ec.build_matrix((slice(None), 0, slice(None)))
244
+ np.testing.assert_equal(ecrm, np.ones((6, 6)))
245
+
246
+ ecrm = ec.build_matrix((slice(None), slice(None), 0))
247
+ np.testing.assert_equal(ecrm, np.ones((2, 2)))
248
+
249
+
250
+ if __name__ == "main":
251
+ unittest.main()