imas-python 1.2.1.dev13__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imas-python might be problematic. Click here for more details.
- imas/_to_xarray.py +73 -0
- imas/_version.py +9 -4
- imas/assets/IDSDef.zip +0 -0
- imas/backends/imas_core/al_context.py +13 -0
- imas/backends/imas_core/db_entry_helpers.py +3 -7
- imas/backends/imas_core/imas_interface.py +1 -1
- imas/backends/imas_core/mdsplus_model.py +34 -35
- imas/backends/netcdf/db_entry_nc.py +13 -12
- imas/backends/netcdf/ids2nc.py +5 -160
- imas/backends/netcdf/ids_tensorizer.py +205 -0
- imas/backends/netcdf/iterators.py +98 -0
- imas/backends/netcdf/nc2ids.py +176 -61
- imas/backends/netcdf/nc_validate.py +2 -1
- imas/ids_convert.py +2 -1
- imas/ids_structure.py +8 -8
- imas/test/test_ids_convert.py +2 -1
- imas/test/test_lazy_loading.py +53 -4
- imas/test/test_nbc_change.py +29 -8
- imas/test/test_nc_validation.py +26 -17
- imas/test/test_to_xarray.py +94 -0
- imas/util.py +75 -3
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/METADATA +16 -8
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/RECORD +27 -23
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/WHEEL +1 -1
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/LICENSE.txt +0 -0
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/entry_points.txt +0 -0
- {imas_python-1.2.1.dev13.dist-info → imas_python-2.0.0.dist-info}/top_level.txt +0 -0
imas/ids_structure.py
CHANGED
|
@@ -6,11 +6,10 @@
|
|
|
6
6
|
import logging
|
|
7
7
|
from copy import deepcopy
|
|
8
8
|
from types import MappingProxyType
|
|
9
|
-
from typing import Generator, List, Optional, Union
|
|
9
|
+
from typing import TYPE_CHECKING, Generator, List, Optional, Union
|
|
10
10
|
|
|
11
11
|
from xxhash import xxh3_64
|
|
12
12
|
|
|
13
|
-
from imas.backends.imas_core.al_context import LazyALContext
|
|
14
13
|
from imas.ids_base import IDSBase, IDSDoc
|
|
15
14
|
from imas.ids_identifiers import IDSIdentifier
|
|
16
15
|
from imas.ids_metadata import IDSDataType, IDSMetadata
|
|
@@ -18,6 +17,9 @@ from imas.ids_path import IDSPath
|
|
|
18
17
|
from imas.ids_primitive import IDSPrimitive
|
|
19
18
|
from imas.ids_struct_array import IDSStructArray
|
|
20
19
|
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from imas.backends.imas_core.al_context import LazyALContext
|
|
22
|
+
|
|
21
23
|
logger = logging.getLogger(__name__)
|
|
22
24
|
|
|
23
25
|
|
|
@@ -32,7 +34,7 @@ class IDSStructure(IDSBase):
|
|
|
32
34
|
|
|
33
35
|
__doc__ = IDSDoc(__doc__)
|
|
34
36
|
_children: "MappingProxyType[str, IDSMetadata]"
|
|
35
|
-
_lazy_context: Optional[LazyALContext]
|
|
37
|
+
_lazy_context: Optional["LazyALContext"]
|
|
36
38
|
|
|
37
39
|
def __init__(self, parent: IDSBase, metadata: IDSMetadata):
|
|
38
40
|
"""Initialize IDSStructure from metadata specification
|
|
@@ -62,10 +64,8 @@ class IDSStructure(IDSBase):
|
|
|
62
64
|
child_meta = self._children[name]
|
|
63
65
|
child = child_meta._node_type(self, child_meta)
|
|
64
66
|
self.__dict__[name] = child # bypass setattr logic below: avoid recursion
|
|
65
|
-
if self._lazy: # lazy load the child
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
_get_child(child, self._lazy_context)
|
|
67
|
+
if self._lazy and self._lazy_context is not None: # lazy load the child
|
|
68
|
+
self._lazy_context.get_child(child)
|
|
69
69
|
return child
|
|
70
70
|
|
|
71
71
|
def _assign_identifier(self, value: Union[IDSIdentifier, str, int]) -> None:
|
|
@@ -168,7 +168,7 @@ class IDSStructure(IDSBase):
|
|
|
168
168
|
return False # Not equal if there is any difference
|
|
169
169
|
return True # Equal when there are no differences
|
|
170
170
|
|
|
171
|
-
def _set_lazy_context(self, ctx: LazyALContext) -> None:
|
|
171
|
+
def _set_lazy_context(self, ctx: "LazyALContext") -> None:
|
|
172
172
|
"""Called by DBEntry during a lazy get/get_slice.
|
|
173
173
|
|
|
174
174
|
Set the context that we can use for retrieving our children.
|
imas/test/test_ids_convert.py
CHANGED
|
@@ -200,7 +200,8 @@ def test_provenance_entry(factory):
|
|
|
200
200
|
timestamp = str(cp4.ids_properties.provenance.node[0].reference[0].timestamp)
|
|
201
201
|
# Check that timestamp adheres to the format YYYY-MM-DDTHH:MM:SSZ
|
|
202
202
|
assert re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", timestamp)
|
|
203
|
-
|
|
203
|
+
timestamp_for_parsing = timestamp.replace("Z", "+00:00")
|
|
204
|
+
dtime = datetime.now(UTC) - datetime.fromisoformat(timestamp_for_parsing)
|
|
204
205
|
assert timedelta(seconds=0) <= dtime < timedelta(seconds=2)
|
|
205
206
|
|
|
206
207
|
|
imas/test/test_lazy_loading.py
CHANGED
|
@@ -3,7 +3,6 @@ from unittest.mock import DEFAULT, patch
|
|
|
3
3
|
|
|
4
4
|
import numpy
|
|
5
5
|
import pytest
|
|
6
|
-
|
|
7
6
|
from imas.backends.imas_core.imas_interface import ll_interface
|
|
8
7
|
from imas.db_entry import DBEntry
|
|
9
8
|
from imas.ids_defs import (
|
|
@@ -22,6 +21,15 @@ def test_lazy_load_aos(backend, worker_id, tmp_path, log_lowlevel_calls):
|
|
|
22
21
|
if backend == ASCII_BACKEND:
|
|
23
22
|
pytest.skip("Lazy loading is not supported by the ASCII backend.")
|
|
24
23
|
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.39.0")
|
|
24
|
+
run_lazy_load_aos(dbentry)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_lazy_load_aos_netcdf(tmp_path):
|
|
28
|
+
dbentry = DBEntry(str(tmp_path / "lazy_load_aos.nc"), "x", dd_version="3.39.0")
|
|
29
|
+
run_lazy_load_aos(dbentry)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run_lazy_load_aos(dbentry):
|
|
25
33
|
ids = dbentry.factory.new("core_profiles")
|
|
26
34
|
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
|
|
27
35
|
ids.profiles_1d.resize(10)
|
|
@@ -46,9 +54,12 @@ def test_lazy_load_aos(backend, worker_id, tmp_path, log_lowlevel_calls):
|
|
|
46
54
|
assert values[method].call_count == 0
|
|
47
55
|
|
|
48
56
|
# Test get_slice
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
57
|
+
try:
|
|
58
|
+
lazy_slice = dbentry.get_slice("core_profiles", 3.5, PREVIOUS_INTERP, lazy=True)
|
|
59
|
+
assert lazy_slice.profiles_1d.shape == (1,)
|
|
60
|
+
assert lazy_slice.profiles_1d[0].time == 3
|
|
61
|
+
except NotImplementedError:
|
|
62
|
+
pass # netCDF backend doesn't implement get_slice
|
|
52
63
|
|
|
53
64
|
dbentry.close()
|
|
54
65
|
|
|
@@ -57,6 +68,15 @@ def test_lazy_loading_distributions_random(backend, worker_id, tmp_path):
|
|
|
57
68
|
if backend == ASCII_BACKEND:
|
|
58
69
|
pytest.skip("Lazy loading is not supported by the ASCII backend.")
|
|
59
70
|
dbentry = open_dbentry(backend, "w", worker_id, tmp_path)
|
|
71
|
+
run_lazy_loading_distributions_random(dbentry)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_lazy_loading_distributions_random_netcdf(tmp_path):
|
|
75
|
+
dbentry = DBEntry(str(tmp_path / "lazy_load_distributions.nc"), "x")
|
|
76
|
+
run_lazy_loading_distributions_random(dbentry)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def run_lazy_loading_distributions_random(dbentry):
|
|
60
80
|
ids = IDSFactory().new("distributions")
|
|
61
81
|
fill_consistent(ids)
|
|
62
82
|
dbentry.put(ids)
|
|
@@ -92,7 +112,15 @@ def test_lazy_load_close_dbentry(requires_imas):
|
|
|
92
112
|
def test_lazy_load_readonly(requires_imas):
|
|
93
113
|
dbentry = DBEntry(MEMORY_BACKEND, "ITER", 1, 1)
|
|
94
114
|
dbentry.create()
|
|
115
|
+
run_lazy_load_readonly(dbentry)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_lazy_load_readonly_netcdf(tmp_path):
|
|
119
|
+
dbentry = DBEntry(str(tmp_path / "lazy_load_readonly.nc"), "x")
|
|
120
|
+
run_lazy_load_readonly(dbentry)
|
|
95
121
|
|
|
122
|
+
|
|
123
|
+
def run_lazy_load_readonly(dbentry):
|
|
96
124
|
ids = dbentry.factory.core_profiles()
|
|
97
125
|
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
|
|
98
126
|
ids.time = [1, 2]
|
|
@@ -165,6 +193,27 @@ def test_lazy_load_with_new_aos(requires_imas):
|
|
|
165
193
|
dbentry.close()
|
|
166
194
|
|
|
167
195
|
|
|
196
|
+
def test_lazy_load_with_new_aos_netcdf(tmp_path):
|
|
197
|
+
fname = str(tmp_path / "new_aos.nc")
|
|
198
|
+
with DBEntry(fname, "x", dd_version="3.30.0") as dbentry:
|
|
199
|
+
et = dbentry.factory.edge_transport()
|
|
200
|
+
|
|
201
|
+
et.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
202
|
+
et.time = [1.0]
|
|
203
|
+
et.model.resize(1)
|
|
204
|
+
et.model[0].ggd.resize(1)
|
|
205
|
+
et.model[0].ggd[0].electrons.particles.d.resize(1)
|
|
206
|
+
et.model[0].ggd[0].electrons.particles.d[0].grid_index = -1
|
|
207
|
+
dbentry.put(et)
|
|
208
|
+
|
|
209
|
+
with DBEntry(fname, "r", dd_version="3.39.0") as entry2:
|
|
210
|
+
lazy_et = entry2.get("edge_transport", lazy=True)
|
|
211
|
+
assert numpy.array_equal(lazy_et.time, [1.0])
|
|
212
|
+
assert lazy_et.model[0].ggd[0].electrons.particles.d[0].grid_index == -1
|
|
213
|
+
# d_radial did not exist in 3.30.0
|
|
214
|
+
assert len(lazy_et.model[0].ggd[0].electrons.particles.d_radial) == 0
|
|
215
|
+
|
|
216
|
+
|
|
168
217
|
def test_lazy_load_with_new_structure(requires_imas):
|
|
169
218
|
dbentry = DBEntry(MEMORY_BACKEND, "ITER", 1, 1, dd_version="3.30.0")
|
|
170
219
|
dbentry.create()
|
imas/test/test_nbc_change.py
CHANGED
|
@@ -9,16 +9,11 @@ import logging
|
|
|
9
9
|
|
|
10
10
|
import numpy as np
|
|
11
11
|
import pytest
|
|
12
|
-
|
|
13
12
|
from imas.db_entry import DBEntry
|
|
14
13
|
from imas.ids_convert import convert_ids
|
|
15
14
|
from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS, MEMORY_BACKEND
|
|
16
15
|
from imas.ids_factory import IDSFactory
|
|
17
|
-
from imas.test.test_helpers import
|
|
18
|
-
compare_children,
|
|
19
|
-
fill_with_random_data,
|
|
20
|
-
open_dbentry,
|
|
21
|
-
)
|
|
16
|
+
from imas.test.test_helpers import compare_children, fill_with_random_data, open_dbentry
|
|
22
17
|
|
|
23
18
|
|
|
24
19
|
@pytest.fixture(autouse=True)
|
|
@@ -97,6 +92,23 @@ def test_nbc_0d_to_1d(caplog, requires_imas):
|
|
|
97
92
|
entry_339.close()
|
|
98
93
|
|
|
99
94
|
|
|
95
|
+
def test_nbc_0d_to_1d_netcdf(caplog, tmp_path):
|
|
96
|
+
# channel/filter_spectrometer/radiance_calibration in spectrometer visible changed
|
|
97
|
+
# from FLT_0D to FLT_1D in DD 3.39.0
|
|
98
|
+
ids = IDSFactory("3.32.0").spectrometer_visible()
|
|
99
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
100
|
+
ids.channel.resize(1)
|
|
101
|
+
ids.channel[0].filter_spectrometer.radiance_calibration = 1.0
|
|
102
|
+
|
|
103
|
+
# Test implicit conversion during get
|
|
104
|
+
with DBEntry(str(tmp_path / "test.nc"), "x", dd_version="3.32.0") as entry_332:
|
|
105
|
+
entry_332.put(ids)
|
|
106
|
+
with DBEntry(str(tmp_path / "test.nc"), "r", dd_version="3.39.0") as entry_339:
|
|
107
|
+
ids_339 = entry_339.get("spectrometer_visible") # implicit conversion
|
|
108
|
+
assert not ids_339.channel[0].filter_spectrometer.radiance_calibration.has_value
|
|
109
|
+
entry_339.close()
|
|
110
|
+
|
|
111
|
+
|
|
100
112
|
def test_nbc_change_aos_renamed():
|
|
101
113
|
"""Test renamed AoS in pulse_schedule: ec/antenna -> ec/launcher.
|
|
102
114
|
|
|
@@ -272,7 +284,7 @@ def test_pulse_schedule_aos_renamed_autofill_up(backend, worker_id, tmp_path):
|
|
|
272
284
|
dbentry.close()
|
|
273
285
|
|
|
274
286
|
|
|
275
|
-
def test_pulse_schedule_multi_rename():
|
|
287
|
+
def test_pulse_schedule_multi_rename(tmp_path):
|
|
276
288
|
# Multiple renames of the same element:
|
|
277
289
|
# DD >= 3.40+: ec/beam
|
|
278
290
|
# DD 3.26-3.40: ec/launcher (but NBC metadata added in 3.28 only)
|
|
@@ -294,9 +306,18 @@ def test_pulse_schedule_multi_rename():
|
|
|
294
306
|
ps["3.40.0"].ec.beam[0].name = name
|
|
295
307
|
|
|
296
308
|
for version1 in ps:
|
|
309
|
+
ncfilename = str(tmp_path / f"{version1}.nc")
|
|
310
|
+
with DBEntry(ncfilename, "x", dd_version=version1) as entry:
|
|
311
|
+
entry.put(ps[version1])
|
|
312
|
+
|
|
297
313
|
for version2 in ps:
|
|
298
314
|
converted = convert_ids(ps[version1], version2)
|
|
299
|
-
compare_children(ps[version2], converted)
|
|
315
|
+
compare_children(ps[version2].ec, converted.ec)
|
|
316
|
+
|
|
317
|
+
# Test with netCDF backend
|
|
318
|
+
with DBEntry(ncfilename, "r", dd_version=version2) as entry:
|
|
319
|
+
converted = entry.get("pulse_schedule")
|
|
320
|
+
compare_children(ps[version2].ec, converted.ec)
|
|
300
321
|
|
|
301
322
|
|
|
302
323
|
def test_autofill_save_newer(ids_name, backend, worker_id, tmp_path):
|
imas/test/test_nc_validation.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import netCDF4
|
|
2
2
|
import numpy as np
|
|
3
3
|
import pytest
|
|
4
|
-
|
|
5
4
|
from imas.backends.netcdf.ids2nc import IDS2NC
|
|
6
5
|
from imas.backends.netcdf.nc2ids import NC2IDS
|
|
7
6
|
from imas.backends.netcdf.nc_validate import validate_netcdf_file
|
|
@@ -32,7 +31,8 @@ def memfile_with_ids(memfile, factory):
|
|
|
32
31
|
ids.profiles_1d[0].zeff = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
|
|
33
32
|
IDS2NC(ids, memfile).run()
|
|
34
33
|
# This one is valid:
|
|
35
|
-
|
|
34
|
+
ids = factory.core_profiles()
|
|
35
|
+
NC2IDS(memfile, ids, ids.metadata, None).run(lazy=False)
|
|
36
36
|
return memfile
|
|
37
37
|
|
|
38
38
|
|
|
@@ -51,66 +51,75 @@ def test_invalid_homogeneous_time(memfile, factory):
|
|
|
51
51
|
|
|
52
52
|
ids = factory.core_profiles()
|
|
53
53
|
with pytest.raises(InvalidNetCDFEntry):
|
|
54
|
-
|
|
54
|
+
# ids_properties.homogeneous_time does not exist
|
|
55
|
+
NC2IDS(empty_group, ids, ids.metadata, None)
|
|
55
56
|
with pytest.raises(InvalidNetCDFEntry):
|
|
56
|
-
NC2IDS(invalid_dtype, ids)
|
|
57
|
+
NC2IDS(invalid_dtype, ids, ids.metadata, None)
|
|
57
58
|
with pytest.raises(InvalidNetCDFEntry):
|
|
58
|
-
NC2IDS(invalid_shape, ids)
|
|
59
|
+
NC2IDS(invalid_shape, ids, ids.metadata, None)
|
|
59
60
|
with pytest.raises(InvalidNetCDFEntry):
|
|
60
|
-
NC2IDS(invalid_value, ids)
|
|
61
|
+
NC2IDS(invalid_value, ids, ids.metadata, None)
|
|
61
62
|
|
|
62
63
|
|
|
63
64
|
def test_invalid_units(memfile_with_ids, factory):
|
|
64
65
|
memfile_with_ids["time"].units = "hours"
|
|
66
|
+
ids = factory.core_profiles()
|
|
65
67
|
with pytest.raises(InvalidNetCDFEntry):
|
|
66
|
-
NC2IDS(memfile_with_ids,
|
|
68
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
67
69
|
|
|
68
70
|
|
|
69
71
|
def test_invalid_documentation(memfile_with_ids, factory, caplog):
|
|
72
|
+
ids = factory.core_profiles()
|
|
70
73
|
with caplog.at_level("WARNING"):
|
|
71
|
-
NC2IDS(memfile_with_ids,
|
|
74
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
72
75
|
assert not caplog.records
|
|
73
76
|
# Invalid docstring logs a warning
|
|
74
77
|
memfile_with_ids["time"].documentation = "https://en.wikipedia.org/wiki/Time"
|
|
75
78
|
with caplog.at_level("WARNING"):
|
|
76
|
-
NC2IDS(memfile_with_ids,
|
|
79
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
77
80
|
assert len(caplog.records) == 1
|
|
78
81
|
|
|
79
82
|
|
|
80
83
|
def test_invalid_dimension_name(memfile_with_ids, factory):
|
|
81
84
|
memfile_with_ids.renameDimension("time", "T")
|
|
85
|
+
ids = factory.core_profiles()
|
|
82
86
|
with pytest.raises(InvalidNetCDFEntry):
|
|
83
|
-
NC2IDS(memfile_with_ids,
|
|
87
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
84
88
|
|
|
85
89
|
|
|
86
90
|
def test_invalid_coordinates(memfile_with_ids, factory):
|
|
87
91
|
memfile_with_ids["profiles_1d.grid.rho_tor_norm"].coordinates = "xyz"
|
|
92
|
+
ids = factory.core_profiles()
|
|
88
93
|
with pytest.raises(InvalidNetCDFEntry):
|
|
89
|
-
NC2IDS(memfile_with_ids,
|
|
94
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
90
95
|
|
|
91
96
|
|
|
92
97
|
def test_invalid_ancillary_variables(memfile_with_ids, factory):
|
|
93
98
|
memfile_with_ids["time"].ancillary_variables = "xyz"
|
|
99
|
+
ids = factory.core_profiles()
|
|
94
100
|
with pytest.raises(InvalidNetCDFEntry):
|
|
95
|
-
NC2IDS(memfile_with_ids,
|
|
101
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
96
102
|
|
|
97
103
|
|
|
98
104
|
def test_extra_attributes(memfile_with_ids, factory):
|
|
99
105
|
memfile_with_ids["time"].new_attribute = [1, 2, 3]
|
|
106
|
+
ids = factory.core_profiles()
|
|
100
107
|
with pytest.raises(InvalidNetCDFEntry):
|
|
101
|
-
NC2IDS(memfile_with_ids,
|
|
108
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
102
109
|
|
|
103
110
|
|
|
104
111
|
def test_shape_array_without_data(memfile_with_ids, factory):
|
|
105
112
|
memfile_with_ids.createVariable("profiles_1d.t_i_average:shape", int, ())
|
|
113
|
+
ids = factory.core_profiles()
|
|
106
114
|
with pytest.raises(InvalidNetCDFEntry):
|
|
107
|
-
NC2IDS(memfile_with_ids,
|
|
115
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
108
116
|
|
|
109
117
|
|
|
110
118
|
def test_shape_array_without_sparse_data(memfile_with_ids, factory):
|
|
111
119
|
memfile_with_ids.createVariable("profiles_1d.grid.rho_tor_norm:shape", int, ())
|
|
120
|
+
ids = factory.core_profiles()
|
|
112
121
|
with pytest.raises(InvalidNetCDFEntry):
|
|
113
|
-
NC2IDS(memfile_with_ids,
|
|
122
|
+
NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False)
|
|
114
123
|
|
|
115
124
|
|
|
116
125
|
def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory):
|
|
@@ -128,7 +137,7 @@ def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory):
|
|
|
128
137
|
("time", "profiles_1d.grid.rho_tor_norm:i"),
|
|
129
138
|
)
|
|
130
139
|
with pytest.raises(InvalidNetCDFEntry):
|
|
131
|
-
NC2IDS(memfile_with_ids, cp).run()
|
|
140
|
+
NC2IDS(memfile_with_ids, cp, cp.metadata, None).run(lazy=False)
|
|
132
141
|
|
|
133
142
|
|
|
134
143
|
def test_shape_array_with_invalid_dtype(memfile_with_ids, factory):
|
|
@@ -144,7 +153,7 @@ def test_shape_array_with_invalid_dtype(memfile_with_ids, factory):
|
|
|
144
153
|
"profiles_1d.t_i_average:shape", float, ("time", "1D")
|
|
145
154
|
)
|
|
146
155
|
with pytest.raises(InvalidNetCDFEntry):
|
|
147
|
-
NC2IDS(memfile_with_ids, cp).run()
|
|
156
|
+
NC2IDS(memfile_with_ids, cp, cp.metadata, None).run(lazy=False)
|
|
148
157
|
|
|
149
158
|
|
|
150
159
|
def test_validate_nc(tmpdir):
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
import imas
|
|
5
|
+
import imas.training
|
|
6
|
+
from imas.util import to_xarray
|
|
7
|
+
|
|
8
|
+
pytest.importorskip("xarray")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture
|
|
12
|
+
def entry(requires_imas, monkeypatch):
|
|
13
|
+
monkeypatch.setenv("IMAS_VERSION", "3.39.0") # Use fixed DD version
|
|
14
|
+
return imas.training.get_training_db_entry()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_to_xarray_invalid_argtype():
|
|
18
|
+
ids = imas.IDSFactory("3.39.0").core_profiles()
|
|
19
|
+
|
|
20
|
+
with pytest.raises(TypeError):
|
|
21
|
+
to_xarray("test")
|
|
22
|
+
with pytest.raises(TypeError):
|
|
23
|
+
to_xarray(ids.time)
|
|
24
|
+
with pytest.raises(TypeError):
|
|
25
|
+
to_xarray(ids.ids_properties)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_to_xarray_invalid_paths():
|
|
29
|
+
ids = imas.IDSFactory("3.39.0").core_profiles()
|
|
30
|
+
|
|
31
|
+
with pytest.raises(ValueError, match="xyz"):
|
|
32
|
+
to_xarray(ids, "xyz")
|
|
33
|
+
with pytest.raises(ValueError, match="ids_properties/xyz"):
|
|
34
|
+
to_xarray(ids, "ids_properties/xyz")
|
|
35
|
+
with pytest.raises(ValueError, match="Xtime"):
|
|
36
|
+
to_xarray(ids, "time", "Xtime")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def validate_trainingdb_electron_temperature_dataset(ds):
|
|
40
|
+
assert ds.sizes == {"time": 3, "profiles_1d.grid.rho_tor_norm:i": 101}
|
|
41
|
+
assert ds.data_vars.keys() == {"profiles_1d.electrons.temperature"}
|
|
42
|
+
assert ds.coords.keys() == {"time", "profiles_1d.grid.rho_tor_norm"}
|
|
43
|
+
|
|
44
|
+
# Check that values are loaded as expected
|
|
45
|
+
assert np.allclose(ds["time"], [3.987222, 432.937598, 792.0])
|
|
46
|
+
assert np.allclose(
|
|
47
|
+
ds.isel(time=1)["profiles_1d.electrons.temperature"][10:13],
|
|
48
|
+
[17728.81703089, 17440.78020568, 17139.35431082],
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_to_xarray_lazy_loaded(entry):
|
|
53
|
+
ids = entry.get("core_profiles", lazy=True)
|
|
54
|
+
|
|
55
|
+
with pytest.raises(RuntimeError):
|
|
56
|
+
to_xarray(ids)
|
|
57
|
+
|
|
58
|
+
ds = to_xarray(ids, "profiles_1d.electrons.temperature")
|
|
59
|
+
validate_trainingdb_electron_temperature_dataset(ds)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_to_xarray_from_trainingdb(entry):
|
|
63
|
+
ids = entry.get("core_profiles")
|
|
64
|
+
|
|
65
|
+
ds = to_xarray(ids)
|
|
66
|
+
validate_trainingdb_electron_temperature_dataset(
|
|
67
|
+
ds["profiles_1d.electrons.temperature"].to_dataset()
|
|
68
|
+
)
|
|
69
|
+
ds = to_xarray(ids, "profiles_1d.electrons.temperature")
|
|
70
|
+
validate_trainingdb_electron_temperature_dataset(ds)
|
|
71
|
+
|
|
72
|
+
ds = to_xarray(
|
|
73
|
+
ids, "profiles_1d.electrons.temperature", "profiles_1d/electrons/density"
|
|
74
|
+
)
|
|
75
|
+
assert ds.data_vars.keys() == {
|
|
76
|
+
"profiles_1d.electrons.temperature",
|
|
77
|
+
"profiles_1d.electrons.density",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_to_xarray():
|
|
82
|
+
ids = imas.IDSFactory("3.39.0").core_profiles()
|
|
83
|
+
|
|
84
|
+
ids.profiles_1d.resize(2)
|
|
85
|
+
ids.profiles_1d[0].electrons.temperature = [1.0, 2.0]
|
|
86
|
+
ids.profiles_1d[0].grid.rho_tor_norm = [0.0, 1.0]
|
|
87
|
+
ids.profiles_1d[0].time = 0.0
|
|
88
|
+
|
|
89
|
+
# These should all be identical:
|
|
90
|
+
ds1 = to_xarray(ids)
|
|
91
|
+
ds2 = to_xarray(ids, "profiles_1d.electrons.temperature")
|
|
92
|
+
ds3 = to_xarray(ids, "profiles_1d/electrons/temperature")
|
|
93
|
+
assert ds1.equals(ds2)
|
|
94
|
+
assert ds2.equals(ds3)
|
imas/util.py
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
# This file is part of IMAS-Python.
|
|
2
2
|
# You should have received the IMAS-Python LICENSE file with this project.
|
|
3
|
-
"""Collection of useful helper methods when working with IMAS-Python.
|
|
4
|
-
"""
|
|
5
|
-
|
|
3
|
+
"""Collection of useful helper methods when working with IMAS-Python."""
|
|
6
4
|
|
|
7
5
|
import logging
|
|
8
6
|
import re
|
|
@@ -524,3 +522,77 @@ def get_data_dictionary_version(obj: Union[IDSBase, DBEntry, IDSFactory]) -> str
|
|
|
524
522
|
if isinstance(obj, IDSBase):
|
|
525
523
|
return obj._version
|
|
526
524
|
raise TypeError(f"Cannot get data dictionary version of '{type(obj)}'")
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def to_xarray(ids: IDSToplevel, *paths: str) -> Any:
|
|
528
|
+
"""Convert an IDS to an xarray Dataset.
|
|
529
|
+
|
|
530
|
+
Args:
|
|
531
|
+
ids: An IDS toplevel element
|
|
532
|
+
paths: Optional list of element paths to convert to xarray. The full IDS will be
|
|
533
|
+
converted to an xarray Dataset if no paths are provided.
|
|
534
|
+
|
|
535
|
+
Paths must not contain indices, and may use a ``/`` or a ``.`` as separator.
|
|
536
|
+
For example, ``"profiles_1d(itime)/electrons/density"`` is not allowed as
|
|
537
|
+
path, use ``"profiles_1d/electrons/density"`` or
|
|
538
|
+
``profiles_1d.electrons.density"`` instead.
|
|
539
|
+
|
|
540
|
+
Coordinates to the quantities in the requested paths will also be included
|
|
541
|
+
in the xarray Dataset.
|
|
542
|
+
|
|
543
|
+
Returns:
|
|
544
|
+
An ``xarray.Dataset`` object.
|
|
545
|
+
|
|
546
|
+
Notes:
|
|
547
|
+
- Lazy loaded IDSs are not supported for full IDS conversion
|
|
548
|
+
(``imas.util.to_xarray(ids)`` will raise an exception for lazy loaded IDSs).
|
|
549
|
+
This function can work with lazy loaded IDSs when paths are explicitly
|
|
550
|
+
provided: this might take a while because it will load all data for the
|
|
551
|
+
provided paths and their coordinates.
|
|
552
|
+
- This function does not accept wildcards for the paths. However, it is possible
|
|
553
|
+
to combine this method with :py:func:`imas.util.find_paths`, see the Examples
|
|
554
|
+
below.
|
|
555
|
+
- This function may return an empty dataset in the following cases:
|
|
556
|
+
|
|
557
|
+
- The provided IDS does not contain any data.
|
|
558
|
+
- The IDS does not contain any data for the provided paths.
|
|
559
|
+
- The provided paths do not point to data nodes, but to (arrays of)
|
|
560
|
+
structures.
|
|
561
|
+
|
|
562
|
+
Examples:
|
|
563
|
+
.. code-block:: python
|
|
564
|
+
|
|
565
|
+
# Convert the whole IDS to an xarray Dataset
|
|
566
|
+
ds = imas.util.to_xarray(ids)
|
|
567
|
+
|
|
568
|
+
# Convert only some elements in the IDS (including their coordinates)
|
|
569
|
+
ds = imas.util.to_xarray(
|
|
570
|
+
ids,
|
|
571
|
+
"profiles_1d/electrons/density",
|
|
572
|
+
"profiles_1d/electrons/temperature",
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
# Paths can be provided with "/" or "." as separator
|
|
576
|
+
ds = imas.util.to_xarray(
|
|
577
|
+
ids,
|
|
578
|
+
"profiles_1d.electrons.density",
|
|
579
|
+
"profiles_1d.electrons.temperature",
|
|
580
|
+
)
|
|
581
|
+
|
|
582
|
+
# Combine with imas.util.find_paths to include all paths containing
|
|
583
|
+
# "profiles_1d" in the xarray conversion:
|
|
584
|
+
profiles_1d_paths = imas.util.find_paths(ids, "profiles_1d")
|
|
585
|
+
assert len(profiles_1d_paths) > 0
|
|
586
|
+
ds = imas.util.to_xarray(ids, *profiles_1d_paths)
|
|
587
|
+
|
|
588
|
+
See Also:
|
|
589
|
+
https://docs.xarray.dev/en/stable/generated/xarray.Dataset.html
|
|
590
|
+
"""
|
|
591
|
+
try:
|
|
592
|
+
import xarray # noqa: F401
|
|
593
|
+
except ImportError:
|
|
594
|
+
raise RuntimeError("xarray is not available, cannot convert the IDS to xarray.")
|
|
595
|
+
|
|
596
|
+
from imas._to_xarray import to_xarray
|
|
597
|
+
|
|
598
|
+
return to_xarray(ids, *paths)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: imas-python
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Pythonic wrappers for the IMAS Access Layer
|
|
5
5
|
Author: ITER Organization
|
|
6
6
|
Author-email: Karel Lucas van de Plassche <karelvandeplassche@gmail.com>, Daan van Vugt <dvanvugt@ignitioncomputing.com>, Maarten Sebregts <msebregts@ignitioncomputing.com>, imas-support@iter.org
|
|
@@ -215,7 +215,7 @@ Requires-Dist: xxhash>=2
|
|
|
215
215
|
Requires-Dist: saxonche
|
|
216
216
|
Requires-Dist: gitpython
|
|
217
217
|
Provides-Extra: all
|
|
218
|
-
Requires-Dist: imas[docs,
|
|
218
|
+
Requires-Dist: imas-python[docs,test]; extra == "all"
|
|
219
219
|
Provides-Extra: docs
|
|
220
220
|
Requires-Dist: sphinx<7.0.0,>=6.0.0; extra == "docs"
|
|
221
221
|
Requires-Dist: sphinx-autosummary-accessors>=0.1.2; extra == "docs"
|
|
@@ -225,6 +225,8 @@ Provides-Extra: netcdf
|
|
|
225
225
|
Requires-Dist: netCDF4>=1.4.1; extra == "netcdf"
|
|
226
226
|
Provides-Extra: h5py
|
|
227
227
|
Requires-Dist: h5py; extra == "h5py"
|
|
228
|
+
Provides-Extra: xarray
|
|
229
|
+
Requires-Dist: xarray; extra == "xarray"
|
|
228
230
|
Provides-Extra: test
|
|
229
231
|
Requires-Dist: pytest>=5.4.1; extra == "test"
|
|
230
232
|
Requires-Dist: pytest-cov>=0.6; extra == "test"
|
|
@@ -234,13 +236,13 @@ Requires-Dist: matplotlib; extra == "test"
|
|
|
234
236
|
Requires-Dist: asv==0.6.1; extra == "test"
|
|
235
237
|
Requires-Dist: virtualenv; extra == "test"
|
|
236
238
|
Requires-Dist: pint; extra == "test"
|
|
237
|
-
Requires-Dist: xarray; extra == "test"
|
|
239
|
+
Requires-Dist: imas-python[h5py,netcdf,xarray]; extra == "test"
|
|
238
240
|
|
|
239
241
|
# IMAS-Python
|
|
240
242
|
|
|
241
|
-
IMAS-Python is a pure-python library to handle
|
|
242
|
-
It is designed for, but not necessarily bound to, interacting
|
|
243
|
-
Data Structures (IDSs) as defined by the Integrated Modelling & Analysis Suite (IMAS)
|
|
243
|
+
IMAS-Python (formerly known as IMASPy its for versions < 2) is a pure-python library to handle
|
|
244
|
+
arbitrarily nested data structures. It is designed for, but not necessarily bound to, interacting
|
|
245
|
+
with Interface Data Structures (IDSs) as defined by the Integrated Modelling & Analysis Suite (IMAS)
|
|
244
246
|
Data Model.
|
|
245
247
|
|
|
246
248
|
|
|
@@ -266,11 +268,17 @@ factory = imas.IDSFactory()
|
|
|
266
268
|
equilibrium = factory.equilibrium()
|
|
267
269
|
print(equilibrium)
|
|
268
270
|
|
|
269
|
-
equilibrium.ids_properties.homogeneous_time = imas.ids_defs.
|
|
271
|
+
equilibrium.ids_properties.homogeneous_time = imas.ids_defs.IDS_TIME_MODE_HOMOGENEOUS
|
|
270
272
|
equilibrium.ids_properties.comment = "testing"
|
|
273
|
+
equilibrium.time = [0.01]
|
|
271
274
|
|
|
272
|
-
with imas.DBEntry("imas:hdf5?path=./testdb","w") as dbentry:
|
|
275
|
+
with imas.DBEntry("imas:hdf5?path=./testdb", "w") as dbentry:
|
|
273
276
|
dbentry.put(equilibrium)
|
|
277
|
+
|
|
278
|
+
# or without imas_core dependency
|
|
279
|
+
with imas.DBEntry("./test.nc", "w") as dbentry:
|
|
280
|
+
dbentry.put(equilibrium)
|
|
281
|
+
|
|
274
282
|
```
|
|
275
283
|
|
|
276
284
|
A quick 5 minutes introduction is available in the documentation generated from `/docs/sources/intro.rst`.
|