imas-python 0.1.dev1618__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imas-python might be problematic. Click here for more details.
- imas/__init__.py +34 -0
- imas/__main__.py +17 -0
- imas/_util.py +260 -0
- imas/_version.py +16 -0
- imas/assets/IDSDef2MDSpreTree.xsl +161 -0
- imas/assets/IDS_fake_toplevel.xml +318 -0
- imas/assets/IDS_minimal.xml +37 -0
- imas/assets/IDS_minimal_2.xml +37 -0
- imas/assets/IDS_minimal_struct_array.xml +43 -0
- imas/assets/IDS_minimal_types.xml +169 -0
- imas/assets/ITER_134173_106_core_profiles.ids +2438 -0
- imas/assets/ITER_134173_106_equilibrium.ids +3172 -0
- imas/assets/README.md +5 -0
- imas/assets/core_profiles.ids +2438 -0
- imas/assets/equilibrium.ids +3172 -0
- imas/backends/__init__.py +12 -0
- imas/backends/db_entry_impl.py +122 -0
- imas/backends/imas_core/__init__.py +4 -0
- imas/backends/imas_core/al_context.py +391 -0
- imas/backends/imas_core/db_entry_al.py +405 -0
- imas/backends/imas_core/db_entry_helpers.py +200 -0
- imas/backends/imas_core/imas_interface.py +216 -0
- imas/backends/imas_core/mdsplus_model.py +367 -0
- imas/backends/imas_core/uda_support.py +56 -0
- imas/backends/netcdf/__init__.py +4 -0
- imas/backends/netcdf/db_entry_nc.py +176 -0
- imas/backends/netcdf/ids2nc.py +306 -0
- imas/backends/netcdf/nc2ids.py +302 -0
- imas/backends/netcdf/nc_metadata.py +357 -0
- imas/backends/netcdf/nc_validate.py +53 -0
- imas/command/cli.py +239 -0
- imas/command/db_analysis.py +290 -0
- imas/command/helpers.py +45 -0
- imas/command/timer.py +77 -0
- imas/db_entry.py +802 -0
- imas/dd_helpers.py +300 -0
- imas/dd_zip.py +297 -0
- imas/exception.py +107 -0
- imas/ids_base.py +149 -0
- imas/ids_convert.py +920 -0
- imas/ids_coordinates.py +400 -0
- imas/ids_data_type.py +115 -0
- imas/ids_defs.py +171 -0
- imas/ids_factory.py +108 -0
- imas/ids_identifiers.py +125 -0
- imas/ids_metadata.py +307 -0
- imas/ids_path.py +254 -0
- imas/ids_primitive.py +510 -0
- imas/ids_struct_array.py +233 -0
- imas/ids_structure.py +331 -0
- imas/ids_toplevel.py +381 -0
- imas/setup_logging.py +64 -0
- imas/test/test_all_dd_versions.py +28 -0
- imas/test/test_cli.py +44 -0
- imas/test/test_dbentry.py +104 -0
- imas/test/test_dd_helpers.py +134 -0
- imas/test/test_dd_zip.py +29 -0
- imas/test/test_deepcopy.py +25 -0
- imas/test/test_exception.py +12 -0
- imas/test/test_get_sample.py +388 -0
- imas/test/test_hash.py +188 -0
- imas/test/test_helpers.py +354 -0
- imas/test/test_identifiers.py +105 -0
- imas/test/test_ids2nc.py +166 -0
- imas/test/test_ids_ascii_data.py +39 -0
- imas/test/test_ids_convert.py +415 -0
- imas/test/test_ids_coordinates.py +165 -0
- imas/test/test_ids_data_type.py +40 -0
- imas/test/test_ids_factory.py +35 -0
- imas/test/test_ids_metadata.py +66 -0
- imas/test/test_ids_mixin.py +20 -0
- imas/test/test_ids_path.py +192 -0
- imas/test/test_ids_primitive.py +109 -0
- imas/test/test_ids_struct_array.py +89 -0
- imas/test/test_ids_structure.py +70 -0
- imas/test/test_ids_toplevel.py +55 -0
- imas/test/test_ids_validate.py +343 -0
- imas/test/test_latest_dd_autofill.py +74 -0
- imas/test/test_latest_dd_manual.py +38 -0
- imas/test/test_latest_dd_resample.py +102 -0
- imas/test/test_lazy_loading.py +201 -0
- imas/test/test_list_occurrences.py +82 -0
- imas/test/test_minimal.py +37 -0
- imas/test/test_minimal_conversion.py +44 -0
- imas/test/test_minimal_io.py +21 -0
- imas/test/test_minimal_struct_array_io.py +60 -0
- imas/test/test_minimal_types.py +436 -0
- imas/test/test_minimal_types_io.py +70 -0
- imas/test/test_minimal_types_io_automatic.py +66 -0
- imas/test/test_nbc_change.py +408 -0
- imas/test/test_nc_autofill.py +13 -0
- imas/test/test_nc_entry.py +28 -0
- imas/test/test_nc_metadata.py +153 -0
- imas/test/test_nc_validation.py +197 -0
- imas/test/test_snippets.py +25 -0
- imas/test/test_static_ids.py +39 -0
- imas/test/test_str_1d.py +99 -0
- imas/test/test_time_slicing.py +182 -0
- imas/test/test_uda_support.py +12 -0
- imas/test/test_util.py +209 -0
- imas/training.py +45 -0
- imas/util.py +526 -0
- imas_python-0.1.dev1618.dist-info/LICENSE.txt +165 -0
- imas_python-0.1.dev1618.dist-info/METADATA +288 -0
- imas_python-0.1.dev1618.dist-info/RECORD +108 -0
- imas_python-0.1.dev1618.dist-info/WHEEL +5 -0
- imas_python-0.1.dev1618.dist-info/entry_points.txt +3 -0
- imas_python-0.1.dev1618.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""A testcase checking if writing and then reading works for the latest full
|
|
2
|
+
data dictionary version.
|
|
3
|
+
|
|
4
|
+
We then specifically check certain fields which have been renamed between versions,
|
|
5
|
+
by writing them as the old and reading as new and vice-versa
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
from imas.db_entry import DBEntry
|
|
14
|
+
from imas.ids_convert import convert_ids
|
|
15
|
+
from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS, MEMORY_BACKEND
|
|
16
|
+
from imas.ids_factory import IDSFactory
|
|
17
|
+
from imas.test.test_helpers import (
|
|
18
|
+
compare_children,
|
|
19
|
+
fill_with_random_data,
|
|
20
|
+
open_dbentry,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.fixture(autouse=True)
|
|
25
|
+
def debug_log(caplog):
|
|
26
|
+
"""Make sure we capture all debug output when tests fail."""
|
|
27
|
+
caplog.set_level(logging.DEBUG, "imas.ids_convert")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_nbc_structure_to_aos(caplog):
|
|
31
|
+
# coils_non_axisymmetric/coil/conductor/cross_section was a structure in 3.39.0 and
|
|
32
|
+
# is an AoS in 3.40.0 and later:
|
|
33
|
+
ids = IDSFactory("3.39.0").new("coils_non_axisymmetric")
|
|
34
|
+
ids.coil.resize(1)
|
|
35
|
+
ids.coil[0].conductor.resize(1)
|
|
36
|
+
ids.coil[0].conductor[0].cross_section.delta_r = [1.0]
|
|
37
|
+
|
|
38
|
+
# Note: there is no corresponding attribute in DD 3.40.0, so this will just resize
|
|
39
|
+
# the cross_section AoS to 1...
|
|
40
|
+
ids_340 = convert_ids(ids, "3.40.0")
|
|
41
|
+
assert len(ids_340.coil[0].conductor[0].cross_section) == 1
|
|
42
|
+
|
|
43
|
+
# The reverse operation has nothing to copy, just ensure it works okay
|
|
44
|
+
caplog.clear()
|
|
45
|
+
with caplog.at_level(logging.WARNING):
|
|
46
|
+
convert_ids(ids_340, "3.39.0")
|
|
47
|
+
assert len(caplog.record_tuples) == 0
|
|
48
|
+
|
|
49
|
+
# Conversion reports a warning when the AOS has size >1
|
|
50
|
+
ids_340.coil[0].conductor[0].cross_section.resize(2)
|
|
51
|
+
with caplog.at_level(logging.WARNING):
|
|
52
|
+
convert_ids(ids_340, "3.39.0")
|
|
53
|
+
assert len(caplog.record_tuples) == 1
|
|
54
|
+
assert caplog.record_tuples[0][:2] == ("imas.ids_convert", logging.WARNING)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_nbc_0d_to_1d(caplog):
|
|
58
|
+
# channel/filter_spectrometer/radiance_calibration in spectrometer visible changed
|
|
59
|
+
# from FLT_0D to FLT_1D in DD 3.39.0
|
|
60
|
+
ids = IDSFactory("3.32.0").spectrometer_visible()
|
|
61
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
62
|
+
ids.channel.resize(1)
|
|
63
|
+
ids.channel[0].filter_spectrometer.radiance_calibration = 1.0
|
|
64
|
+
|
|
65
|
+
# Convert to 3.39.0, when type changed to FLT_1D
|
|
66
|
+
ids_339 = convert_ids(ids, "3.39.0")
|
|
67
|
+
assert np.array_equal(
|
|
68
|
+
ids_339.channel[0].filter_spectrometer.radiance_calibration, [1.0]
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# Convert back
|
|
72
|
+
ids_back = convert_ids(ids_339, "3.32.0")
|
|
73
|
+
assert ids_back.channel[0].filter_spectrometer.radiance_calibration == 1.0
|
|
74
|
+
|
|
75
|
+
# This is not supported
|
|
76
|
+
ids_339.channel[0].filter_spectrometer.radiance_calibration = [1.0, 2.0]
|
|
77
|
+
caplog.clear()
|
|
78
|
+
with caplog.at_level(logging.WARNING):
|
|
79
|
+
ids_back = convert_ids(ids_339, "3.32.0")
|
|
80
|
+
assert not ids_back.channel[0].filter_spectrometer.radiance_calibration.has_value
|
|
81
|
+
assert len(caplog.record_tuples) == 1
|
|
82
|
+
assert caplog.record_tuples[0][:2] == ("imas.ids_convert", logging.WARNING)
|
|
83
|
+
|
|
84
|
+
# Test implicit conversion during get / put
|
|
85
|
+
entry_339 = DBEntry(MEMORY_BACKEND, "test", 1, 1, dd_version="3.39.0")
|
|
86
|
+
entry_339.create()
|
|
87
|
+
entry_339.put(ids) # implicit conversion during put()
|
|
88
|
+
ids_339 = entry_339.get("spectrometer_visible")
|
|
89
|
+
assert not ids_339.channel[0].filter_spectrometer.radiance_calibration.has_value
|
|
90
|
+
|
|
91
|
+
entry_332 = DBEntry(MEMORY_BACKEND, "test", 1, 1, dd_version="3.32.0")
|
|
92
|
+
entry_332.open()
|
|
93
|
+
ids_back = entry_332.get("spectrometer_visible") # implicit conversion back
|
|
94
|
+
assert not ids_back.channel[0].filter_spectrometer.radiance_calibration.has_value
|
|
95
|
+
|
|
96
|
+
# Note: closing entry_332 as well results in a double free, so we don't
|
|
97
|
+
entry_339.close()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_nbc_change_aos_renamed():
|
|
101
|
+
"""Test renamed AoS in pulse_schedule: ec/antenna -> ec/launcher.
|
|
102
|
+
|
|
103
|
+
Also tests renamed structures:
|
|
104
|
+
- ec/antenna/launching_angle_pol -> ec/launcher/steering_angle_pol
|
|
105
|
+
- ec/antenna/launching_angle_tor -> ec/launcher/steering_angle_tor
|
|
106
|
+
"""
|
|
107
|
+
# AOS was renamed at v3.26.0. NBC metadata introduced in 3.28.0
|
|
108
|
+
ps = IDSFactory("3.28.0").new("pulse_schedule")
|
|
109
|
+
ps.ec.launcher.resize(2)
|
|
110
|
+
for i in range(2):
|
|
111
|
+
ps.ec.launcher[i].name = f"test{i}"
|
|
112
|
+
|
|
113
|
+
# Test conversion from 3.28.0 -> 3.25.0
|
|
114
|
+
ps2 = convert_ids(ps, "3.25.0")
|
|
115
|
+
assert len(ps2.ec.antenna.value) == 2
|
|
116
|
+
for i in range(2):
|
|
117
|
+
assert ps2.ec.antenna[i].name == f"test{i}"
|
|
118
|
+
|
|
119
|
+
# Test conversion from 3.25.0 -> 3.28.0
|
|
120
|
+
ps3 = convert_ids(ps2, "3.28.0")
|
|
121
|
+
assert len(ps3.ec.launcher.value) == 2
|
|
122
|
+
for i in range(2):
|
|
123
|
+
assert ps3.ec.launcher[i].name == f"test{i}"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_nbc_change_leaf_renamed():
|
|
127
|
+
"""Test renamed leaf in reflectometer_profile: position/r/data -> position/r"""
|
|
128
|
+
# Leaf was renamed at 3.23.3. NBC metadata introduced in 3.28.0
|
|
129
|
+
rp = IDSFactory("3.28.0").new("reflectometer_profile")
|
|
130
|
+
rp.channel.resize(1)
|
|
131
|
+
data = np.linspace([0, 1, 2], [1, 2, 3], 5)
|
|
132
|
+
rp.channel[0].position.r = data
|
|
133
|
+
|
|
134
|
+
# Test conversion from 3.28.0 -> 3.23.0
|
|
135
|
+
rp2 = convert_ids(rp, "3.23.0")
|
|
136
|
+
assert np.array_equal(rp2.channel[0].position.r.data.value, data)
|
|
137
|
+
|
|
138
|
+
# Test conversion from 3.23.0 -> 3.28.0
|
|
139
|
+
rp3 = convert_ids(rp2, "3.28.0")
|
|
140
|
+
assert np.array_equal(rp3.channel[0].position.r.value, data)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_ids_convert_deepcopy():
|
|
144
|
+
time = np.linspace(0, 1, 10)
|
|
145
|
+
|
|
146
|
+
cp = IDSFactory("3.28.0").new("core_profiles")
|
|
147
|
+
cp.time = time
|
|
148
|
+
assert cp.time.value is time
|
|
149
|
+
|
|
150
|
+
cp2 = convert_ids(cp, "3.28.0") # Converting to the same version should also work
|
|
151
|
+
assert cp2.time.value is time
|
|
152
|
+
|
|
153
|
+
cp3 = convert_ids(cp, "3.28.0", deepcopy=True)
|
|
154
|
+
assert cp3.time.value is not time
|
|
155
|
+
assert np.array_equal(cp3.time.value, time)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_pulse_schedule_aos_renamed_up(backend, worker_id, tmp_path):
|
|
159
|
+
"""pulse_schedule/ec/launcher was renamed from pulse_schedule/ec/antenna
|
|
160
|
+
in version 3.26.0."""
|
|
161
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.28.0")
|
|
162
|
+
ids = IDSFactory("3.25.0").new("pulse_schedule")
|
|
163
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
164
|
+
ids.ec.antenna.resize(1)
|
|
165
|
+
ids.ec.antenna[0].name = "test"
|
|
166
|
+
|
|
167
|
+
# Test automatic conversion up
|
|
168
|
+
dbentry.put(ids)
|
|
169
|
+
|
|
170
|
+
# Now load back and ensure no conversion is done
|
|
171
|
+
ids2 = dbentry.get("pulse_schedule")
|
|
172
|
+
assert ids2.ec.launcher[0].name.value == "test"
|
|
173
|
+
dbentry.close()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_pulse_schedule_aos_renamed_autodetect_up(backend, worker_id, tmp_path):
|
|
177
|
+
"""pulse_schedule/ec/launcher was renamed from pulse_schedule/ec/antenna
|
|
178
|
+
in version 3.26.0."""
|
|
179
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.25.0")
|
|
180
|
+
ids = dbentry.factory.new("pulse_schedule")
|
|
181
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
182
|
+
ids.ec.antenna.resize(1)
|
|
183
|
+
ids.ec.antenna[0].name = "test"
|
|
184
|
+
|
|
185
|
+
# No conversion required
|
|
186
|
+
dbentry.put(ids)
|
|
187
|
+
|
|
188
|
+
# Now load back with a newer dbentry version, which does a conversion
|
|
189
|
+
dbentry2 = open_dbentry(backend, "r", worker_id, tmp_path, dd_version="3.28.0")
|
|
190
|
+
ids2 = dbentry2.get("pulse_schedule")
|
|
191
|
+
assert ids2.ec.launcher[0].name.value == "test"
|
|
192
|
+
|
|
193
|
+
dbentry.close()
|
|
194
|
+
if backend != MEMORY_BACKEND: # MEM backend already cleaned up, prevent SEGFAULT
|
|
195
|
+
dbentry2.close()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def test_pulse_schedule_aos_renamed_down(backend, worker_id, tmp_path):
|
|
199
|
+
"""pulse_schedule/ec/launcher was renamed from pulse_schedule/ec/antenna
|
|
200
|
+
in version 3.26.0."""
|
|
201
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.25.0")
|
|
202
|
+
ids = IDSFactory("3.28.0").new("pulse_schedule")
|
|
203
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
204
|
+
ids.ec.launcher.resize(1)
|
|
205
|
+
ids.ec.launcher[0].name = "test"
|
|
206
|
+
|
|
207
|
+
# Test automatic conversion down
|
|
208
|
+
dbentry.put(ids)
|
|
209
|
+
|
|
210
|
+
# Now load back and ensure no conversion is done
|
|
211
|
+
ids2 = dbentry.get("pulse_schedule")
|
|
212
|
+
assert ids2.ec.antenna[0].name.value == "test"
|
|
213
|
+
|
|
214
|
+
dbentry.close()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def test_pulse_schedule_aos_renamed_autodetect_down(backend, worker_id, tmp_path):
|
|
218
|
+
"""pulse_schedule/ec/launcher was renamed from pulse_schedule/ec/antenna
|
|
219
|
+
in version 3.26.0."""
|
|
220
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.28.0")
|
|
221
|
+
ids = dbentry.factory.new("pulse_schedule")
|
|
222
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
|
|
223
|
+
ids.ec.launcher.resize(1)
|
|
224
|
+
ids.ec.launcher[0].name = "test"
|
|
225
|
+
|
|
226
|
+
# No conversion required
|
|
227
|
+
dbentry.put(ids)
|
|
228
|
+
|
|
229
|
+
# Now load back with a newer dbentry version, which does a conversion
|
|
230
|
+
dbentry2 = open_dbentry(backend, "r", worker_id, tmp_path, dd_version="3.25.0")
|
|
231
|
+
ids2 = dbentry2.get("pulse_schedule")
|
|
232
|
+
assert ids2.ec.antenna[0].name.value == "test"
|
|
233
|
+
|
|
234
|
+
dbentry.close()
|
|
235
|
+
if backend != MEMORY_BACKEND: # MEM backend already cleaned up, prevent SEGFAULT
|
|
236
|
+
dbentry2.close()
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def test_pulse_schedule_aos_renamed_autofill_up(backend, worker_id, tmp_path):
|
|
240
|
+
"""pulse_schedule/ec/launcher was renamed from pulse_schedule/ec/antenna
|
|
241
|
+
in version 3.26.0."""
|
|
242
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.25.0")
|
|
243
|
+
ids = IDSFactory("3.28.0").new("pulse_schedule")
|
|
244
|
+
fill_with_random_data(ids)
|
|
245
|
+
dbentry.put(ids)
|
|
246
|
+
|
|
247
|
+
ids2 = dbentry.get("pulse_schedule")
|
|
248
|
+
|
|
249
|
+
# test the antenna/launcher only
|
|
250
|
+
assert len(ids.ec.launcher.value) == len(ids2.ec.antenna.value)
|
|
251
|
+
for ch1, ch2 in zip(ids.ec.launcher, ids2.ec.antenna):
|
|
252
|
+
# compare_children does not work since also the fields were changed.
|
|
253
|
+
# manually check the common fields
|
|
254
|
+
assert ch1.name == ch2.name
|
|
255
|
+
assert ch1.identifier == ch2.identifier
|
|
256
|
+
assert ch1.power_type.name == ch2.power_type.name
|
|
257
|
+
assert ch1.power_type.index == ch2.power_type.index
|
|
258
|
+
for new, old in [
|
|
259
|
+
(ch1.power, ch2.power),
|
|
260
|
+
(ch1.frequency, ch2.frequency),
|
|
261
|
+
(ch1.deposition_rho_tor_norm, ch2.deposition_rho_tor_norm),
|
|
262
|
+
(ch1.steering_angle_pol, ch2.launching_angle_pol),
|
|
263
|
+
(ch1.steering_angle_tor, ch2.launching_angle_tor),
|
|
264
|
+
]:
|
|
265
|
+
assert new.reference_name == old.reference_name
|
|
266
|
+
for name in ["data", "data_error_upper", "data_error_lower", "time"]:
|
|
267
|
+
assert np.array_equal(new.reference[name], old.reference[name])
|
|
268
|
+
assert new.reference.data_error_index == old.reference.data_error_index
|
|
269
|
+
assert np.array_equal(new.reference.time, old.reference.time)
|
|
270
|
+
assert new.reference_type == old.reference_type
|
|
271
|
+
assert new.envelope_type == old.envelope_type
|
|
272
|
+
dbentry.close()
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def test_pulse_schedule_multi_rename():
|
|
276
|
+
# Multiple renames of the same element:
|
|
277
|
+
# DD >= 3.40+: ec/beam
|
|
278
|
+
# DD 3.26-3.40: ec/launcher (but NBC metadata added in 3.28 only)
|
|
279
|
+
# DD < 3.26: ec/antenna
|
|
280
|
+
ps = {
|
|
281
|
+
version: IDSFactory(version).new("pulse_schedule")
|
|
282
|
+
for version in ["3.25.0", "3.30.0", "3.39.0", "3.40.0"]
|
|
283
|
+
}
|
|
284
|
+
for ids in ps.values():
|
|
285
|
+
ids.ids_properties.homogeneous_time = 0
|
|
286
|
+
name = "This is the name of the first item in the AOS"
|
|
287
|
+
ps["3.25.0"].ec.antenna.resize(1)
|
|
288
|
+
ps["3.25.0"].ec.antenna[0].name = name
|
|
289
|
+
ps["3.30.0"].ec.launcher.resize(1)
|
|
290
|
+
ps["3.30.0"].ec.launcher[0].name = name
|
|
291
|
+
ps["3.39.0"].ec.launcher.resize(1)
|
|
292
|
+
ps["3.39.0"].ec.launcher[0].name = name
|
|
293
|
+
ps["3.40.0"].ec.beam.resize(1)
|
|
294
|
+
ps["3.40.0"].ec.beam[0].name = name
|
|
295
|
+
|
|
296
|
+
for version1 in ps:
|
|
297
|
+
for version2 in ps:
|
|
298
|
+
converted = convert_ids(ps[version1], version2)
|
|
299
|
+
compare_children(ps[version2], converted)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def test_autofill_save_newer(ids_name, backend, worker_id, tmp_path):
|
|
303
|
+
"""Create an ids, autofill it, save it as a newer version, read it back
|
|
304
|
+
and check that it's the same.
|
|
305
|
+
|
|
306
|
+
TODO: we should also check newer IDSes, since this only checks variables that
|
|
307
|
+
existed in 3.25.0. Doing all versions for all IDSes is too slow however.
|
|
308
|
+
"""
|
|
309
|
+
dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.30.0")
|
|
310
|
+
factory = IDSFactory(version="3.25.0")
|
|
311
|
+
if not factory.exists(ids_name):
|
|
312
|
+
pytest.skip("IDS %s not defined for version 3.25.0" % (ids_name,))
|
|
313
|
+
ids = factory.new(ids_name)
|
|
314
|
+
fill_with_random_data(ids)
|
|
315
|
+
|
|
316
|
+
dbentry.put(ids)
|
|
317
|
+
|
|
318
|
+
dbentry2 = open_dbentry(backend, "r", worker_id, tmp_path, dd_version="3.25.0")
|
|
319
|
+
ids2 = dbentry2.get(ids_name)
|
|
320
|
+
|
|
321
|
+
# Some elements were removed between 3.25.0 and 3.30.0, so the conversion discards
|
|
322
|
+
# the affected data. Pass as deleted_paths to compare_children
|
|
323
|
+
deleted_paths = {
|
|
324
|
+
"coils_non_axisymmetric": {"is_periodic", "coils_n"},
|
|
325
|
+
"ece": {"channel/harmonic/data"},
|
|
326
|
+
"langmuir_probes": {
|
|
327
|
+
"embedded/j_ion_parallel/data",
|
|
328
|
+
"embedded/j_ion_parallel/validity_timed",
|
|
329
|
+
"embedded/j_ion_parallel/validity",
|
|
330
|
+
"reciprocating/plunge/potential_floating",
|
|
331
|
+
"reciprocating/plunge/t_e",
|
|
332
|
+
"reciprocating/plunge/t_i",
|
|
333
|
+
"reciprocating/plunge/saturation_current_ion",
|
|
334
|
+
"reciprocating/plunge/heat_flux_parallel",
|
|
335
|
+
},
|
|
336
|
+
"magnetics": {"method/diamagnetic_flux/data"},
|
|
337
|
+
"pulse_schedule": {
|
|
338
|
+
"ec/antenna/phase/reference_name",
|
|
339
|
+
"ec/antenna/phase/reference/data",
|
|
340
|
+
"ec/antenna/phase/reference/time",
|
|
341
|
+
"ec/antenna/phase/reference_type",
|
|
342
|
+
"ec/antenna/phase/envelope_type",
|
|
343
|
+
},
|
|
344
|
+
"spectrometer_x_ray_crystal": {
|
|
345
|
+
"camera/center/r",
|
|
346
|
+
"camera/center/z",
|
|
347
|
+
"camera/center/phi",
|
|
348
|
+
},
|
|
349
|
+
}.get(ids_name, [])
|
|
350
|
+
compare_children(ids, ids2, deleted_paths=deleted_paths)
|
|
351
|
+
|
|
352
|
+
# Compare outcome of implicit conversion at put with explicit convert_ids
|
|
353
|
+
implicit_3_30 = dbentry.get(ids_name)
|
|
354
|
+
explicit_3_30 = convert_ids(ids, version="3.30.0")
|
|
355
|
+
compare_children(implicit_3_30, explicit_3_30)
|
|
356
|
+
|
|
357
|
+
# Compare outcome of explicit conversion back to 3.25.0
|
|
358
|
+
compare_children(ids2, convert_ids(explicit_3_30, "3.25.0"))
|
|
359
|
+
|
|
360
|
+
dbentry.close()
|
|
361
|
+
if backend != MEMORY_BACKEND: # MEM backend already cleaned up, prevent SEGFAULT
|
|
362
|
+
dbentry2.close()
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def test_convert_min_to_max_v3(ids_name, latest_factory3):
|
|
366
|
+
"""Convert from DD 3.22.0 to the last DDv3 release."""
|
|
367
|
+
factory = IDSFactory("3.22.0")
|
|
368
|
+
if not factory.exists(ids_name):
|
|
369
|
+
pytest.skip("IDS %s not defined for version 3.22.0" % (ids_name,))
|
|
370
|
+
if not latest_factory3.exists(ids_name):
|
|
371
|
+
pytest.skip(f"IDS {ids_name} not defined for version {latest_factory3.version}")
|
|
372
|
+
|
|
373
|
+
ids = factory.new(ids_name)
|
|
374
|
+
fill_with_random_data(ids)
|
|
375
|
+
convert_ids(ids, latest_factory3.version)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def test_convert_max_to_min_v3(ids_name, latest_factory3):
|
|
379
|
+
"""Convert from the last DDv3 release to DD 3.22.0."""
|
|
380
|
+
factory = IDSFactory("3.22.0")
|
|
381
|
+
if not factory.exists(ids_name):
|
|
382
|
+
pytest.skip(f"IDS {ids_name} not defined for version 3.22.0")
|
|
383
|
+
if not latest_factory3.exists(ids_name):
|
|
384
|
+
pytest.skip(f"IDS {ids_name} not defined for version {latest_factory3.version}")
|
|
385
|
+
|
|
386
|
+
ids = latest_factory3.new(ids_name)
|
|
387
|
+
fill_with_random_data(ids)
|
|
388
|
+
convert_ids(ids, None, factory=factory)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def test_convert_3_to_newest(ids_name, latest_factory3, latest_factory):
|
|
392
|
+
"""Convert from the last DDv3 release to the last released DD."""
|
|
393
|
+
if not latest_factory3.exists(ids_name) or not latest_factory.exists(ids_name):
|
|
394
|
+
pytest.skip(f"IDS {ids_name} not defined for both versions.")
|
|
395
|
+
|
|
396
|
+
ids = latest_factory3.new(ids_name)
|
|
397
|
+
fill_with_random_data(ids)
|
|
398
|
+
convert_ids(ids, None, factory=latest_factory)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def test_convert_newest_to_3(ids_name, latest_factory3, latest_factory):
|
|
402
|
+
"""Convert from the last released DD to the last DDv3 release."""
|
|
403
|
+
if not latest_factory3.exists(ids_name) or not latest_factory.exists(ids_name):
|
|
404
|
+
pytest.skip(f"IDS {ids_name} not defined for both versions.")
|
|
405
|
+
|
|
406
|
+
ids = latest_factory.new(ids_name)
|
|
407
|
+
fill_with_random_data(ids)
|
|
408
|
+
convert_ids(ids, None, factory=latest_factory3)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from imas.db_entry import DBEntry
|
|
2
|
+
from imas.test.test_helpers import compare_children, fill_consistent
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_nc_latest_dd_autofill_put_get(ids_name, tmp_path):
|
|
6
|
+
with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry:
|
|
7
|
+
ids = entry.factory.new(ids_name)
|
|
8
|
+
fill_consistent(ids, 0.5)
|
|
9
|
+
|
|
10
|
+
entry.put(ids)
|
|
11
|
+
ids2 = entry.get(ids_name)
|
|
12
|
+
|
|
13
|
+
compare_children(ids, ids2)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from imas.db_entry import DBEntry
|
|
4
|
+
from imas.exception import DataEntryException
|
|
5
|
+
from imas.ids_defs import IDS_TIME_MODE_INDEPENDENT
|
|
6
|
+
from imas.ids_factory import IDSFactory
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_readwrite(tmp_path):
|
|
10
|
+
fname = tmp_path / "test-rw.nc"
|
|
11
|
+
ids = IDSFactory().core_profiles()
|
|
12
|
+
ids.ids_properties.homogeneous_time = IDS_TIME_MODE_INDEPENDENT
|
|
13
|
+
|
|
14
|
+
with pytest.raises(FileNotFoundError):
|
|
15
|
+
DBEntry(fname, "r") # File does not exist
|
|
16
|
+
with DBEntry(fname, "x") as entry:
|
|
17
|
+
entry.put(ids)
|
|
18
|
+
with DBEntry(fname, "w") as entry:
|
|
19
|
+
with pytest.raises(DataEntryException):
|
|
20
|
+
entry.get("core_profiles") # File overwritten, IDS does not exist
|
|
21
|
+
entry.put(ids)
|
|
22
|
+
with pytest.raises(OSError):
|
|
23
|
+
DBEntry(fname, "x") # file already exists
|
|
24
|
+
with DBEntry(fname, "a") as entry:
|
|
25
|
+
with pytest.raises(RuntimeError): # FIXME: change error class
|
|
26
|
+
entry.put(ids) # Cannot overwrite existing IDS
|
|
27
|
+
# But we can write a new occurrence
|
|
28
|
+
entry.put(ids, 1)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
from imas.backends.netcdf.nc_metadata import NCMetadata, _get_aos_label_coordinates
|
|
2
|
+
from imas.ids_factory import IDSFactory
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_generate_nc_metadata(ids_name):
|
|
6
|
+
ids = IDSFactory().new(ids_name)
|
|
7
|
+
NCMetadata(ids.metadata)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_get_aos_label_coordinates():
|
|
11
|
+
cp = IDSFactory("3.39.0").core_profiles()
|
|
12
|
+
pfa = IDSFactory("3.39.0").pf_active()
|
|
13
|
+
|
|
14
|
+
assert _get_aos_label_coordinates(cp.metadata["profiles_1d"]) == []
|
|
15
|
+
expected = ["profiles_1d.ion.label"]
|
|
16
|
+
assert _get_aos_label_coordinates(cp.metadata["profiles_1d/ion"]) == expected
|
|
17
|
+
expected = ["coil.name", "coil.identifier"]
|
|
18
|
+
assert _get_aos_label_coordinates(pfa.metadata["coil"]) == expected
|
|
19
|
+
expected = ["coil.element.name", "coil.element.identifier"]
|
|
20
|
+
assert _get_aos_label_coordinates(pfa.metadata["coil.element"]) == expected
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_aos_label_coordinates():
|
|
24
|
+
cp = NCMetadata(IDSFactory("3.39.0").core_profiles().metadata)
|
|
25
|
+
pfa = NCMetadata(IDSFactory("3.39.0").pf_active().metadata)
|
|
26
|
+
|
|
27
|
+
assert cp.get_dimensions("profiles_1d/ion", True) == ("time", "profiles_1d.ion:i")
|
|
28
|
+
assert cp.get_coordinates("profiles_1d/ion", True) == (
|
|
29
|
+
"time",
|
|
30
|
+
"profiles_1d.ion.label",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
assert pfa.get_dimensions("coil", True) == ("coil:i",)
|
|
34
|
+
assert pfa.get_coordinates("coil", True) == ("coil.name", "coil.identifier")
|
|
35
|
+
assert pfa.get_dimensions("coil/element", True) == ("coil:i", "coil.element:i")
|
|
36
|
+
assert pfa.get_coordinates("coil/element", True) == (
|
|
37
|
+
"coil.name",
|
|
38
|
+
"coil.identifier",
|
|
39
|
+
"coil.element.name",
|
|
40
|
+
"coil.element.identifier",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_time_mode():
|
|
45
|
+
cp = NCMetadata(IDSFactory("3.39.0").core_profiles().metadata)
|
|
46
|
+
mag = NCMetadata(IDSFactory("3.39.0").magnetics().metadata)
|
|
47
|
+
|
|
48
|
+
# These quantities always have root time as dimension
|
|
49
|
+
assert cp.get_dimensions("time", True) == ("time",)
|
|
50
|
+
assert cp.get_dimensions("time", False) == ("time",)
|
|
51
|
+
assert cp.get_dimensions("global_quantities/ip", True) == ("time",)
|
|
52
|
+
assert cp.get_dimensions("global_quantities/ip", False) == ("time",)
|
|
53
|
+
|
|
54
|
+
assert cp.get_coordinates("time", True) == ()
|
|
55
|
+
assert cp.get_coordinates("time", False) == ()
|
|
56
|
+
assert cp.get_coordinates("global_quantities/ip", True) == ("time",)
|
|
57
|
+
assert cp.get_coordinates("global_quantities/ip", False) == ("time",)
|
|
58
|
+
|
|
59
|
+
# Dynamic array of structures
|
|
60
|
+
assert cp.get_dimensions("profiles_1d", True) == ("time",)
|
|
61
|
+
assert cp.get_dimensions("profiles_1d", False) == ("profiles_1d.time",)
|
|
62
|
+
assert cp.get_dimensions("profiles_1d/grid/rho_tor", True)[0] == "time"
|
|
63
|
+
assert cp.get_dimensions("profiles_1d/grid/rho_tor", False)[0] == "profiles_1d.time"
|
|
64
|
+
|
|
65
|
+
assert cp.get_coordinates("profiles_1d", True) == ("time",)
|
|
66
|
+
assert cp.get_coordinates("profiles_1d", False) == ("profiles_1d.time",)
|
|
67
|
+
coors = cp.get_coordinates("profiles_1d/grid/rho_tor", True)
|
|
68
|
+
assert coors == ("time", "profiles_1d.grid.rho_tor_norm")
|
|
69
|
+
coors = cp.get_coordinates("profiles_1d/grid/rho_tor", False)
|
|
70
|
+
assert coors == ("profiles_1d.time", "profiles_1d.grid.rho_tor_norm")
|
|
71
|
+
|
|
72
|
+
# Sibling time nodes
|
|
73
|
+
assert mag.get_dimensions("flux_loop/flux/data", True) == ("flux_loop:i", "time")
|
|
74
|
+
dims = mag.get_dimensions("flux_loop/flux/data", False)
|
|
75
|
+
assert dims == ("flux_loop:i", "flux_loop.flux.time:i")
|
|
76
|
+
|
|
77
|
+
coors = mag.get_coordinates("flux_loop/flux/data", True)
|
|
78
|
+
assert coors == ("flux_loop.name", "flux_loop.identifier", "time")
|
|
79
|
+
coors = mag.get_coordinates("flux_loop/flux/data", False)
|
|
80
|
+
assert coors == ("flux_loop.name", "flux_loop.identifier", "flux_loop.flux.time")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_dd3_alternative_coordinates():
|
|
84
|
+
distr = NCMetadata(IDSFactory("3.39.0").distributions().metadata)
|
|
85
|
+
|
|
86
|
+
# Dimension names use the first listed of the alternatives
|
|
87
|
+
assert distr.get_dimensions("distribution/profiles_2d/density", True) == (
|
|
88
|
+
"distribution:i",
|
|
89
|
+
"time",
|
|
90
|
+
"distribution.profiles_2d.grid.r:i",
|
|
91
|
+
"distribution.profiles_2d.grid.z:i",
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Auxiliary coordinates list all, including the alternatives
|
|
95
|
+
assert distr.get_coordinates("distribution/profiles_2d/density", True) == (
|
|
96
|
+
"time",
|
|
97
|
+
"distribution.profiles_2d.grid.r",
|
|
98
|
+
"distribution.profiles_2d.grid.rho_tor_norm",
|
|
99
|
+
"distribution.profiles_2d.grid.z",
|
|
100
|
+
"distribution.profiles_2d.grid.theta_geometric",
|
|
101
|
+
"distribution.profiles_2d.grid.theta_straight",
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_dd4_alternative_coordinates():
|
|
106
|
+
cp = NCMetadata(IDSFactory("4.0.0").core_profiles().metadata)
|
|
107
|
+
|
|
108
|
+
# Dimension names use the first listed of the alternatives
|
|
109
|
+
dims = cp.get_dimensions("profiles_1d/j_phi", True)
|
|
110
|
+
assert dims == ("time", "profiles_1d.grid.rho_tor_norm:i")
|
|
111
|
+
|
|
112
|
+
# Auxiliary coordinates list all, including the alternatives
|
|
113
|
+
assert cp.get_coordinates("profiles_1d/j_phi", True) == (
|
|
114
|
+
"time",
|
|
115
|
+
"profiles_1d.grid.rho_tor_norm",
|
|
116
|
+
"profiles_1d.grid.rho_tor",
|
|
117
|
+
"profiles_1d.grid.psi",
|
|
118
|
+
"profiles_1d.grid.volume",
|
|
119
|
+
"profiles_1d.grid.area",
|
|
120
|
+
"profiles_1d.grid.surface",
|
|
121
|
+
"profiles_1d.grid.rho_pol_norm",
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_coordinate_same_as():
|
|
126
|
+
cir = NCMetadata(IDSFactory("3.39.0").camera_ir().metadata)
|
|
127
|
+
|
|
128
|
+
# Both dimensions have coordinate_same_as set:
|
|
129
|
+
dims = cir.get_dimensions("calibration/transmission_barrel", True)
|
|
130
|
+
assert dims == ("frame.surface_temperature:i", "frame.surface_temperature:j")
|
|
131
|
+
|
|
132
|
+
# coordinate_same_as shouldn't become auxiliary coordinates
|
|
133
|
+
assert cir.get_coordinates("calibration/transmission_barrel", True) == ()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_tensorization():
|
|
137
|
+
eq = NCMetadata(IDSFactory("3.39.0").equilibrium().metadata)
|
|
138
|
+
|
|
139
|
+
assert eq.get_dimensions(
|
|
140
|
+
"grids_ggd/grid/space/objects_per_dimension/object/geometry", False
|
|
141
|
+
) == (
|
|
142
|
+
"grids_ggd.time",
|
|
143
|
+
"grids_ggd.grid:i",
|
|
144
|
+
"grids_ggd.grid.space:i",
|
|
145
|
+
"grids_ggd.grid.space.objects_per_dimension:i",
|
|
146
|
+
"grids_ggd.grid.space.objects_per_dimension.object:i",
|
|
147
|
+
"grids_ggd.grid.space.objects_per_dimension.object.geometry:i",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
coors = eq.get_coordinates(
|
|
151
|
+
"grids_ggd/grid/space/objects_per_dimension/object/geometry", False
|
|
152
|
+
)
|
|
153
|
+
assert coors == ("grids_ggd.time",)
|