imas-python 0.1.dev1618__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imas-python might be problematic. Click here for more details.
- imas/__init__.py +34 -0
- imas/__main__.py +17 -0
- imas/_util.py +260 -0
- imas/_version.py +16 -0
- imas/assets/IDSDef2MDSpreTree.xsl +161 -0
- imas/assets/IDS_fake_toplevel.xml +318 -0
- imas/assets/IDS_minimal.xml +37 -0
- imas/assets/IDS_minimal_2.xml +37 -0
- imas/assets/IDS_minimal_struct_array.xml +43 -0
- imas/assets/IDS_minimal_types.xml +169 -0
- imas/assets/ITER_134173_106_core_profiles.ids +2438 -0
- imas/assets/ITER_134173_106_equilibrium.ids +3172 -0
- imas/assets/README.md +5 -0
- imas/assets/core_profiles.ids +2438 -0
- imas/assets/equilibrium.ids +3172 -0
- imas/backends/__init__.py +12 -0
- imas/backends/db_entry_impl.py +122 -0
- imas/backends/imas_core/__init__.py +4 -0
- imas/backends/imas_core/al_context.py +391 -0
- imas/backends/imas_core/db_entry_al.py +405 -0
- imas/backends/imas_core/db_entry_helpers.py +200 -0
- imas/backends/imas_core/imas_interface.py +216 -0
- imas/backends/imas_core/mdsplus_model.py +367 -0
- imas/backends/imas_core/uda_support.py +56 -0
- imas/backends/netcdf/__init__.py +4 -0
- imas/backends/netcdf/db_entry_nc.py +176 -0
- imas/backends/netcdf/ids2nc.py +306 -0
- imas/backends/netcdf/nc2ids.py +302 -0
- imas/backends/netcdf/nc_metadata.py +357 -0
- imas/backends/netcdf/nc_validate.py +53 -0
- imas/command/cli.py +239 -0
- imas/command/db_analysis.py +290 -0
- imas/command/helpers.py +45 -0
- imas/command/timer.py +77 -0
- imas/db_entry.py +802 -0
- imas/dd_helpers.py +300 -0
- imas/dd_zip.py +297 -0
- imas/exception.py +107 -0
- imas/ids_base.py +149 -0
- imas/ids_convert.py +920 -0
- imas/ids_coordinates.py +400 -0
- imas/ids_data_type.py +115 -0
- imas/ids_defs.py +171 -0
- imas/ids_factory.py +108 -0
- imas/ids_identifiers.py +125 -0
- imas/ids_metadata.py +307 -0
- imas/ids_path.py +254 -0
- imas/ids_primitive.py +510 -0
- imas/ids_struct_array.py +233 -0
- imas/ids_structure.py +331 -0
- imas/ids_toplevel.py +381 -0
- imas/setup_logging.py +64 -0
- imas/test/test_all_dd_versions.py +28 -0
- imas/test/test_cli.py +44 -0
- imas/test/test_dbentry.py +104 -0
- imas/test/test_dd_helpers.py +134 -0
- imas/test/test_dd_zip.py +29 -0
- imas/test/test_deepcopy.py +25 -0
- imas/test/test_exception.py +12 -0
- imas/test/test_get_sample.py +388 -0
- imas/test/test_hash.py +188 -0
- imas/test/test_helpers.py +354 -0
- imas/test/test_identifiers.py +105 -0
- imas/test/test_ids2nc.py +166 -0
- imas/test/test_ids_ascii_data.py +39 -0
- imas/test/test_ids_convert.py +415 -0
- imas/test/test_ids_coordinates.py +165 -0
- imas/test/test_ids_data_type.py +40 -0
- imas/test/test_ids_factory.py +35 -0
- imas/test/test_ids_metadata.py +66 -0
- imas/test/test_ids_mixin.py +20 -0
- imas/test/test_ids_path.py +192 -0
- imas/test/test_ids_primitive.py +109 -0
- imas/test/test_ids_struct_array.py +89 -0
- imas/test/test_ids_structure.py +70 -0
- imas/test/test_ids_toplevel.py +55 -0
- imas/test/test_ids_validate.py +343 -0
- imas/test/test_latest_dd_autofill.py +74 -0
- imas/test/test_latest_dd_manual.py +38 -0
- imas/test/test_latest_dd_resample.py +102 -0
- imas/test/test_lazy_loading.py +201 -0
- imas/test/test_list_occurrences.py +82 -0
- imas/test/test_minimal.py +37 -0
- imas/test/test_minimal_conversion.py +44 -0
- imas/test/test_minimal_io.py +21 -0
- imas/test/test_minimal_struct_array_io.py +60 -0
- imas/test/test_minimal_types.py +436 -0
- imas/test/test_minimal_types_io.py +70 -0
- imas/test/test_minimal_types_io_automatic.py +66 -0
- imas/test/test_nbc_change.py +408 -0
- imas/test/test_nc_autofill.py +13 -0
- imas/test/test_nc_entry.py +28 -0
- imas/test/test_nc_metadata.py +153 -0
- imas/test/test_nc_validation.py +197 -0
- imas/test/test_snippets.py +25 -0
- imas/test/test_static_ids.py +39 -0
- imas/test/test_str_1d.py +99 -0
- imas/test/test_time_slicing.py +182 -0
- imas/test/test_uda_support.py +12 -0
- imas/test/test_util.py +209 -0
- imas/training.py +45 -0
- imas/util.py +526 -0
- imas_python-0.1.dev1618.dist-info/LICENSE.txt +165 -0
- imas_python-0.1.dev1618.dist-info/METADATA +288 -0
- imas_python-0.1.dev1618.dist-info/RECORD +108 -0
- imas_python-0.1.dev1618.dist-info/WHEEL +5 -0
- imas_python-0.1.dev1618.dist-info/entry_points.txt +3 -0
- imas_python-0.1.dev1618.dist-info/top_level.txt +1 -0
imas/test/test_hash.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import struct
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
from xxhash import xxh3_64_digest
|
|
6
|
+
|
|
7
|
+
import imas
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture
|
|
11
|
+
def minimal(ids_minimal_types):
|
|
12
|
+
return imas.IDSFactory(xml_path=ids_minimal_types).new("minimal")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_hash_str0d(minimal):
|
|
16
|
+
s = "Test str_0d hash"
|
|
17
|
+
minimal.str_0d = "Test str_0d hash"
|
|
18
|
+
expected = xxh3_64_digest(s.encode("utf-8"))
|
|
19
|
+
assert expected == b"r\x9d\x8dC.JN\x0e"
|
|
20
|
+
assert imas.util.calc_hash(minimal.str_0d) == expected
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_hash_str1d(minimal):
|
|
24
|
+
string_list = [
|
|
25
|
+
"Test str_1d hash",
|
|
26
|
+
"Of course, there must be ",
|
|
27
|
+
"multiple entries to test!",
|
|
28
|
+
]
|
|
29
|
+
minimal.str_1d = string_list
|
|
30
|
+
hashes = list(map(xxh3_64_digest, string_list))
|
|
31
|
+
expected = xxh3_64_digest(struct.pack("<Q", len(string_list)) + b"".join(hashes))
|
|
32
|
+
assert expected == b"\x98\x011\x9dx+\x0e\xc0"
|
|
33
|
+
assert imas.util.calc_hash(minimal.str_1d) == expected
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_hash_int0d(minimal):
|
|
37
|
+
i = 273409
|
|
38
|
+
minimal.int_0d = i
|
|
39
|
+
expected = xxh3_64_digest(struct.pack("<i", i))
|
|
40
|
+
assert expected == b"D\x1an\x8b\xbe\x99\x9a\t"
|
|
41
|
+
assert imas.util.calc_hash(minimal.int_0d) == expected
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_hash_flt0d(minimal):
|
|
45
|
+
f = 3.141592
|
|
46
|
+
minimal.flt_0d = f
|
|
47
|
+
expected = xxh3_64_digest(struct.pack("<d", f))
|
|
48
|
+
assert imas.util.calc_hash(minimal.flt_0d) == expected
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_hash_cpx0d(minimal):
|
|
52
|
+
c = 3.141592 - 2.718281j
|
|
53
|
+
minimal.cpx_0d = c
|
|
54
|
+
expected = xxh3_64_digest(struct.pack("<dd", c.real, c.imag))
|
|
55
|
+
assert expected == b"\x18`\xcek\x82\xa0\x18\x0e"
|
|
56
|
+
assert imas.util.calc_hash(minimal.cpx_0d) == expected
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.mark.parametrize("n", [1, 2, 3])
|
|
60
|
+
def test_hash_int_nd(minimal, n):
|
|
61
|
+
name = f"int_{n}d"
|
|
62
|
+
arr = np.fromfunction(
|
|
63
|
+
lambda i, j=0, k=0: (i << 16) + (j << 8) + k, (5,) * n, dtype=int
|
|
64
|
+
)
|
|
65
|
+
minimal[name] = arr
|
|
66
|
+
flattened = np.ravel(arr, order="F")
|
|
67
|
+
expected = xxh3_64_digest(
|
|
68
|
+
struct.pack("<B", n)
|
|
69
|
+
+ (struct.pack("<Q", 5) * n)
|
|
70
|
+
+ b"".join(struct.pack("<i", i) for i in flattened)
|
|
71
|
+
)
|
|
72
|
+
assert imas.util.calc_hash(minimal[name]) == expected
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@pytest.mark.parametrize("n", [1, 2, 3, 4, 5, 6])
|
|
76
|
+
def test_hash_flt_nd(minimal, n):
|
|
77
|
+
name = f"flt_{n}d"
|
|
78
|
+
arr = np.random.random((5,) * n)
|
|
79
|
+
minimal[name] = arr
|
|
80
|
+
flattened = np.ravel(arr, order="F")
|
|
81
|
+
expected = xxh3_64_digest(
|
|
82
|
+
struct.pack("<B", n)
|
|
83
|
+
+ (struct.pack("<Q", 5) * n)
|
|
84
|
+
+ b"".join(struct.pack("<d", f) for f in flattened)
|
|
85
|
+
)
|
|
86
|
+
assert imas.util.calc_hash(minimal[name]) == expected
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@pytest.mark.parametrize("n", [1, 2, 3, 4, 5, 6])
|
|
90
|
+
def test_hash_cpx_nd(minimal, n):
|
|
91
|
+
name = f"cpx_{n}d"
|
|
92
|
+
arr = np.random.random((5,) * n) + np.random.random((5,) * n) * 1j
|
|
93
|
+
minimal[name] = arr
|
|
94
|
+
flattened = np.ravel(arr, order="F")
|
|
95
|
+
expected = xxh3_64_digest(
|
|
96
|
+
struct.pack("<B", n)
|
|
97
|
+
+ (struct.pack("<Q", 5) * n)
|
|
98
|
+
+ b"".join(struct.pack("<dd", c.real, c.imag) for c in flattened)
|
|
99
|
+
)
|
|
100
|
+
assert imas.util.calc_hash(minimal[name]) == expected
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_hash_aos():
|
|
104
|
+
cp = imas.IDSFactory("3.39.0").core_profiles()
|
|
105
|
+
assert imas.util.calc_hash(cp.profiles_1d) == xxh3_64_digest(b"\0\0\0\0\0\0\0\0")
|
|
106
|
+
cp.profiles_1d.resize(3)
|
|
107
|
+
# Check that the empty struct hashes are as expected
|
|
108
|
+
for p1d in cp.profiles_1d:
|
|
109
|
+
assert imas.util.calc_hash(p1d) == xxh3_64_digest(b"")
|
|
110
|
+
assert imas.util.calc_hash(cp.profiles_1d) == xxh3_64_digest(
|
|
111
|
+
b"\x03\0\0\0\0\0\0\0" + 3 * xxh3_64_digest(b"")
|
|
112
|
+
)
|
|
113
|
+
# Set some data on the structures
|
|
114
|
+
for p1d in cp.profiles_1d:
|
|
115
|
+
p1d.time = 1.0
|
|
116
|
+
struct_hash = imas.util.calc_hash(cp.profiles_1d[0])
|
|
117
|
+
assert struct_hash == imas.util.calc_hash(cp.profiles_1d[1])
|
|
118
|
+
assert struct_hash == imas.util.calc_hash(cp.profiles_1d[2])
|
|
119
|
+
assert imas.util.calc_hash(cp.profiles_1d) == xxh3_64_digest(
|
|
120
|
+
b"\x03\0\0\0\0\0\0\0" + 3 * struct_hash
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_hash_struct():
|
|
125
|
+
cp = imas.IDSFactory("3.39.0").core_profiles()
|
|
126
|
+
assert imas.util.calc_hash(cp.code) == xxh3_64_digest(b"")
|
|
127
|
+
|
|
128
|
+
cp.code.name = "name"
|
|
129
|
+
expected = xxh3_64_digest(b"name" + xxh3_64_digest(b"name"))
|
|
130
|
+
assert imas.util.calc_hash(cp.code) == expected
|
|
131
|
+
|
|
132
|
+
cp.code.version = "version"
|
|
133
|
+
expected = xxh3_64_digest(
|
|
134
|
+
b"name" + xxh3_64_digest(b"name") + b"version" + xxh3_64_digest(b"version")
|
|
135
|
+
)
|
|
136
|
+
assert imas.util.calc_hash(cp.code) == expected
|
|
137
|
+
|
|
138
|
+
cp.code.parameters = "parameters"
|
|
139
|
+
expected = xxh3_64_digest(
|
|
140
|
+
b"name"
|
|
141
|
+
+ xxh3_64_digest(b"name")
|
|
142
|
+
+ b"parameters"
|
|
143
|
+
+ xxh3_64_digest(b"parameters")
|
|
144
|
+
+ b"version"
|
|
145
|
+
+ xxh3_64_digest(b"version")
|
|
146
|
+
)
|
|
147
|
+
assert imas.util.calc_hash(cp.code) == expected
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_hash_ids_properties():
|
|
151
|
+
cp = imas.IDSFactory("3.39.0").core_profiles()
|
|
152
|
+
assert imas.util.calc_hash(cp.ids_properties) == xxh3_64_digest(b"")
|
|
153
|
+
|
|
154
|
+
cp.ids_properties.version_put.data_dictionary = "dd"
|
|
155
|
+
cp.ids_properties.version_put.access_layer = "al"
|
|
156
|
+
cp.ids_properties.version_put.access_layer_language = "lang"
|
|
157
|
+
# This should be ignored for calculating the hash:
|
|
158
|
+
assert imas.util.calc_hash(cp.ids_properties) == xxh3_64_digest(b"")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_hash_ids():
|
|
162
|
+
cp = imas.IDSFactory().core_profiles()
|
|
163
|
+
cp.ids_properties.homogeneous_time = 1
|
|
164
|
+
cp.ids_properties.comment = "Testing hash function"
|
|
165
|
+
cp.code.name = "imas-python"
|
|
166
|
+
cp.time = [1.0, 2.0, 3.0, 4.0]
|
|
167
|
+
cp.profiles_1d.resize(4)
|
|
168
|
+
for p1d in cp.profiles_1d:
|
|
169
|
+
p1d.grid.rho_tor_norm = [1.0, 2.0]
|
|
170
|
+
p1d.electrons.temperature = [1e6, 2e6]
|
|
171
|
+
assert imas.util.calc_hash(cp.ids_properties) == b"3Fw\xab:w7K"
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def test_different_hashes():
|
|
175
|
+
eq = imas.IDSFactory().equilibrium()
|
|
176
|
+
eq.time_slice.resize(1)
|
|
177
|
+
eq.time_slice[0].profiles_2d.resize(1)
|
|
178
|
+
p2d = eq.time_slice[0].profiles_2d[0]
|
|
179
|
+
# Test that [[1.0, 2.0]] and [[1.0], [2.0]] get different hashes, while they have
|
|
180
|
+
# the same underlying data:
|
|
181
|
+
p2d.r = [[1.0, 2.0]]
|
|
182
|
+
bytes1 = p2d.r.tobytes(order="F")
|
|
183
|
+
hash1 = imas.util.calc_hash(p2d.r)
|
|
184
|
+
p2d.r = [[1.0], [2.0]]
|
|
185
|
+
bytes2 = p2d.r.tobytes(order="F")
|
|
186
|
+
hash2 = imas.util.calc_hash(p2d.r)
|
|
187
|
+
assert bytes1 == bytes2
|
|
188
|
+
assert hash1 != hash2
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import random
|
|
3
|
+
import string
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pytest
|
|
7
|
+
from packaging.version import Version
|
|
8
|
+
|
|
9
|
+
from imas.db_entry import DBEntry
|
|
10
|
+
from imas.ids_data_type import IDSDataType
|
|
11
|
+
from imas.ids_defs import (
|
|
12
|
+
ASCII_BACKEND,
|
|
13
|
+
IDS_TIME_MODE_HETEROGENEOUS,
|
|
14
|
+
IDS_TIME_MODE_HOMOGENEOUS,
|
|
15
|
+
IDS_TIME_MODE_INDEPENDENT,
|
|
16
|
+
)
|
|
17
|
+
from imas.ids_metadata import IDSType
|
|
18
|
+
from imas.ids_primitive import IDSPrimitive, IDSString1D
|
|
19
|
+
from imas.ids_struct_array import IDSStructArray
|
|
20
|
+
from imas.ids_structure import IDSStructure
|
|
21
|
+
from imas.ids_toplevel import IDSToplevel
|
|
22
|
+
from imas.util import idsdiffgen, visit_children
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
BASE_STRING = string.ascii_uppercase + string.digits
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def randdims(ndims):
|
|
30
|
+
"""Return a list of n random numbers representing
|
|
31
|
+
the shapes in n dimensions"""
|
|
32
|
+
return random.sample(range(1, 7), ndims)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def random_string():
|
|
36
|
+
return "".join(random.choices(BASE_STRING, k=random.randint(0, 128)))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def random_data(ids_type, ndims):
|
|
40
|
+
if ids_type is IDSDataType.STR:
|
|
41
|
+
if ndims == 0:
|
|
42
|
+
return random_string()
|
|
43
|
+
elif ndims == 1:
|
|
44
|
+
return [random_string() for _ in range(random.randint(0, 3))]
|
|
45
|
+
else:
|
|
46
|
+
raise NotImplementedError(
|
|
47
|
+
"Strings of dimension 2 or higher are not supported"
|
|
48
|
+
)
|
|
49
|
+
elif ids_type is IDSDataType.INT:
|
|
50
|
+
return np.random.randint(0, 2**31 - 1, size=randdims(ndims), dtype=np.int32)
|
|
51
|
+
elif ids_type is IDSDataType.FLT:
|
|
52
|
+
return np.random.random_sample(size=randdims(ndims))
|
|
53
|
+
elif ids_type is IDSDataType.CPX:
|
|
54
|
+
size = randdims(ndims)
|
|
55
|
+
return np.random.random_sample(size) + 1j * np.random.random_sample(size)
|
|
56
|
+
else:
|
|
57
|
+
raise ValueError("Unknown data type %s requested to fill", ids_type)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def fill_with_random_data(structure, max_children=3):
|
|
61
|
+
"""Fill a structure with random data.
|
|
62
|
+
|
|
63
|
+
Sets homogeneous_time to homogeneous _always_.
|
|
64
|
+
TODO: also test other time types
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
structure: IDS object to fill
|
|
68
|
+
max_children: The maximum amount of children to create for IDSStructArrays.
|
|
69
|
+
"""
|
|
70
|
+
is_toplevel = isinstance(structure, IDSToplevel)
|
|
71
|
+
for child_name in structure._children:
|
|
72
|
+
if not is_toplevel and child_name == "time":
|
|
73
|
+
continue # skip non-root time arrays when in HOMOGENEOUS_TIME
|
|
74
|
+
child = structure[child_name]
|
|
75
|
+
|
|
76
|
+
if isinstance(child, IDSStructure):
|
|
77
|
+
fill_with_random_data(child, max_children)
|
|
78
|
+
elif isinstance(child, IDSStructArray):
|
|
79
|
+
n_children = min(child.metadata.maxoccur or max_children, max_children)
|
|
80
|
+
child.resize(n_children)
|
|
81
|
+
# choose which child will get the max number of grand-children
|
|
82
|
+
max_child = random.randrange(n_children)
|
|
83
|
+
for i, ch in enumerate(child.value):
|
|
84
|
+
max_grand_children = max_children if i == max_child else 1
|
|
85
|
+
fill_with_random_data(ch, max_grand_children)
|
|
86
|
+
# Delete empty structures at the back
|
|
87
|
+
while len(child) > 0 and not child[-1].has_value:
|
|
88
|
+
child.resize(len(child) - 1, keep=True)
|
|
89
|
+
else: # leaf node
|
|
90
|
+
if child_name == "homogeneous_time":
|
|
91
|
+
child.value = IDS_TIME_MODE_HOMOGENEOUS
|
|
92
|
+
else:
|
|
93
|
+
child.value = random_data(child.metadata.data_type, child.metadata.ndim)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def maybe_set_random_value(primitive: IDSPrimitive, leave_empty: float) -> None:
|
|
97
|
+
"""Set the value of an IDS primitive with a certain chance.
|
|
98
|
+
|
|
99
|
+
If the IDSPrimitive has coordinates, then the size of the coordinates is taken into
|
|
100
|
+
account as well.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
primitive: IDSPrimitive to set the value of
|
|
104
|
+
leave_empty: Chance that this primitive remains empty.
|
|
105
|
+
"""
|
|
106
|
+
# Skip obsolescent nodes
|
|
107
|
+
if getattr(primitive.metadata, "lifecycle_status", None) == "obsolescent":
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
if random.random() < leave_empty:
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
ndim = primitive.metadata.ndim
|
|
114
|
+
if ndim == 0:
|
|
115
|
+
primitive.value = random_data(primitive.metadata.data_type, ndim)
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
shape = []
|
|
119
|
+
for dim, coordinate in enumerate(primitive.metadata.coordinates):
|
|
120
|
+
same_as = primitive.metadata.coordinates_same_as[dim]
|
|
121
|
+
if not coordinate.has_validation and not same_as.has_validation:
|
|
122
|
+
if primitive.metadata.name.endswith("_error_upper"):
|
|
123
|
+
# <name>_error_upper should only be filled when <name> is
|
|
124
|
+
name = primitive.metadata.name[: -len("_error_upper")]
|
|
125
|
+
data = primitive._parent[name]
|
|
126
|
+
if not data.has_value:
|
|
127
|
+
return
|
|
128
|
+
size = data.shape[dim]
|
|
129
|
+
elif primitive.metadata.name.endswith("_error_lower"):
|
|
130
|
+
# <name>_error_lower should only be filled when <name>_error_upper is
|
|
131
|
+
name = primitive.metadata.name[: -len("_error_lower")] + "_error_upper"
|
|
132
|
+
data = primitive._parent[name]
|
|
133
|
+
if not data.has_value:
|
|
134
|
+
return
|
|
135
|
+
size = data.shape[dim]
|
|
136
|
+
else:
|
|
137
|
+
# we can independently choose a size for this dimension:
|
|
138
|
+
size = random.randint(1, 6)
|
|
139
|
+
elif coordinate.references or same_as.references:
|
|
140
|
+
try:
|
|
141
|
+
if coordinate.references:
|
|
142
|
+
refs = [ref.goto(primitive) for ref in coordinate.references]
|
|
143
|
+
filled_refs = [ref for ref in refs if len(ref) > 0]
|
|
144
|
+
assert len(filled_refs) in (0, 1)
|
|
145
|
+
coordinate_element = filled_refs[0] if filled_refs else refs[0]
|
|
146
|
+
else:
|
|
147
|
+
coordinate_element = same_as.references[0].goto(primitive)
|
|
148
|
+
except (ValueError, AttributeError):
|
|
149
|
+
# Ignore invalid coordinate specs
|
|
150
|
+
coordinate_element = np.ones((1,) * 6)
|
|
151
|
+
|
|
152
|
+
if len(coordinate_element) == 0:
|
|
153
|
+
# Scale chance of not setting a coordinate by our number of dimensions,
|
|
154
|
+
# such that overall there is roughly a 50% chance that any coordinate
|
|
155
|
+
# remains empty
|
|
156
|
+
maybe_set_random_value(coordinate_element, 0.5**ndim)
|
|
157
|
+
size = coordinate_element.shape[0 if coordinate.references else dim]
|
|
158
|
+
|
|
159
|
+
if coordinate.size: # coordinateX = <path> OR 1...1
|
|
160
|
+
# Coin flip whether to use the size as determined by
|
|
161
|
+
# coordinate.references, or the size from coordinate.size
|
|
162
|
+
if random.random() < 0.5:
|
|
163
|
+
size = coordinate.size
|
|
164
|
+
else:
|
|
165
|
+
size = coordinate.size
|
|
166
|
+
if size == 0:
|
|
167
|
+
return # Leave empty
|
|
168
|
+
shape.append(size)
|
|
169
|
+
|
|
170
|
+
if primitive.metadata.data_type is IDSDataType.STR:
|
|
171
|
+
primitive.value = [random_string() for i in range(shape[0])]
|
|
172
|
+
elif primitive.metadata.data_type is IDSDataType.INT:
|
|
173
|
+
primitive.value = np.random.randint(
|
|
174
|
+
-(2**31), 2**31 - 1, size=shape, dtype=np.int32
|
|
175
|
+
)
|
|
176
|
+
elif primitive.metadata.data_type is IDSDataType.FLT:
|
|
177
|
+
primitive.value = np.random.random_sample(size=shape)
|
|
178
|
+
elif primitive.metadata.data_type is IDSDataType.CPX:
|
|
179
|
+
val = np.random.random_sample(shape) + 1j * np.random.random_sample(shape)
|
|
180
|
+
primitive.value = val
|
|
181
|
+
else:
|
|
182
|
+
raise ValueError(f"Invalid IDS data type: {primitive.metadata.data_type}")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2):
|
|
186
|
+
"""Fill a structure with random data, such that coordinate sizes are consistent.
|
|
187
|
+
|
|
188
|
+
Sets homogeneous_time to heterogeneous (always).
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
structure: IDSStructure object to (recursively fill)
|
|
192
|
+
leave_empty: factor (0-1) of nodes to leave empty
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
Nothing: if the provided IDSStructue is an IDSToplevel
|
|
196
|
+
exclusive_coordinates: list of IDSPrimitives that have exclusive alternative
|
|
197
|
+
coordinates. These are initially not filled, and only at the very end of
|
|
198
|
+
filling an IDSToplevel, a choice is made between the exclusive coordinates.
|
|
199
|
+
"""
|
|
200
|
+
if isinstance(structure, IDSToplevel):
|
|
201
|
+
unsupported_ids_name = (
|
|
202
|
+
"amns_data"
|
|
203
|
+
if Version(structure._version) < Version("3.42.0")
|
|
204
|
+
else "thomson_scattering"
|
|
205
|
+
)
|
|
206
|
+
if structure.metadata.name == unsupported_ids_name:
|
|
207
|
+
pytest.skip(
|
|
208
|
+
f"fill_consistent doesn't support IDS {structure.metadata.name} "
|
|
209
|
+
f"for Data Dictionary version {structure._version}."
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
time_mode = IDS_TIME_MODE_HETEROGENEOUS
|
|
213
|
+
if structure.metadata.type is IDSType.CONSTANT:
|
|
214
|
+
time_mode = IDS_TIME_MODE_INDEPENDENT
|
|
215
|
+
structure.ids_properties.homogeneous_time = time_mode
|
|
216
|
+
|
|
217
|
+
exclusive_coordinates = []
|
|
218
|
+
|
|
219
|
+
for child in structure:
|
|
220
|
+
if isinstance(child, IDSStructure):
|
|
221
|
+
exclusive_coordinates.extend(fill_consistent(child, leave_empty))
|
|
222
|
+
|
|
223
|
+
elif isinstance(child, IDSStructArray):
|
|
224
|
+
if child.metadata.coordinates[0].references:
|
|
225
|
+
try:
|
|
226
|
+
coor = child.coordinates[0]
|
|
227
|
+
except RuntimeError: # Ignore failed coordinate retrieval
|
|
228
|
+
coor = []
|
|
229
|
+
if len(coor) == 0:
|
|
230
|
+
if isinstance(coor, IDSPrimitive):
|
|
231
|
+
# maybe fill with random data:
|
|
232
|
+
try:
|
|
233
|
+
maybe_set_random_value(coor, leave_empty)
|
|
234
|
+
except (RuntimeError, ValueError):
|
|
235
|
+
pass
|
|
236
|
+
child.resize(len(coor))
|
|
237
|
+
else: # a numpy array is returned, resize to coordinate size or 1
|
|
238
|
+
child.resize(child.metadata.coordinates[0].size or 1)
|
|
239
|
+
if child.metadata.type.is_dynamic:
|
|
240
|
+
# This is a dynamic AoS with time coordinate inside: we must
|
|
241
|
+
# set the time coordinate to something else than EMPTY_FLOAT
|
|
242
|
+
# to pass validation:
|
|
243
|
+
child[0].time = 0.0
|
|
244
|
+
else:
|
|
245
|
+
child.resize(child.metadata.coordinates[0].size or 1)
|
|
246
|
+
for ele in child:
|
|
247
|
+
exclusive_coordinates.extend(fill_consistent(ele, leave_empty))
|
|
248
|
+
|
|
249
|
+
else: # IDSPrimitive
|
|
250
|
+
coordinates = child.metadata.coordinates
|
|
251
|
+
if child.metadata.path_string == "ids_properties/homogeneous_time":
|
|
252
|
+
pass # We already set homogeneous_time
|
|
253
|
+
elif child.has_value:
|
|
254
|
+
pass # Already encountered somewhere
|
|
255
|
+
elif any(len(coordinate.references) > 1 for coordinate in coordinates):
|
|
256
|
+
exclusive_coordinates.append(child)
|
|
257
|
+
else:
|
|
258
|
+
try:
|
|
259
|
+
maybe_set_random_value(child, leave_empty)
|
|
260
|
+
except (RuntimeError, ValueError):
|
|
261
|
+
pass
|
|
262
|
+
|
|
263
|
+
if isinstance(structure, IDSToplevel):
|
|
264
|
+
# handle exclusive_coordinates
|
|
265
|
+
for element in exclusive_coordinates:
|
|
266
|
+
for dim, coordinate in enumerate(element.metadata.coordinates):
|
|
267
|
+
try:
|
|
268
|
+
refs = [ref.goto(element) for ref in coordinate.references]
|
|
269
|
+
except RuntimeError:
|
|
270
|
+
break # Ignore paths that cannot be resolved
|
|
271
|
+
filled_refs = [ref for ref in refs if len(ref) > 0]
|
|
272
|
+
if len(filled_refs) == 0:
|
|
273
|
+
continue
|
|
274
|
+
|
|
275
|
+
# Unset conflicting coordinates
|
|
276
|
+
while len(filled_refs) > 1:
|
|
277
|
+
random.shuffle(filled_refs)
|
|
278
|
+
coor = filled_refs.pop()
|
|
279
|
+
unset_coordinate(coor)
|
|
280
|
+
|
|
281
|
+
maybe_set_random_value(element, leave_empty)
|
|
282
|
+
else:
|
|
283
|
+
return exclusive_coordinates
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def unset_coordinate(coordinate):
|
|
287
|
+
# Unset the coordinate quantity
|
|
288
|
+
coordinate.value = []
|
|
289
|
+
# Find all elements that also have this as a coordinate and unset...
|
|
290
|
+
parent = coordinate._dd_parent
|
|
291
|
+
while parent.metadata.data_type is not IDSDataType.STRUCT_ARRAY:
|
|
292
|
+
parent = parent._dd_parent
|
|
293
|
+
|
|
294
|
+
def callback(element):
|
|
295
|
+
if hasattr(element, "coordinates") and element.has_value:
|
|
296
|
+
for ele_coor in element.coordinates:
|
|
297
|
+
if ele_coor is coordinate:
|
|
298
|
+
element.value = []
|
|
299
|
+
return
|
|
300
|
+
|
|
301
|
+
visit_children(callback, parent)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def compare_children(st1, st2, deleted_paths=set(), accept_lazy=False):
|
|
305
|
+
"""Perform a deep compare of two structures using asserts.
|
|
306
|
+
|
|
307
|
+
All paths in ``deleted_paths`` are asserted that they are deleted in st2.
|
|
308
|
+
"""
|
|
309
|
+
for description, node1, node2 in idsdiffgen(st1, st2, accept_lazy=accept_lazy):
|
|
310
|
+
if node1 is not None:
|
|
311
|
+
# Check if this is a deleted path
|
|
312
|
+
path = node1.metadata.path_string
|
|
313
|
+
# No duplicate entries for _error_upper, _error_lower and _error_index
|
|
314
|
+
path = path.partition("_error_")[0]
|
|
315
|
+
if path in deleted_paths:
|
|
316
|
+
assert node2 is None
|
|
317
|
+
continue
|
|
318
|
+
# Workaround for https://jira.iter.org/browse/IMAS-4948:
|
|
319
|
+
if isinstance(node1, IDSString1D) or isinstance(node2, IDSString1D):
|
|
320
|
+
assert (
|
|
321
|
+
node1 == node2
|
|
322
|
+
or (node1 == [""] and not node2)
|
|
323
|
+
or (node2 == [""] and not node1)
|
|
324
|
+
)
|
|
325
|
+
continue
|
|
326
|
+
# Not a deleted path
|
|
327
|
+
assert False, ("Unequal nodes", node1, node2)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def open_dbentry(
|
|
331
|
+
backend, mode, worker_id, tmp_path, dd_version=None, xml_path=None
|
|
332
|
+
) -> DBEntry:
|
|
333
|
+
"""Open a DBEntry, with a tmpdir in place of the user argument"""
|
|
334
|
+
if worker_id == "master":
|
|
335
|
+
pulse = 1
|
|
336
|
+
else:
|
|
337
|
+
pulse = int(worker_id[2:]) + 1
|
|
338
|
+
|
|
339
|
+
dbentry = DBEntry(
|
|
340
|
+
backend,
|
|
341
|
+
"test",
|
|
342
|
+
pulse,
|
|
343
|
+
0,
|
|
344
|
+
str(tmp_path),
|
|
345
|
+
dd_version=dd_version,
|
|
346
|
+
xml_path=xml_path,
|
|
347
|
+
)
|
|
348
|
+
options = f"-prefix {tmp_path}/" if backend == ASCII_BACKEND else None
|
|
349
|
+
if mode == "w":
|
|
350
|
+
dbentry.create(options=options)
|
|
351
|
+
else:
|
|
352
|
+
dbentry.open(options=options)
|
|
353
|
+
|
|
354
|
+
return dbentry
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from imas.dd_zip import dd_identifiers
|
|
4
|
+
from imas.ids_factory import IDSFactory
|
|
5
|
+
from imas.ids_identifiers import IDSIdentifier, identifiers
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_list_identifiers():
|
|
9
|
+
assert identifiers.identifiers == dd_identifiers()
|
|
10
|
+
# Check a known identifier, which we'll also use in more tests
|
|
11
|
+
assert "core_source_identifier" in identifiers.identifiers
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_identifier_enum():
|
|
15
|
+
csid = identifiers.core_source_identifier
|
|
16
|
+
# Test item access
|
|
17
|
+
assert csid is identifiers["core_source_identifier"]
|
|
18
|
+
|
|
19
|
+
# Class and inheritance tests
|
|
20
|
+
assert csid.__name__ == "core_source_identifier"
|
|
21
|
+
assert csid.__qualname__ == "imas.ids_identifiers.core_source_identifier"
|
|
22
|
+
assert issubclass(csid, IDSIdentifier)
|
|
23
|
+
assert isinstance(csid.total, csid)
|
|
24
|
+
assert isinstance(csid.total, IDSIdentifier)
|
|
25
|
+
|
|
26
|
+
# Check access methods
|
|
27
|
+
assert csid.total is csid(1)
|
|
28
|
+
assert csid.total is csid["total"]
|
|
29
|
+
|
|
30
|
+
# Check attributes
|
|
31
|
+
assert csid.total.name == "total"
|
|
32
|
+
assert csid.total.index == csid.total.value == 1
|
|
33
|
+
assert isinstance(csid.total.description, str)
|
|
34
|
+
assert csid.total.description != ""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_identifier_struct_assignment(caplog):
|
|
38
|
+
csid = identifiers.core_source_identifier
|
|
39
|
+
cs = IDSFactory("3.39.0").core_sources()
|
|
40
|
+
cs.source.resize(3)
|
|
41
|
+
assert cs.source[0].identifier.metadata.identifier_enum is csid
|
|
42
|
+
# Test assignment options: identifier instance, index and name
|
|
43
|
+
cs.source[0].identifier = csid.total
|
|
44
|
+
cs.source[1].identifier = "total"
|
|
45
|
+
cs.source[2].identifier = 1
|
|
46
|
+
for source in cs.source:
|
|
47
|
+
assert source.identifier.name == "total"
|
|
48
|
+
assert source.identifier.index == 1
|
|
49
|
+
assert source.identifier.description == csid.total.description
|
|
50
|
+
# Test equality of identifier structure and enum:
|
|
51
|
+
assert source.identifier == csid.total
|
|
52
|
+
assert source.identifier != csid(0)
|
|
53
|
+
# Test fuzzy equality
|
|
54
|
+
caplog.clear()
|
|
55
|
+
# Empty description is okay
|
|
56
|
+
source.identifier.description = ""
|
|
57
|
+
assert source.identifier == csid.total
|
|
58
|
+
assert not caplog.records
|
|
59
|
+
# Incorrect description logs a warning
|
|
60
|
+
source.identifier.description = "XYZ"
|
|
61
|
+
assert source.identifier == csid.total
|
|
62
|
+
assert len(caplog.records) == 1
|
|
63
|
+
assert caplog.records[0].levelname == "WARNING"
|
|
64
|
+
source.identifier.description = ""
|
|
65
|
+
# Empty name is okay
|
|
66
|
+
source.identifier.name = ""
|
|
67
|
+
assert source.identifier == csid.total
|
|
68
|
+
# But non-matching name is not okay
|
|
69
|
+
source.identifier.name = "XYZ"
|
|
70
|
+
assert source.identifier != csid.total
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_identifier_aos_assignment():
|
|
74
|
+
cfid = identifiers.pf_active_coil_function_identifier
|
|
75
|
+
pfa = IDSFactory("3.39.0").pf_active()
|
|
76
|
+
pfa.coil.resize(1)
|
|
77
|
+
pfa.coil[0].function.resize(3)
|
|
78
|
+
assert pfa.coil[0].function.metadata.identifier_enum is cfid
|
|
79
|
+
# Test assignment options: identifier instance, index and name
|
|
80
|
+
pfa.coil[0].function[0] = cfid.flux
|
|
81
|
+
pfa.coil[0].function[1] = "flux"
|
|
82
|
+
pfa.coil[0].function[2] = 0
|
|
83
|
+
for function in pfa.coil[0].function:
|
|
84
|
+
assert function.name == "flux"
|
|
85
|
+
assert function.index == 0
|
|
86
|
+
assert function.description == cfid.flux.description
|
|
87
|
+
# Test equality of identifier structure and enum:
|
|
88
|
+
assert function == cfid.flux
|
|
89
|
+
assert function != cfid.b_field_shaping
|
|
90
|
+
assert pfa.coil[0].function[0] == cfid.flux
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_invalid_identifier_assignment():
|
|
94
|
+
cfid = identifiers.pf_active_coil_function_identifier
|
|
95
|
+
cs = IDSFactory("3.39.0").core_sources()
|
|
96
|
+
cs.source.resize(1)
|
|
97
|
+
|
|
98
|
+
with pytest.raises(TypeError):
|
|
99
|
+
# Incorrect identifier type
|
|
100
|
+
cs.source[0].identifier = cfid.flux
|
|
101
|
+
with pytest.raises(ValueError):
|
|
102
|
+
cs.source[0].identifier = "identifier names never contain spaces"
|
|
103
|
+
with pytest.raises(ValueError):
|
|
104
|
+
# negative identifiers are reserved for user-defined identifiers
|
|
105
|
+
cs.source[0].identifier = -1
|