imas-python 0.1.dev1618__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imas-python might be problematic. Click here for more details.
- imas/__init__.py +34 -0
- imas/__main__.py +17 -0
- imas/_util.py +260 -0
- imas/_version.py +16 -0
- imas/assets/IDSDef2MDSpreTree.xsl +161 -0
- imas/assets/IDS_fake_toplevel.xml +318 -0
- imas/assets/IDS_minimal.xml +37 -0
- imas/assets/IDS_minimal_2.xml +37 -0
- imas/assets/IDS_minimal_struct_array.xml +43 -0
- imas/assets/IDS_minimal_types.xml +169 -0
- imas/assets/ITER_134173_106_core_profiles.ids +2438 -0
- imas/assets/ITER_134173_106_equilibrium.ids +3172 -0
- imas/assets/README.md +5 -0
- imas/assets/core_profiles.ids +2438 -0
- imas/assets/equilibrium.ids +3172 -0
- imas/backends/__init__.py +12 -0
- imas/backends/db_entry_impl.py +122 -0
- imas/backends/imas_core/__init__.py +4 -0
- imas/backends/imas_core/al_context.py +391 -0
- imas/backends/imas_core/db_entry_al.py +405 -0
- imas/backends/imas_core/db_entry_helpers.py +200 -0
- imas/backends/imas_core/imas_interface.py +216 -0
- imas/backends/imas_core/mdsplus_model.py +367 -0
- imas/backends/imas_core/uda_support.py +56 -0
- imas/backends/netcdf/__init__.py +4 -0
- imas/backends/netcdf/db_entry_nc.py +176 -0
- imas/backends/netcdf/ids2nc.py +306 -0
- imas/backends/netcdf/nc2ids.py +302 -0
- imas/backends/netcdf/nc_metadata.py +357 -0
- imas/backends/netcdf/nc_validate.py +53 -0
- imas/command/cli.py +239 -0
- imas/command/db_analysis.py +290 -0
- imas/command/helpers.py +45 -0
- imas/command/timer.py +77 -0
- imas/db_entry.py +802 -0
- imas/dd_helpers.py +300 -0
- imas/dd_zip.py +297 -0
- imas/exception.py +107 -0
- imas/ids_base.py +149 -0
- imas/ids_convert.py +920 -0
- imas/ids_coordinates.py +400 -0
- imas/ids_data_type.py +115 -0
- imas/ids_defs.py +171 -0
- imas/ids_factory.py +108 -0
- imas/ids_identifiers.py +125 -0
- imas/ids_metadata.py +307 -0
- imas/ids_path.py +254 -0
- imas/ids_primitive.py +510 -0
- imas/ids_struct_array.py +233 -0
- imas/ids_structure.py +331 -0
- imas/ids_toplevel.py +381 -0
- imas/setup_logging.py +64 -0
- imas/test/test_all_dd_versions.py +28 -0
- imas/test/test_cli.py +44 -0
- imas/test/test_dbentry.py +104 -0
- imas/test/test_dd_helpers.py +134 -0
- imas/test/test_dd_zip.py +29 -0
- imas/test/test_deepcopy.py +25 -0
- imas/test/test_exception.py +12 -0
- imas/test/test_get_sample.py +388 -0
- imas/test/test_hash.py +188 -0
- imas/test/test_helpers.py +354 -0
- imas/test/test_identifiers.py +105 -0
- imas/test/test_ids2nc.py +166 -0
- imas/test/test_ids_ascii_data.py +39 -0
- imas/test/test_ids_convert.py +415 -0
- imas/test/test_ids_coordinates.py +165 -0
- imas/test/test_ids_data_type.py +40 -0
- imas/test/test_ids_factory.py +35 -0
- imas/test/test_ids_metadata.py +66 -0
- imas/test/test_ids_mixin.py +20 -0
- imas/test/test_ids_path.py +192 -0
- imas/test/test_ids_primitive.py +109 -0
- imas/test/test_ids_struct_array.py +89 -0
- imas/test/test_ids_structure.py +70 -0
- imas/test/test_ids_toplevel.py +55 -0
- imas/test/test_ids_validate.py +343 -0
- imas/test/test_latest_dd_autofill.py +74 -0
- imas/test/test_latest_dd_manual.py +38 -0
- imas/test/test_latest_dd_resample.py +102 -0
- imas/test/test_lazy_loading.py +201 -0
- imas/test/test_list_occurrences.py +82 -0
- imas/test/test_minimal.py +37 -0
- imas/test/test_minimal_conversion.py +44 -0
- imas/test/test_minimal_io.py +21 -0
- imas/test/test_minimal_struct_array_io.py +60 -0
- imas/test/test_minimal_types.py +436 -0
- imas/test/test_minimal_types_io.py +70 -0
- imas/test/test_minimal_types_io_automatic.py +66 -0
- imas/test/test_nbc_change.py +408 -0
- imas/test/test_nc_autofill.py +13 -0
- imas/test/test_nc_entry.py +28 -0
- imas/test/test_nc_metadata.py +153 -0
- imas/test/test_nc_validation.py +197 -0
- imas/test/test_snippets.py +25 -0
- imas/test/test_static_ids.py +39 -0
- imas/test/test_str_1d.py +99 -0
- imas/test/test_time_slicing.py +182 -0
- imas/test/test_uda_support.py +12 -0
- imas/test/test_util.py +209 -0
- imas/training.py +45 -0
- imas/util.py +526 -0
- imas_python-0.1.dev1618.dist-info/LICENSE.txt +165 -0
- imas_python-0.1.dev1618.dist-info/METADATA +288 -0
- imas_python-0.1.dev1618.dist-info/RECORD +108 -0
- imas_python-0.1.dev1618.dist-info/WHEEL +5 -0
- imas_python-0.1.dev1618.dist-info/entry_points.txt +3 -0
- imas_python-0.1.dev1618.dist-info/top_level.txt +1 -0
imas/ids_toplevel.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
# This file is part of imas-python.
|
|
2
|
+
# You should have received the imas-python LICENSE file with this project.
|
|
3
|
+
"""Represents a Top-level IDS (like ``core_profiles``, ``equilibrium``, etc)
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING, Optional
|
|
11
|
+
|
|
12
|
+
import numpy
|
|
13
|
+
|
|
14
|
+
import imas
|
|
15
|
+
from imas.backends.imas_core.imas_interface import ll_interface, lowlevel
|
|
16
|
+
from imas.exception import ValidationError
|
|
17
|
+
from imas.ids_base import IDSDoc
|
|
18
|
+
from imas.ids_defs import (
|
|
19
|
+
ASCII_BACKEND,
|
|
20
|
+
ASCII_SERIALIZER_PROTOCOL,
|
|
21
|
+
CHAR_DATA,
|
|
22
|
+
DEFAULT_SERIALIZER_PROTOCOL,
|
|
23
|
+
FLEXBUFFERS_SERIALIZER_PROTOCOL,
|
|
24
|
+
IDS_TIME_MODE_INDEPENDENT,
|
|
25
|
+
IDS_TIME_MODE_UNKNOWN,
|
|
26
|
+
IDS_TIME_MODES,
|
|
27
|
+
needs_imas,
|
|
28
|
+
)
|
|
29
|
+
from imas.ids_metadata import IDSMetadata, IDSType, get_toplevel_metadata
|
|
30
|
+
from imas.ids_structure import IDSStructure
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
from imas.db_entry import DBEntry
|
|
34
|
+
from imas.ids_factory import IDSFactory
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
_FLEXBUFFERS_URI = "imas:flexbuffers?path=/"
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _serializer_tmpdir() -> str:
|
|
42
|
+
env_tmpdir = os.getenv("IMAS_AL_SERIALIZER_TMP_DIR")
|
|
43
|
+
if env_tmpdir:
|
|
44
|
+
return env_tmpdir
|
|
45
|
+
return "/dev/shm" if os.path.exists("/dev/shm") else "."
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _create_serialization_dbentry(filepath: str, dd_version: str) -> "DBEntry":
|
|
49
|
+
"""Create a temporary DBEntry for use in the ASCII serialization protocol."""
|
|
50
|
+
if ll_interface._al_version.major == 4: # AL4 compatibility
|
|
51
|
+
dbentry = imas.DBEntry(
|
|
52
|
+
ASCII_BACKEND, "serialize", 1, 1, "serialize", dd_version=dd_version
|
|
53
|
+
)
|
|
54
|
+
dbentry.create(options=f"-fullpath {filepath}")
|
|
55
|
+
return dbentry
|
|
56
|
+
else: # AL5
|
|
57
|
+
path = Path(filepath)
|
|
58
|
+
return imas.DBEntry(
|
|
59
|
+
f"imas:ascii?path={path.parent};filename={path.name}",
|
|
60
|
+
"w",
|
|
61
|
+
dd_version=dd_version,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class IDSToplevel(IDSStructure):
|
|
66
|
+
"""This is any IDS Structure which has ids_properties as child node
|
|
67
|
+
|
|
68
|
+
At minimum, one should fill ids_properties/homogeneous_time
|
|
69
|
+
IF a quantity is filled, the coordinates of that quantity must be filled as well
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
__doc__ = IDSDoc(__doc__)
|
|
73
|
+
_path = "" # Path to ourselves without the IDS name and slashes
|
|
74
|
+
|
|
75
|
+
def __init__(self, parent: "IDSFactory", structure_xml, lazy=False):
|
|
76
|
+
"""Save backend_version and backend_xml and build translation layer.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
parent: Parent of ``self``.
|
|
80
|
+
structure_xml: XML structure that defines this IDS toplevel.
|
|
81
|
+
lazy: Whether this toplevel is used for a lazy-loaded get() or get_slice()
|
|
82
|
+
"""
|
|
83
|
+
self._lazy = lazy
|
|
84
|
+
# structure_xml might be an IDSMetadata already when initializing from __copy__
|
|
85
|
+
# or __deepcopy__
|
|
86
|
+
if isinstance(structure_xml, IDSMetadata):
|
|
87
|
+
metadata = structure_xml
|
|
88
|
+
else:
|
|
89
|
+
metadata = get_toplevel_metadata(structure_xml)
|
|
90
|
+
super().__init__(parent, metadata)
|
|
91
|
+
|
|
92
|
+
def __deepcopy__(self, memo):
|
|
93
|
+
copy = super().__deepcopy__(memo)
|
|
94
|
+
copy._lazy = self._lazy
|
|
95
|
+
return copy
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def _dd_version(self) -> str:
|
|
99
|
+
return self._version
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def _time_mode(self) -> int:
|
|
103
|
+
"""Retrieve the time mode from `/ids_properties/homogeneous_time`"""
|
|
104
|
+
return self.ids_properties.homogeneous_time
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def default_serializer_protocol():
|
|
108
|
+
"""Return the default serializer protocol."""
|
|
109
|
+
return DEFAULT_SERIALIZER_PROTOCOL
|
|
110
|
+
|
|
111
|
+
@needs_imas
|
|
112
|
+
def serialize(self, protocol=None) -> bytes:
|
|
113
|
+
"""Serialize this IDS to a data buffer.
|
|
114
|
+
|
|
115
|
+
The data buffer can be deserialized from any Access Layer High-Level Interface
|
|
116
|
+
that supports this.
|
|
117
|
+
|
|
118
|
+
Example:
|
|
119
|
+
|
|
120
|
+
.. code-block: python
|
|
121
|
+
|
|
122
|
+
core_profiles = imas.IDSFactory().core_profiles()
|
|
123
|
+
# fill core_profiles with data
|
|
124
|
+
...
|
|
125
|
+
|
|
126
|
+
data = core_profiles.serialize()
|
|
127
|
+
|
|
128
|
+
# For example, send `data` to another program with libmuscle.
|
|
129
|
+
# Then deserialize on the receiving side:
|
|
130
|
+
|
|
131
|
+
core_profiles = imas.IDSFactory().core_profiles()
|
|
132
|
+
core_profiles.deserialize(data)
|
|
133
|
+
# Use core_profiles:
|
|
134
|
+
...
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
protocol: Which serialization protocol to use. Uses
|
|
138
|
+
``DEFAULT_SERIALIZER_PROTOCOL`` when none specified. One of:
|
|
139
|
+
|
|
140
|
+
- :const:`~imas.ids_defs.ASCII_SERIALIZER_PROTOCOL`
|
|
141
|
+
- :const:`~imas.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL`
|
|
142
|
+
- :const:`~imas.ids_defs.DEFAULT_SERIALIZER_PROTOCOL`
|
|
143
|
+
|
|
144
|
+
The flexbuffers serializer protocol is only available when using
|
|
145
|
+
``imas_core >= 5.3``. It's the default protocol when it is available.
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Data buffer that can be deserialized using :meth:`deserialize`.
|
|
149
|
+
"""
|
|
150
|
+
if protocol is None:
|
|
151
|
+
protocol = self.default_serializer_protocol()
|
|
152
|
+
dd_version = self._dd_version
|
|
153
|
+
if self.ids_properties.homogeneous_time == IDS_TIME_MODE_UNKNOWN:
|
|
154
|
+
raise ValueError("IDS is found to be EMPTY (homogeneous_time undefined)")
|
|
155
|
+
if protocol == ASCII_SERIALIZER_PROTOCOL:
|
|
156
|
+
tmpdir = _serializer_tmpdir()
|
|
157
|
+
filepath = tempfile.mktemp(prefix="al_serialize_", dir=tmpdir)
|
|
158
|
+
dbentry = _create_serialization_dbentry(filepath, dd_version)
|
|
159
|
+
dbentry.put(self)
|
|
160
|
+
dbentry.close()
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
# read contents of tmpfile
|
|
164
|
+
with open(filepath, "rb") as f:
|
|
165
|
+
data = f.read()
|
|
166
|
+
finally:
|
|
167
|
+
os.unlink(filepath) # remove tmpfile from disk
|
|
168
|
+
return bytes([ASCII_SERIALIZER_PROTOCOL]) + data
|
|
169
|
+
if protocol == FLEXBUFFERS_SERIALIZER_PROTOCOL:
|
|
170
|
+
# Note: FLEXBUFFERS_SERIALIZER_PROTOCOL is None when imas_core doesn't
|
|
171
|
+
# support this format
|
|
172
|
+
with imas.DBEntry(_FLEXBUFFERS_URI, "w", dd_version=dd_version) as entry:
|
|
173
|
+
entry.put(self)
|
|
174
|
+
# Read serialized buffer
|
|
175
|
+
status, buffer = lowlevel.al_read_data_array(
|
|
176
|
+
entry._dbe_impl._db_ctx.ctx, "<buffer>", "", CHAR_DATA, 1
|
|
177
|
+
)
|
|
178
|
+
return bytes(buffer)
|
|
179
|
+
raise ValueError(f"Unrecognized serialization protocol: {protocol}")
|
|
180
|
+
|
|
181
|
+
@needs_imas
|
|
182
|
+
def deserialize(self, data: bytes) -> None:
|
|
183
|
+
"""Deserialize the data buffer into this IDS.
|
|
184
|
+
|
|
185
|
+
See :meth:`serialize` for an example.
|
|
186
|
+
|
|
187
|
+
Args:
|
|
188
|
+
data: binary data created by serializing an IDS.
|
|
189
|
+
"""
|
|
190
|
+
if len(data) <= 1:
|
|
191
|
+
raise ValueError("No data provided")
|
|
192
|
+
protocol = int(data[0]) # first byte of data contains serialization protocol
|
|
193
|
+
dd_version = self._dd_version
|
|
194
|
+
if protocol == ASCII_SERIALIZER_PROTOCOL:
|
|
195
|
+
tmpdir = _serializer_tmpdir()
|
|
196
|
+
filepath = tempfile.mktemp(prefix="al_serialize_", dir=tmpdir)
|
|
197
|
+
# write data into tmpfile
|
|
198
|
+
try:
|
|
199
|
+
with open(filepath, "wb") as f:
|
|
200
|
+
f.write(data[1:])
|
|
201
|
+
# Temporarily open an ASCII backend for deserialization from tmpfile
|
|
202
|
+
dbentry = _create_serialization_dbentry(filepath, dd_version)
|
|
203
|
+
dbentry.get(self.metadata.name, destination=self)
|
|
204
|
+
dbentry.close()
|
|
205
|
+
finally:
|
|
206
|
+
# tmpfile may not exist depending if an error occurs in above code
|
|
207
|
+
if os.path.exists(filepath):
|
|
208
|
+
os.unlink(filepath)
|
|
209
|
+
elif protocol == FLEXBUFFERS_SERIALIZER_PROTOCOL:
|
|
210
|
+
with imas.DBEntry(_FLEXBUFFERS_URI, "r", dd_version=dd_version) as entry:
|
|
211
|
+
# Write serialized buffer to the flexbuffers backend
|
|
212
|
+
buffer = numpy.frombuffer(data, dtype=numpy.int8)
|
|
213
|
+
lowlevel._al_write_data_array(
|
|
214
|
+
entry._dbe_impl._db_ctx.ctx, b"<buffer>", b"", buffer, CHAR_DATA
|
|
215
|
+
)
|
|
216
|
+
entry.get(self.metadata.name, destination=self)
|
|
217
|
+
elif protocol == 61: # Fallthrough
|
|
218
|
+
raise NotImplementedError(
|
|
219
|
+
"FLEXBUFFERS_SERIALIZER_PROTOCOL is not implemented by the installed "
|
|
220
|
+
"version of imas_core. Please update imas_core to 5.3 or newer."
|
|
221
|
+
)
|
|
222
|
+
else:
|
|
223
|
+
raise ValueError(f"Unrecognized serialization protocol: {protocol}")
|
|
224
|
+
|
|
225
|
+
def validate(self):
|
|
226
|
+
"""Validate the contents of this IDS.
|
|
227
|
+
|
|
228
|
+
The following sanity checks are executed on this IDS:
|
|
229
|
+
|
|
230
|
+
- The IDS must have a valid time mode (``ids_properties.homogeneous_time``)
|
|
231
|
+
- For all non-empty quantities with coordinates:
|
|
232
|
+
|
|
233
|
+
- If coordinates have an exact size (e.g. coordinate1 = 1...3), the size in
|
|
234
|
+
that dimension must match this.
|
|
235
|
+
- If coordinates refer to other elements (e.g. coordinate1 = time), the size
|
|
236
|
+
in that dimension must be the same as the size of the referred quantity.
|
|
237
|
+
|
|
238
|
+
Note that time is a special coordinate:
|
|
239
|
+
|
|
240
|
+
- When using homogeneous time, the time coordinate is the /time node.
|
|
241
|
+
- When using heterogeneous time, the time coordinate is the one specified
|
|
242
|
+
by the coordinate. For dynamic Array of Structures, the time element is
|
|
243
|
+
a FLT_0D inside the AoS (see ``profiles_1d`` in the core_profiles IDS).
|
|
244
|
+
In such cases the time element must be set.
|
|
245
|
+
- When using independent time mode, no time-dependent quantities may be
|
|
246
|
+
set.
|
|
247
|
+
|
|
248
|
+
- If a "same_as" coordinate is specified (e.g. coordinate2_same_as = r), the
|
|
249
|
+
size in that dimension must be the same as the size in that dimension of
|
|
250
|
+
the referred quantity.
|
|
251
|
+
|
|
252
|
+
If any check fails, a ValidationError is raised that describes the problem.
|
|
253
|
+
|
|
254
|
+
Example:
|
|
255
|
+
|
|
256
|
+
>>> core_profiles = imas.IDSFactory().core_profiles()
|
|
257
|
+
>>> core_profiles.validate() # Did not set homogeneous_time
|
|
258
|
+
[...]
|
|
259
|
+
imas.exception.ValidationError: Invalid value for ids_properties/homogeneous_time: IDSPrimitive("/core_profiles/ids_properties/homogeneous_time", -999999999)
|
|
260
|
+
>>> core_profiles.ids_properties.homogeneous_time = imas.ids_defs.IDS_TIME_MODE_HOMOGENEOUS
|
|
261
|
+
>>> core_profiles.validate() # No error: IDS is valid
|
|
262
|
+
>>> core_profiles.profiles_1d.resize(1)
|
|
263
|
+
>>> core_profiles.validate()
|
|
264
|
+
[...]
|
|
265
|
+
imas.exception.CoordinateError: Dimension 1 of element profiles_1d has incorrect size 1. Expected size is 0 (size of coordinate time).
|
|
266
|
+
>>> core_profiles.time = [1]
|
|
267
|
+
>>> core_profiles.validate() # No error: IDS is valid
|
|
268
|
+
|
|
269
|
+
""" # noqa: E501 (line too long)
|
|
270
|
+
if self._lazy:
|
|
271
|
+
logger.warning(
|
|
272
|
+
"Validating lazy loaded IDS '%s': this will not validate the full "
|
|
273
|
+
"IDS, only the loaded data is validated.",
|
|
274
|
+
self.metadata.name,
|
|
275
|
+
)
|
|
276
|
+
time_mode = self._time_mode
|
|
277
|
+
if time_mode not in IDS_TIME_MODES:
|
|
278
|
+
raise ValidationError(
|
|
279
|
+
f"Invalid value for ids_properties/homogeneous_time: {time_mode.value}"
|
|
280
|
+
)
|
|
281
|
+
if self.metadata.type is IDSType.CONSTANT: # IMAS-3330 static IDS
|
|
282
|
+
if time_mode != IDS_TIME_MODE_INDEPENDENT:
|
|
283
|
+
raise ValidationError(
|
|
284
|
+
f"Invalid value for ids_properties/homogeneous_time: {time_mode}. "
|
|
285
|
+
"The IDS is static, therefore homogeneous_time must be "
|
|
286
|
+
f"IDS_TIME_MODE_INDEPENDENT ({IDS_TIME_MODE_INDEPENDENT})."
|
|
287
|
+
)
|
|
288
|
+
try:
|
|
289
|
+
self._validate()
|
|
290
|
+
except ValidationError as exc:
|
|
291
|
+
# hide recursive stack trace from user
|
|
292
|
+
logger.debug("Original stack-trace of ValidationError: ", exc_info=1)
|
|
293
|
+
raise exc.with_traceback(None) from None
|
|
294
|
+
|
|
295
|
+
def _validate(self):
|
|
296
|
+
# Override to skip the self.metadata.type.is_dynamic check in IDSBase._validate
|
|
297
|
+
# accept_lazy=True: users are warned in IDSToplevel.validate()
|
|
298
|
+
for child in self.iter_nonempty_(accept_lazy=True):
|
|
299
|
+
child._validate()
|
|
300
|
+
|
|
301
|
+
@needs_imas
|
|
302
|
+
def get(self, occurrence: int = 0, db_entry: Optional["DBEntry"] = None) -> None:
|
|
303
|
+
"""Get data from AL backend storage format.
|
|
304
|
+
|
|
305
|
+
This method exists for API compatibility with the IMAS python HLI.
|
|
306
|
+
See :py:meth:`DBEntry.get <imas.db_entry.DBEntry.get>`.
|
|
307
|
+
"""
|
|
308
|
+
if db_entry is None:
|
|
309
|
+
raise NotImplementedError()
|
|
310
|
+
db_entry.get(self.metadata.name, occurrence, destination=self)
|
|
311
|
+
|
|
312
|
+
@needs_imas
|
|
313
|
+
def getSlice(
|
|
314
|
+
self,
|
|
315
|
+
time_requested: float,
|
|
316
|
+
interpolation_method: int,
|
|
317
|
+
occurrence: int = 0,
|
|
318
|
+
db_entry: Optional["DBEntry"] = None,
|
|
319
|
+
) -> None:
|
|
320
|
+
"""Get a slice from the backend.
|
|
321
|
+
|
|
322
|
+
This method exists for API compatibility with the IMAS python HLI.
|
|
323
|
+
See :py:meth:`DBEntry.get_slice <imas.db_entry.DBEntry.get_slice>`.
|
|
324
|
+
"""
|
|
325
|
+
if db_entry is None:
|
|
326
|
+
raise NotImplementedError()
|
|
327
|
+
db_entry.get_slice(
|
|
328
|
+
self.metadata.name,
|
|
329
|
+
time_requested,
|
|
330
|
+
interpolation_method,
|
|
331
|
+
occurrence,
|
|
332
|
+
destination=self,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
@needs_imas
|
|
336
|
+
def putSlice(
|
|
337
|
+
self, occurrence: int = 0, db_entry: Optional["DBEntry"] = None
|
|
338
|
+
) -> None:
|
|
339
|
+
"""Put a single slice into the backend.
|
|
340
|
+
|
|
341
|
+
This method exists for API compatibility with the IMAS python HLI.
|
|
342
|
+
See :py:meth:`DBEntry.put_slice <imas.db_entry.DBEntry.put_slice>`.
|
|
343
|
+
"""
|
|
344
|
+
if db_entry is None:
|
|
345
|
+
raise NotImplementedError()
|
|
346
|
+
db_entry.put_slice(self, occurrence)
|
|
347
|
+
|
|
348
|
+
@needs_imas
|
|
349
|
+
def deleteData(
|
|
350
|
+
self, occurrence: int = 0, db_entry: Optional["DBEntry"] = None
|
|
351
|
+
) -> None:
|
|
352
|
+
"""Delete AL backend storage data.
|
|
353
|
+
|
|
354
|
+
This method exists for API compatibility with the IMAS python HLI.
|
|
355
|
+
See :py:meth:`DBEntry.delete_data <imas.db_entry.DBEntry.delete_data>`.
|
|
356
|
+
"""
|
|
357
|
+
if db_entry is None:
|
|
358
|
+
raise NotImplementedError()
|
|
359
|
+
db_entry.delete_data(self, occurrence)
|
|
360
|
+
|
|
361
|
+
@needs_imas
|
|
362
|
+
def put(self, occurrence: int = 0, db_entry: Optional["DBEntry"] = None) -> None:
|
|
363
|
+
"""Put this IDS to the backend.
|
|
364
|
+
|
|
365
|
+
This method exists for API compatibility with the IMAS python HLI.
|
|
366
|
+
See :py:meth:`DBEntry.put <imas.db_entry.DBEntry.put>`.
|
|
367
|
+
"""
|
|
368
|
+
if db_entry is None:
|
|
369
|
+
raise NotImplementedError()
|
|
370
|
+
db_entry.put(self, occurrence)
|
|
371
|
+
|
|
372
|
+
def __repr__(self):
|
|
373
|
+
my_repr = f"<{type(self).__name__}"
|
|
374
|
+
my_repr += f" (IDS:{self.metadata.name})>"
|
|
375
|
+
return my_repr
|
|
376
|
+
|
|
377
|
+
@property
|
|
378
|
+
def _toplevel(self) -> "IDSToplevel":
|
|
379
|
+
"""Return ourselves"""
|
|
380
|
+
# Used to cut off recursive call
|
|
381
|
+
return self
|
imas/setup_logging.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# This file is part of imas-python.
|
|
2
|
+
# You should have received the imas-python LICENSE file with this project.
|
|
3
|
+
"""Create a default log handler when imas-python is imported.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class _PrettyFormatter(logging.Formatter):
|
|
11
|
+
"""Logging Formatter to add colors and count warning / errors"""
|
|
12
|
+
|
|
13
|
+
light_grey = "\x1b[90m"
|
|
14
|
+
yellow = "\x1b[33m"
|
|
15
|
+
red = "\x1b[31m"
|
|
16
|
+
bold_red = "\x1b[31;1m"
|
|
17
|
+
reset = "\x1b[0m"
|
|
18
|
+
|
|
19
|
+
formatstr = (
|
|
20
|
+
"%(asctime)s %(levelname)-8s %(message)s "
|
|
21
|
+
f"{light_grey}@%(filename)s:%(lineno)d{reset}"
|
|
22
|
+
)
|
|
23
|
+
time_format = "%H:%M:%S"
|
|
24
|
+
|
|
25
|
+
FORMATS = {
|
|
26
|
+
logging.DEBUG: logging.Formatter(light_grey + formatstr, time_format),
|
|
27
|
+
logging.INFO: logging.Formatter(formatstr, time_format),
|
|
28
|
+
logging.WARNING: logging.Formatter(yellow + formatstr, time_format),
|
|
29
|
+
logging.ERROR: logging.Formatter(red + formatstr, time_format),
|
|
30
|
+
logging.CRITICAL: logging.Formatter(bold_red + formatstr, time_format),
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
def format(self, record):
|
|
34
|
+
formatter = self.FORMATS.get(record.levelno)
|
|
35
|
+
return formatter.format(record)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_messages():
|
|
39
|
+
"""Print out a message on each logging level"""
|
|
40
|
+
logger = logging.getLogger("imas.testlogger")
|
|
41
|
+
logger.debug("Debug message")
|
|
42
|
+
logger.info("Info message")
|
|
43
|
+
logger.warning("Warning message")
|
|
44
|
+
logger.error("Error message")
|
|
45
|
+
logger.critical("Critical message")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def connect_formatter(logger):
|
|
49
|
+
"""Connect general formatter to given logger"""
|
|
50
|
+
ch = logging.StreamHandler()
|
|
51
|
+
ch.setLevel(logging.DEBUG)
|
|
52
|
+
ch.setFormatter(_PrettyFormatter())
|
|
53
|
+
logger.addHandler(ch)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Log to console by default, and output it all
|
|
57
|
+
logger = logging.getLogger("imas")
|
|
58
|
+
connect_formatter(logger)
|
|
59
|
+
|
|
60
|
+
loglevel = os.getenv("IMASPY_LOGLEVEL") or "INFO"
|
|
61
|
+
logger.setLevel(loglevel)
|
|
62
|
+
|
|
63
|
+
if __name__ == "__main__":
|
|
64
|
+
test_messages()
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from imas import dd_zip, ids_metadata
|
|
4
|
+
from imas.ids_factory import IDSFactory
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.fixture
|
|
8
|
+
def skip_object_caches(monkeypatch: pytest.MonkeyPatch):
|
|
9
|
+
monkeypatch.setattr(dd_zip, "_load_etree", dd_zip._load_etree.__wrapped__)
|
|
10
|
+
monkeypatch.setattr(
|
|
11
|
+
ids_metadata,
|
|
12
|
+
"get_toplevel_metadata",
|
|
13
|
+
ids_metadata.get_toplevel_metadata.__wrapped__,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture(params=dd_zip.dd_xml_versions())
|
|
18
|
+
def dd_version(request):
|
|
19
|
+
return request.param
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.slow
|
|
23
|
+
def test_create_ids_dd_version(dd_version, skip_object_caches):
|
|
24
|
+
# Test creation of all IDSs, test that IDSMetadata is correctly instantiated for
|
|
25
|
+
# all known DD verions
|
|
26
|
+
factory = IDSFactory(version=dd_version)
|
|
27
|
+
for ids_name in factory:
|
|
28
|
+
factory.new(ids_name)
|
imas/test/test_cli.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from click.testing import CliRunner
|
|
5
|
+
from packaging.version import Version
|
|
6
|
+
|
|
7
|
+
from imas.backends.imas_core.imas_interface import ll_interface
|
|
8
|
+
from imas.command.cli import print_version
|
|
9
|
+
from imas.command.db_analysis import analyze_db, process_db_analysis
|
|
10
|
+
from imas.db_entry import DBEntry
|
|
11
|
+
from imas.test.test_helpers import fill_with_random_data
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.mark.cli
|
|
15
|
+
def test_imas_version():
|
|
16
|
+
runner = CliRunner()
|
|
17
|
+
result = runner.invoke(print_version)
|
|
18
|
+
assert result.exit_code == 0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.cli
|
|
22
|
+
@pytest.mark.skipif(ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5")
|
|
23
|
+
def test_db_analysis(tmp_path):
|
|
24
|
+
# This only tests the happy flow, error handling is not tested
|
|
25
|
+
db_path = tmp_path / "test_db_analysis"
|
|
26
|
+
with DBEntry(f"imas:hdf5?path={db_path}", "w") as entry:
|
|
27
|
+
ids = entry.factory.core_profiles()
|
|
28
|
+
fill_with_random_data(ids)
|
|
29
|
+
entry.put(ids)
|
|
30
|
+
|
|
31
|
+
runner = CliRunner()
|
|
32
|
+
with runner.isolated_filesystem(temp_dir=tmp_path) as td:
|
|
33
|
+
analyze_result = runner.invoke(analyze_db, [str(db_path)])
|
|
34
|
+
assert analyze_result.exit_code == 0, analyze_result.output
|
|
35
|
+
|
|
36
|
+
outfile = Path(td) / "imas-db-analysis.json.gz"
|
|
37
|
+
assert outfile.exists()
|
|
38
|
+
|
|
39
|
+
# Show detailed output for core_profiles, and then an empty input to exit cleanly:
|
|
40
|
+
process_result = runner.invoke(
|
|
41
|
+
process_db_analysis, [str(outfile)], input="core_profiles\n\n"
|
|
42
|
+
)
|
|
43
|
+
assert process_result.exit_code == 0, process_result.output
|
|
44
|
+
assert "core_profiles" in process_result.output
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
import imas
|
|
4
|
+
import imas.ids_defs
|
|
5
|
+
from imas.backends.imas_core.imas_interface import has_imas, ll_interface
|
|
6
|
+
from imas.exception import UnknownDDVersion
|
|
7
|
+
from imas.test.test_helpers import compare_children, open_dbentry
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_dbentry_contextmanager(requires_imas):
|
|
11
|
+
entry = imas.DBEntry(imas.ids_defs.MEMORY_BACKEND, "test", 1, 1)
|
|
12
|
+
entry.create()
|
|
13
|
+
ids = entry.factory.core_profiles()
|
|
14
|
+
ids.ids_properties.homogeneous_time = 0
|
|
15
|
+
ids.ids_properties.comment = "test context manager"
|
|
16
|
+
entry.put(ids)
|
|
17
|
+
|
|
18
|
+
with imas.DBEntry(imas.ids_defs.MEMORY_BACKEND, "test", 1, 1) as entry2:
|
|
19
|
+
ids2 = entry2.get("core_profiles")
|
|
20
|
+
assert ids2.ids_properties.comment == ids.ids_properties.comment
|
|
21
|
+
|
|
22
|
+
# Check that entry2 was closed
|
|
23
|
+
assert entry2._dbe_impl is None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.mark.skipif(
|
|
27
|
+
not has_imas or ll_interface._al_version.major < 5,
|
|
28
|
+
reason="URI API not available",
|
|
29
|
+
)
|
|
30
|
+
def test_dbentry_contextmanager_uri(tmp_path):
|
|
31
|
+
entry = imas.DBEntry(f"imas:ascii?path={tmp_path}/testdb", "w")
|
|
32
|
+
ids = entry.factory.core_profiles()
|
|
33
|
+
ids.ids_properties.homogeneous_time = 0
|
|
34
|
+
ids.ids_properties.comment = "test context manager"
|
|
35
|
+
entry.put(ids)
|
|
36
|
+
|
|
37
|
+
with imas.DBEntry(f"imas:ascii?path={tmp_path}/testdb", "r") as entry2:
|
|
38
|
+
ids2 = entry2.get("core_profiles")
|
|
39
|
+
assert ids2.ids_properties.comment == ids.ids_properties.comment
|
|
40
|
+
|
|
41
|
+
# Check that entry2 was closed
|
|
42
|
+
assert entry2._dbe_impl is None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_entry_attrs(entry: imas.DBEntry):
|
|
46
|
+
return (
|
|
47
|
+
entry.backend_id,
|
|
48
|
+
entry.db_name,
|
|
49
|
+
entry.pulse,
|
|
50
|
+
entry.run,
|
|
51
|
+
entry.user_name,
|
|
52
|
+
entry.data_version,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_dbentry_constructor():
|
|
57
|
+
with pytest.raises(TypeError):
|
|
58
|
+
imas.DBEntry() # no arguments
|
|
59
|
+
with pytest.raises(TypeError):
|
|
60
|
+
imas.DBEntry(1) # not enough arguments
|
|
61
|
+
with pytest.raises(TypeError):
|
|
62
|
+
imas.DBEntry(1, 2, 3) # not enough arguments
|
|
63
|
+
with pytest.raises(TypeError):
|
|
64
|
+
imas.DBEntry(1, 2, 3, 4, 5, 6, 7) # too many arguments
|
|
65
|
+
with pytest.raises(TypeError):
|
|
66
|
+
imas.DBEntry("test", uri="test") # Double URI argument
|
|
67
|
+
with pytest.raises(TypeError):
|
|
68
|
+
imas.DBEntry(1, 2, 3, 4, shot=5) # Multiple values for argument pulse
|
|
69
|
+
with pytest.raises(ValueError):
|
|
70
|
+
imas.DBEntry(1, 2, pulse=3, run=4, shot=5) # Both shot and pulse
|
|
71
|
+
|
|
72
|
+
entry = imas.DBEntry(1, 2, 3, 4)
|
|
73
|
+
assert get_entry_attrs(entry) == (1, 2, 3, 4, None, None)
|
|
74
|
+
entry = imas.DBEntry(backend_id=1, db_name=2, pulse=3, run=4)
|
|
75
|
+
assert get_entry_attrs(entry) == (1, 2, 3, 4, None, None)
|
|
76
|
+
# Shot behaves as alias of pulse
|
|
77
|
+
entry = imas.DBEntry(backend_id=1, db_name=2, shot=3, run=4)
|
|
78
|
+
assert get_entry_attrs(entry) == (1, 2, 3, 4, None, None)
|
|
79
|
+
entry = imas.DBEntry(1, 2, 3, 4, 5, 6)
|
|
80
|
+
assert get_entry_attrs(entry) == (1, 2, 3, 4, 5, 6)
|
|
81
|
+
entry = imas.DBEntry(1, 2, 3, 4, data_version=6)
|
|
82
|
+
assert get_entry_attrs(entry) == (1, 2, 3, 4, None, 6)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path):
|
|
86
|
+
entry = open_dbentry(imas.ids_defs.MEMORY_BACKEND, "w", worker_id, tmp_path)
|
|
87
|
+
ids = entry.factory.core_profiles()
|
|
88
|
+
ids.ids_properties.homogeneous_time = 0
|
|
89
|
+
ids.ids_properties.comment = "Test unknown DD version"
|
|
90
|
+
# Put this IDS with an invalid DD version
|
|
91
|
+
with monkeypatch.context() as m:
|
|
92
|
+
m.setattr(entry.factory, "_version", "invalid DD version")
|
|
93
|
+
assert entry.dd_version == "invalid DD version"
|
|
94
|
+
entry.put(ids)
|
|
95
|
+
|
|
96
|
+
with pytest.raises(UnknownDDVersion) as exc_info:
|
|
97
|
+
entry.get("core_profiles")
|
|
98
|
+
assert "ignore_unknown_dd_version" in str(exc_info.value)
|
|
99
|
+
ids2 = entry.get("core_profiles", ignore_unknown_dd_version=True)
|
|
100
|
+
assert ids2.ids_properties.version_put.data_dictionary == "invalid DD version"
|
|
101
|
+
compare_children(ids, ids2)
|
|
102
|
+
# Test that autoconvert plays nicely with this option as well
|
|
103
|
+
ids3 = entry.get("core_profiles", ignore_unknown_dd_version=True, autoconvert=False)
|
|
104
|
+
compare_children(ids, ids3)
|