imas-python 2.1.0.post1__py3-none-any.whl → 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- imas/__init__.py +43 -15
- imas/_to_xarray.py +1 -1
- imas/_version.py +2 -2
- imas/backends/db_entry_impl.py +1 -1
- imas/backends/imas_core/al_context.py +3 -3
- imas/backends/imas_core/db_entry_al.py +26 -19
- imas/backends/imas_core/imas_interface.py +9 -22
- imas/backends/imas_core/mdsplus_model.py +19 -2
- imas/backends/imas_core/uda_support.py +2 -2
- imas/backends/netcdf/db_entry_nc.py +1 -1
- imas/backends/netcdf/nc2ids.py +10 -2
- imas/backends/netcdf/nc_metadata.py +3 -4
- imas/command/cli.py +68 -3
- imas/convert_core_edge_plasma.py +124 -0
- imas/db_entry.py +3 -3
- imas/exception.py +2 -4
- imas/ids_base.py +3 -3
- imas/ids_convert.py +3 -5
- imas/ids_defs.py +37 -77
- imas/ids_factory.py +18 -1
- imas/ids_metadata.py +6 -6
- imas/ids_toplevel.py +3 -9
- imas/test/test_cli.py +38 -3
- imas/test/test_convert_core_edge_plasma.py +68 -0
- imas/test/test_dbentry.py +3 -3
- imas/test/test_dbentry_uda.py +80 -0
- imas/test/test_exception.py +1 -1
- imas/test/test_ids_ascii_data.py +1 -1
- imas/test/test_ids_toplevel.py +1 -1
- imas/test/test_ids_validate.py +1 -1
- imas/test/test_latest_dd_autofill.py +1 -3
- imas/test/test_lazy_loading.py +5 -5
- imas/test/test_nbc_change.py +1 -1
- imas/test/test_snippets.py +1 -1
- imas/test/test_static_ids.py +1 -1
- imas/test/test_to_xarray.py +1 -1
- imas/test/test_util.py +3 -3
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/METADATA +1 -1
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/RECORD +43 -40
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/WHEEL +1 -1
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/entry_points.txt +0 -0
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/licenses/LICENSE.txt +0 -0
- {imas_python-2.1.0.post1.dist-info → imas_python-2.2.0.dist-info}/top_level.txt +0 -0
imas/__init__.py
CHANGED
|
@@ -1,30 +1,58 @@
|
|
|
1
1
|
# This file is part of IMAS-Python.
|
|
2
2
|
# You should have received the IMAS-Python LICENSE file with this project.
|
|
3
3
|
|
|
4
|
-
# isort: skip_file
|
|
5
|
-
|
|
6
4
|
from packaging.version import Version as _V
|
|
7
5
|
|
|
8
|
-
from ._version import version as __version__ # noqa: F401
|
|
9
|
-
from ._version import version_tuple # noqa: F401
|
|
10
|
-
|
|
11
6
|
# Import logging _first_
|
|
12
|
-
|
|
7
|
+
# isort: off
|
|
8
|
+
from . import setup_logging # noqa: F401
|
|
9
|
+
|
|
10
|
+
# isort: on
|
|
13
11
|
|
|
14
|
-
#
|
|
12
|
+
# Ensure that `imas.util` is loaded when importing imas
|
|
13
|
+
from . import util # noqa: F401
|
|
14
|
+
|
|
15
|
+
# Public API:
|
|
16
|
+
from ._version import version as __version__
|
|
17
|
+
from ._version import version_tuple
|
|
18
|
+
from .convert_core_edge_plasma import (
|
|
19
|
+
convert_to_plasma_profiles,
|
|
20
|
+
convert_to_plasma_sources,
|
|
21
|
+
convert_to_plasma_transport,
|
|
22
|
+
)
|
|
15
23
|
from .db_entry import DBEntry
|
|
16
|
-
from .ids_factory import IDSFactory
|
|
17
24
|
from .ids_convert import convert_ids
|
|
25
|
+
from .ids_data_type import IDSDataType
|
|
26
|
+
from .ids_factory import IDSFactory
|
|
18
27
|
from .ids_identifiers import identifiers
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
from . import
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
util,
|
|
25
|
-
)
|
|
28
|
+
from .ids_metadata import IDSMetadata, IDSType
|
|
29
|
+
from .ids_primitive import IDSPrimitive
|
|
30
|
+
from .ids_struct_array import IDSStructArray
|
|
31
|
+
from .ids_structure import IDSStructure
|
|
32
|
+
from .ids_toplevel import IDSToplevel
|
|
26
33
|
|
|
27
34
|
PUBLISHED_DOCUMENTATION_ROOT = "https://imas-python.readthedocs.io/en/latest/"
|
|
28
35
|
"""URL to the published documentation."""
|
|
29
36
|
OLDEST_SUPPORTED_VERSION = _V("3.22.0")
|
|
30
37
|
"""Oldest Data Dictionary version that is supported by IMAS-Python."""
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"__version__",
|
|
41
|
+
"version_tuple",
|
|
42
|
+
"DBEntry",
|
|
43
|
+
"IDSDataType",
|
|
44
|
+
"IDSFactory",
|
|
45
|
+
"IDSMetadata",
|
|
46
|
+
"IDSPrimitive",
|
|
47
|
+
"IDSStructure",
|
|
48
|
+
"IDSStructArray",
|
|
49
|
+
"IDSToplevel",
|
|
50
|
+
"IDSType",
|
|
51
|
+
"convert_ids",
|
|
52
|
+
"convert_to_plasma_profiles",
|
|
53
|
+
"convert_to_plasma_sources",
|
|
54
|
+
"convert_to_plasma_transport",
|
|
55
|
+
"identifiers",
|
|
56
|
+
"PUBLISHED_DOCUMENTATION_ROOT",
|
|
57
|
+
"OLDEST_SUPPORTED_VERSION",
|
|
58
|
+
]
|
imas/_to_xarray.py
CHANGED
|
@@ -27,7 +27,7 @@ def to_xarray(ids: IDSToplevel, *paths: str) -> xarray.Dataset:
|
|
|
27
27
|
# block checks if the paths are valid, and by using "metadata.path_string" we ensure
|
|
28
28
|
# that / are used as separator.
|
|
29
29
|
try:
|
|
30
|
-
paths = [ids.metadata[path].path_string for path in paths]
|
|
30
|
+
paths: list[str] = [ids.metadata[path].path_string for path in paths]
|
|
31
31
|
except KeyError as exc:
|
|
32
32
|
raise ValueError(str(exc)) from None
|
|
33
33
|
|
imas/_version.py
CHANGED
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '2.
|
|
32
|
-
__version_tuple__ = version_tuple = (2,
|
|
31
|
+
__version__ = version = '2.2.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (2, 2, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
imas/backends/db_entry_impl.py
CHANGED
|
@@ -115,7 +115,7 @@ class ALContext:
|
|
|
115
115
|
tmin: float,
|
|
116
116
|
tmax: float,
|
|
117
117
|
dtime: Optional[numpy.ndarray],
|
|
118
|
-
interpolation_method: int,
|
|
118
|
+
interpolation_method: Optional[int],
|
|
119
119
|
) -> "ALContext":
|
|
120
120
|
"""Begin a new timerange action for use in a ``with`` context."""
|
|
121
121
|
ctx = ll_interface.begin_timerange_action(
|
|
@@ -163,7 +163,7 @@ class ALContext:
|
|
|
163
163
|
"""Call ual_write_data with this context."""
|
|
164
164
|
status = ll_interface.write_data(self.ctx, path, timebasepath, data)
|
|
165
165
|
if status != 0:
|
|
166
|
-
raise LowlevelError(f"write data at {path!r}
|
|
166
|
+
raise LowlevelError(f"write data at {path!r}", status)
|
|
167
167
|
|
|
168
168
|
def list_all_occurrences(self, ids_name: str) -> List[int]:
|
|
169
169
|
"""List all occurrences of this IDS."""
|
|
@@ -359,7 +359,7 @@ class LazyALContext:
|
|
|
359
359
|
tmin: float,
|
|
360
360
|
tmax: float,
|
|
361
361
|
dtime: Optional[numpy.ndarray],
|
|
362
|
-
interpolation_method: int,
|
|
362
|
+
interpolation_method: Optional[int],
|
|
363
363
|
) -> Iterator["LazyALContext"]:
|
|
364
364
|
"""Lazily start a lowlevel timerange action, see
|
|
365
365
|
:meth:`ALContext.timerange_action`.
|
|
@@ -38,8 +38,8 @@ from imas.ids_toplevel import IDSToplevel
|
|
|
38
38
|
|
|
39
39
|
from .al_context import ALContext, LazyALContext
|
|
40
40
|
from .db_entry_helpers import delete_children, get_children, put_children
|
|
41
|
-
from .imas_interface import LLInterfaceError,
|
|
42
|
-
from .mdsplus_model import mdsplus_model_dir
|
|
41
|
+
from .imas_interface import LLInterfaceError, ll_interface
|
|
42
|
+
from .mdsplus_model import mdsplus_model_dir, get_mdsplus_model_var
|
|
43
43
|
from .uda_support import extract_idsdef, get_dd_version_from_idsdef_xml
|
|
44
44
|
|
|
45
45
|
_OPEN_MODES = {
|
|
@@ -52,14 +52,6 @@ _OPEN_MODES = {
|
|
|
52
52
|
logger = logging.getLogger(__name__)
|
|
53
53
|
|
|
54
54
|
|
|
55
|
-
def require_imas_available():
|
|
56
|
-
if not has_imas:
|
|
57
|
-
raise RuntimeError(
|
|
58
|
-
"The IMAS Core library is not available. Please install 'imas_core', "
|
|
59
|
-
"or load a supported IMAS module if you use an HPC environment."
|
|
60
|
-
)
|
|
61
|
-
|
|
62
|
-
|
|
63
55
|
class ALDBEntryImpl(DBEntryImpl):
|
|
64
56
|
"""DBEntry implementation using imas_core as a backend."""
|
|
65
57
|
|
|
@@ -86,7 +78,6 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
86
78
|
|
|
87
79
|
@classmethod
|
|
88
80
|
def from_uri(cls, uri: str, mode: str, factory: IDSFactory) -> "ALDBEntryImpl":
|
|
89
|
-
require_imas_available()
|
|
90
81
|
if mode not in _OPEN_MODES:
|
|
91
82
|
modes = list(_OPEN_MODES)
|
|
92
83
|
raise ValueError(f"Unknown mode {mode!r}, was expecting any of {modes}")
|
|
@@ -105,9 +96,6 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
105
96
|
options: Any,
|
|
106
97
|
factory: IDSFactory,
|
|
107
98
|
) -> "ALDBEntryImpl":
|
|
108
|
-
# Raise an error if imas is not available
|
|
109
|
-
require_imas_available()
|
|
110
|
-
|
|
111
99
|
# Set defaults
|
|
112
100
|
user_name = user_name or getpass.getuser()
|
|
113
101
|
data_version = data_version or factory.dd_version
|
|
@@ -131,7 +119,7 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
131
119
|
# Building the MDS+ models is required when creating a new Data Entry
|
|
132
120
|
ids_path = mdsplus_model_dir(factory)
|
|
133
121
|
if ids_path:
|
|
134
|
-
os.environ[
|
|
122
|
+
os.environ[get_mdsplus_model_var()] = ids_path
|
|
135
123
|
|
|
136
124
|
elif backend == "uda":
|
|
137
125
|
# Set IDSDEF_PATH to point the UDA backend to the selected DD version
|
|
@@ -149,7 +137,7 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
149
137
|
if idsdef_path is None:
|
|
150
138
|
# Extract XML from the DD zip and point UDA to it
|
|
151
139
|
idsdef_path = extract_idsdef(factory.version)
|
|
152
|
-
os.environ["IDSDEF_PATH"] = idsdef_path
|
|
140
|
+
os.environ["IDSDEF_PATH"] = str(idsdef_path)
|
|
153
141
|
|
|
154
142
|
elif backend in ["hdf5", "memory", "ascii", "flexbuffers"]:
|
|
155
143
|
pass # nothing to set up
|
|
@@ -184,7 +172,7 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
184
172
|
destination: IDSToplevel,
|
|
185
173
|
lazy: bool,
|
|
186
174
|
nbc_map: Optional[NBCPathMap],
|
|
187
|
-
) ->
|
|
175
|
+
) -> IDSToplevel:
|
|
188
176
|
if self._db_ctx is None:
|
|
189
177
|
raise RuntimeError("Database entry is not open.")
|
|
190
178
|
if lazy and self.backend == "ascii":
|
|
@@ -268,6 +256,22 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
268
256
|
raise DataEntryException(
|
|
269
257
|
f"IDS {ids_name!r}, occurrence {occurrence} is empty."
|
|
270
258
|
)
|
|
259
|
+
|
|
260
|
+
# UDA caching doesn't play well when the DD version of the on-disk IDS doesn't
|
|
261
|
+
# match the DD version of this DBEntry. See GH#97
|
|
262
|
+
if self.backend == "uda" and dd_version != self._ids_factory.dd_version:
|
|
263
|
+
cache_mode = self._querydict.get("cache_mode")
|
|
264
|
+
fetch = self._querydict.get("fetch")
|
|
265
|
+
if cache_mode != "none" and fetch not in ("1", "true"):
|
|
266
|
+
raise RuntimeError(
|
|
267
|
+
f"The Data Dictionary version of the data ({dd_version}) is "
|
|
268
|
+
"different from the Data Dictionary version of the DBEntry "
|
|
269
|
+
f"({self._ids_factory.dd_version}). This is not supported when "
|
|
270
|
+
f"using the UDA backend. See {imas.PUBLISHED_DOCUMENTATION_ROOT}"
|
|
271
|
+
"multi-dd.html#uda-backend-caching-and-data-dictionary-versions "
|
|
272
|
+
"for more details and workarounds."
|
|
273
|
+
)
|
|
274
|
+
|
|
271
275
|
return dd_version
|
|
272
276
|
|
|
273
277
|
def put(self, ids: IDSToplevel, occurrence: int, is_slice: bool) -> None:
|
|
@@ -299,7 +303,7 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
299
303
|
if occurrence != 0:
|
|
300
304
|
ll_path += f"/{occurrence}"
|
|
301
305
|
|
|
302
|
-
time_mode = ids.ids_properties.homogeneous_time
|
|
306
|
+
time_mode = int(ids.ids_properties.homogeneous_time)
|
|
303
307
|
if is_slice:
|
|
304
308
|
with self._db_ctx.global_action(ll_path, READ_OP) as read_ctx:
|
|
305
309
|
db_time_mode = read_ctx.read_data(
|
|
@@ -344,9 +348,12 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
344
348
|
ll_path += f"/{occurrence}"
|
|
345
349
|
ids = self._ids_factory.new(ids_name)
|
|
346
350
|
with self._db_ctx.global_action(ll_path, WRITE_OP) as write_ctx:
|
|
347
|
-
delete_children(ids.metadata, write_ctx
|
|
351
|
+
delete_children(ids.metadata, write_ctx)
|
|
348
352
|
|
|
349
353
|
def list_all_occurrences(self, ids_name: str) -> List[int]:
|
|
354
|
+
if self._db_ctx is None:
|
|
355
|
+
raise RuntimeError("Database entry is not open.")
|
|
356
|
+
|
|
350
357
|
try:
|
|
351
358
|
occurrence_list = self._db_ctx.list_all_occurrences(ids_name)
|
|
352
359
|
except LLInterfaceError:
|
|
@@ -12,30 +12,17 @@ import logging
|
|
|
12
12
|
|
|
13
13
|
from packaging.version import Version
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
# Import the Access Layer module
|
|
16
|
+
# First try to import imas_core, which is available since AL 5.2
|
|
17
|
+
from imas_core import _al_lowlevel as lowlevel
|
|
18
|
+
from imas_core import imasdef # noqa: F401
|
|
16
19
|
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
17
21
|
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
from imas_core import _al_lowlevel as lowlevel
|
|
23
|
-
from imas_core import imasdef
|
|
24
|
-
|
|
25
|
-
# Enable throwing exceptions from the _al_lowlevel interface
|
|
26
|
-
enable_exceptions = getattr(lowlevel, "imas_core_config_enable_exceptions", None)
|
|
27
|
-
if enable_exceptions:
|
|
28
|
-
enable_exceptions()
|
|
29
|
-
|
|
30
|
-
except ImportError as exc:
|
|
31
|
-
imas = None
|
|
32
|
-
has_imas = False
|
|
33
|
-
imasdef = None
|
|
34
|
-
lowlevel = None
|
|
35
|
-
logger.warning(
|
|
36
|
-
"Could not import 'imas_core': %s. Some functionality is not available.",
|
|
37
|
-
exc,
|
|
38
|
-
)
|
|
22
|
+
# Enable throwing exceptions from the _al_lowlevel interface
|
|
23
|
+
enable_exceptions = getattr(lowlevel, "imas_core_config_enable_exceptions", None)
|
|
24
|
+
if enable_exceptions:
|
|
25
|
+
enable_exceptions()
|
|
39
26
|
|
|
40
27
|
|
|
41
28
|
class LLInterfaceError(RuntimeError):
|
|
@@ -14,6 +14,7 @@ import uuid
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
from subprocess import CalledProcessError, check_output
|
|
16
16
|
from zlib import crc32
|
|
17
|
+
from packaging.version import Version
|
|
17
18
|
|
|
18
19
|
try:
|
|
19
20
|
from importlib.resources import as_file, files
|
|
@@ -24,6 +25,8 @@ from imas.dd_zip import get_dd_xml, get_dd_xml_crc
|
|
|
24
25
|
from imas.exception import MDSPlusModelError
|
|
25
26
|
from imas.ids_factory import IDSFactory
|
|
26
27
|
|
|
28
|
+
from .imas_interface import ll_interface
|
|
29
|
+
|
|
27
30
|
logger = logging.getLogger(__name__)
|
|
28
31
|
|
|
29
32
|
|
|
@@ -292,6 +295,20 @@ def create_model_ids_xml(cache_dir_path, fname, version):
|
|
|
292
295
|
raise e
|
|
293
296
|
|
|
294
297
|
|
|
298
|
+
def get_mdsplus_model_var() -> str:
|
|
299
|
+
"""
|
|
300
|
+
Return the environemnt variable name used by IMAS-Core to locate models:
|
|
301
|
+
|
|
302
|
+
- 'ids_path' for IMAS-Core<5.6
|
|
303
|
+
- 'MDSPLUS_MODELS_PATH' for IMAS-Core>=5.6
|
|
304
|
+
"""
|
|
305
|
+
return (
|
|
306
|
+
"ids_path"
|
|
307
|
+
if ll_interface._al_version < Version("5.6.0")
|
|
308
|
+
else "MDSPLUS_MODELS_PATH"
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
295
312
|
def create_mdsplus_model(cache_dir_path: Path) -> None:
|
|
296
313
|
"""Use jtraverser to compile a valid MDS model file."""
|
|
297
314
|
try:
|
|
@@ -322,7 +339,7 @@ def create_mdsplus_model(cache_dir_path: Path) -> None:
|
|
|
322
339
|
env={
|
|
323
340
|
"PATH": os.environ.get("PATH", ""),
|
|
324
341
|
"LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", ""),
|
|
325
|
-
|
|
342
|
+
get_mdsplus_model_var(): str(cache_dir_path),
|
|
326
343
|
},
|
|
327
344
|
)
|
|
328
345
|
# Touch a file to show that we have finished the model
|
|
@@ -349,7 +366,7 @@ def jTraverser_jar() -> Path:
|
|
|
349
366
|
for component in os.environ.get("CLASSPATH", "").split(":"):
|
|
350
367
|
if component.endswith(".jar"):
|
|
351
368
|
if re.search(".*jTraverser.jar", component):
|
|
352
|
-
return component
|
|
369
|
+
return Path(component)
|
|
353
370
|
else: # assume its a directory (strip any '*' suffix)
|
|
354
371
|
search_dirs.append(component.rstrip("*"))
|
|
355
372
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Union
|
|
3
|
+
from typing import Union, Optional
|
|
4
4
|
from xml.etree import ElementTree as ET
|
|
5
5
|
|
|
6
6
|
from imas import dd_zip
|
|
@@ -10,7 +10,7 @@ from .mdsplus_model import _get_xdg_cache_dir
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def get_dd_version_from_idsdef_xml(path: Union[str, Path]) -> str:
|
|
13
|
+
def get_dd_version_from_idsdef_xml(path: Union[str, Path]) -> Optional[str]:
|
|
14
14
|
"""Parse the IDSDef.xml up to the point where the Data Dictionary version is set.
|
|
15
15
|
|
|
16
16
|
Returns:
|
|
@@ -100,7 +100,7 @@ class NCDBEntryImpl(DBEntryImpl):
|
|
|
100
100
|
destination: IDSToplevel,
|
|
101
101
|
lazy: bool,
|
|
102
102
|
nbc_map: Optional[NBCPathMap],
|
|
103
|
-
) ->
|
|
103
|
+
) -> IDSToplevel:
|
|
104
104
|
# Feature compatibility checks
|
|
105
105
|
if parameters is not None:
|
|
106
106
|
if isinstance(parameters, GetSliceParameters):
|
imas/backends/netcdf/nc2ids.py
CHANGED
|
@@ -157,6 +157,8 @@ class NC2IDS:
|
|
|
157
157
|
for index, node in indexed_tree_iter(self.ids, target_metadata):
|
|
158
158
|
value = data[index]
|
|
159
159
|
if value != getattr(var, "_FillValue", None):
|
|
160
|
+
if isinstance(value, np.generic):
|
|
161
|
+
value = value.item()
|
|
160
162
|
# NOTE: bypassing IDSPrimitive.value.setter logic
|
|
161
163
|
node._IDSPrimitive__value = value
|
|
162
164
|
|
|
@@ -166,10 +168,16 @@ class NC2IDS:
|
|
|
166
168
|
# here, we'll let IDSPrimitive.value.setter take care of it:
|
|
167
169
|
self.ids[target_metadata.path].value = data
|
|
168
170
|
|
|
169
|
-
|
|
171
|
+
# We need to unpack 0D ints, floats and complex numbers. For better
|
|
172
|
+
# performance this check is done outside the for-loop:
|
|
173
|
+
elif metadata.ndim or metadata.data_type is IDSDataType.STR:
|
|
170
174
|
for index, node in indexed_tree_iter(self.ids, target_metadata):
|
|
171
175
|
# NOTE: bypassing IDSPrimitive.value.setter logic
|
|
172
176
|
node._IDSPrimitive__value = data[index]
|
|
177
|
+
else:
|
|
178
|
+
for index, node in indexed_tree_iter(self.ids, target_metadata):
|
|
179
|
+
# NOTE: bypassing IDSPrimitive.value.setter logic
|
|
180
|
+
node._IDSPrimitive__value = data[index].item() # Unpack 0D value
|
|
173
181
|
|
|
174
182
|
def validate_variables(self) -> None:
|
|
175
183
|
"""Validate that all variables in the netCDF Group exist and match the DD."""
|
|
@@ -365,7 +373,7 @@ class LazyContext:
|
|
|
365
373
|
value = var[self.index]
|
|
366
374
|
|
|
367
375
|
if value is not None:
|
|
368
|
-
if isinstance(value, np.ndarray):
|
|
376
|
+
if isinstance(value, (np.ndarray, np.generic)):
|
|
369
377
|
if value.ndim == 0: # Unpack 0D numpy arrays:
|
|
370
378
|
value = value.item()
|
|
371
379
|
else:
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# This file is part of IMAS-Python.
|
|
2
2
|
# You should have received the IMAS-Python LICENSE file with this project.
|
|
3
|
-
"""NetCDF metadata for dimensions and tensorization of IDSs.
|
|
4
|
-
"""
|
|
3
|
+
"""NetCDF metadata for dimensions and tensorization of IDSs."""
|
|
5
4
|
|
|
6
5
|
from functools import lru_cache
|
|
7
6
|
from typing import Dict, List, Optional, Set, Tuple
|
|
@@ -89,7 +88,7 @@ class NCMetadata:
|
|
|
89
88
|
# Add cache for public API
|
|
90
89
|
self.get_dimensions = lru_cache(maxsize=None)(self.get_dimensions)
|
|
91
90
|
|
|
92
|
-
def get_coordinates(self, path: str, homogeneous_time: bool) -> Tuple[str]:
|
|
91
|
+
def get_coordinates(self, path: str, homogeneous_time: bool) -> Tuple[str, ...]:
|
|
93
92
|
"""Get the coordinates (adhering to CF conventions) for a netCDF variable.
|
|
94
93
|
|
|
95
94
|
Args:
|
|
@@ -109,7 +108,7 @@ class NCMetadata:
|
|
|
109
108
|
for coord in self.coordinates[path]
|
|
110
109
|
)
|
|
111
110
|
|
|
112
|
-
def get_dimensions(self, path: str, homogeneous_time: bool) -> Tuple[str]:
|
|
111
|
+
def get_dimensions(self, path: str, homogeneous_time: bool) -> Tuple[str, ...]:
|
|
113
112
|
"""Get the dimensions for a netCDF variable.
|
|
114
113
|
|
|
115
114
|
Args:
|
imas/command/cli.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# This file is part of IMAS-Python.
|
|
2
2
|
# You should have received the IMAS-Python LICENSE file with this project.
|
|
3
|
-
"""
|
|
3
|
+
"""Main CLI entry point"""
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
import sys
|
|
@@ -22,7 +22,13 @@ from rich.table import Table
|
|
|
22
22
|
|
|
23
23
|
import imas
|
|
24
24
|
import imas.backends.imas_core.imas_interface
|
|
25
|
-
from imas import
|
|
25
|
+
from imas import (
|
|
26
|
+
DBEntry,
|
|
27
|
+
dd_zip,
|
|
28
|
+
convert_to_plasma_profiles,
|
|
29
|
+
convert_to_plasma_sources,
|
|
30
|
+
convert_to_plasma_transport,
|
|
31
|
+
)
|
|
26
32
|
from imas.backends.imas_core.imas_interface import ll_interface
|
|
27
33
|
from imas.command.db_analysis import analyze_db, process_db_analysis
|
|
28
34
|
from imas.command.helpers import min_version_guard, setup_rich_log_handler
|
|
@@ -109,6 +115,23 @@ def print_ids(uri, ids, occurrence, print_all):
|
|
|
109
115
|
imas.util.print_tree(ids_obj, not print_all)
|
|
110
116
|
|
|
111
117
|
|
|
118
|
+
def _check_convert_to_plasma_ids(idss_with_occurrences):
|
|
119
|
+
"""Check if no plasma_ IDS is present when converting a core_ or edge_ IDS."""
|
|
120
|
+
idsnames = {ids_name for ids_name, _ in idss_with_occurrences}
|
|
121
|
+
for suffix in ("_profiles", "_sources", "_transport"):
|
|
122
|
+
if f"plasma{suffix}" in idsnames:
|
|
123
|
+
if f"core{suffix}" in idsnames:
|
|
124
|
+
overlap = "core"
|
|
125
|
+
elif f"edge{suffix}" in idsnames:
|
|
126
|
+
overlap = "edge"
|
|
127
|
+
else:
|
|
128
|
+
continue
|
|
129
|
+
raise RuntimeError(
|
|
130
|
+
f"Cannot convert {overlap}{suffix} IDS to plasma{suffix}: "
|
|
131
|
+
f"there already exists a plasma{suffix} IDS in the data source."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
112
135
|
@cli.command("convert", no_args_is_help=True)
|
|
113
136
|
@click.argument("uri_in")
|
|
114
137
|
@click.argument("dd_version")
|
|
@@ -127,8 +150,21 @@ If not provided, all IDSs in the data entry are converted.",
|
|
|
127
150
|
is_flag=True,
|
|
128
151
|
help="Don't add provenance metadata to the converted IDS.",
|
|
129
152
|
)
|
|
153
|
+
@click.option(
|
|
154
|
+
"--convert-to-plasma-ids",
|
|
155
|
+
is_flag=True,
|
|
156
|
+
help="Convert core/edge profiles/transport/sources to the corresponding plasma IDS",
|
|
157
|
+
)
|
|
130
158
|
def convert_ids(
|
|
131
|
-
uri_in
|
|
159
|
+
uri_in: str,
|
|
160
|
+
dd_version: str,
|
|
161
|
+
uri_out: str,
|
|
162
|
+
ids: str,
|
|
163
|
+
occurrence: int,
|
|
164
|
+
quiet: bool,
|
|
165
|
+
timeit: bool,
|
|
166
|
+
no_provenance: bool,
|
|
167
|
+
convert_to_plasma_ids: bool,
|
|
132
168
|
):
|
|
133
169
|
"""Convert a Data Entry (or a single IDS) to the target DD version.
|
|
134
170
|
|
|
@@ -174,6 +210,10 @@ def convert_ids(
|
|
|
174
210
|
else:
|
|
175
211
|
idss_with_occurrences.append((ids_name, occurrence))
|
|
176
212
|
|
|
213
|
+
if convert_to_plasma_ids: # Sanity checks for conversion to plasma IDSs
|
|
214
|
+
_check_convert_to_plasma_ids(idss_with_occurrences)
|
|
215
|
+
next_plasma_occurrence = {"_profiles": 0, "_transport": 0, "_sources": 0}
|
|
216
|
+
|
|
177
217
|
# Create progress bar and task
|
|
178
218
|
columns = (
|
|
179
219
|
TimeElapsedColumn(),
|
|
@@ -209,6 +249,31 @@ def convert_ids(
|
|
|
209
249
|
provenance_origin_uri=provenance_origin_uri,
|
|
210
250
|
)
|
|
211
251
|
|
|
252
|
+
# Convert to plasma_profiles/plasma_sources/plasma_transport IDS
|
|
253
|
+
if convert_to_plasma_ids and ids_name.startswith(("core", "edge")):
|
|
254
|
+
suffix = ids_name[4:]
|
|
255
|
+
# This branch also matches core_instant_changes: check that suffix is ok
|
|
256
|
+
if suffix in next_plasma_occurrence:
|
|
257
|
+
logger.info(
|
|
258
|
+
"Storing IDS %s/%d as plasma%s/%d",
|
|
259
|
+
ids_name,
|
|
260
|
+
occurrence,
|
|
261
|
+
suffix,
|
|
262
|
+
next_plasma_occurrence[suffix],
|
|
263
|
+
)
|
|
264
|
+
occurrence = next_plasma_occurrence[suffix]
|
|
265
|
+
next_plasma_occurrence[suffix] += 1
|
|
266
|
+
|
|
267
|
+
name2 = f"[bold green]plasma{suffix}[/][green]/{occurrence}[/]"
|
|
268
|
+
progress.update(task, description=f"Converting {name} to {name2}")
|
|
269
|
+
if suffix == "_profiles":
|
|
270
|
+
ids2 = convert_to_plasma_profiles(ids2)
|
|
271
|
+
elif suffix == "_sources":
|
|
272
|
+
ids2 = convert_to_plasma_sources(ids2)
|
|
273
|
+
elif suffix == "_transport":
|
|
274
|
+
ids2 = convert_to_plasma_transport(ids2)
|
|
275
|
+
name = name2
|
|
276
|
+
|
|
212
277
|
# Store in output entry:
|
|
213
278
|
progress.update(task, description=f"Storing {name}", advance=1)
|
|
214
279
|
with timer("Put", name):
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# This file is part of IMAS-Python.
|
|
2
|
+
# You should have received the IMAS-Python LICENSE file with this project.
|
|
3
|
+
"""Logic to convert core/edge IDSs to their corresponding plasma ID."""
|
|
4
|
+
|
|
5
|
+
from packaging.version import Version
|
|
6
|
+
|
|
7
|
+
from imas.ids_toplevel import IDSToplevel
|
|
8
|
+
from imas.ids_factory import IDSFactory
|
|
9
|
+
from imas.exception import IDSNameError
|
|
10
|
+
from imas.ids_convert import DDVersionMap, NBCPathMap, _copy_structure
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def convert_to_plasma_profiles(
|
|
14
|
+
core_or_edge_profiles: IDSToplevel, *, deepcopy: bool = False
|
|
15
|
+
) -> IDSToplevel:
|
|
16
|
+
"""Convert a core_profiles or edge_profiles IDS to a plasma_profiles IDS.
|
|
17
|
+
|
|
18
|
+
The input IDS must use a Data Dictionary version for which the plasma_profiles IDS
|
|
19
|
+
exists (3.42.0 or newer).
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
core_or_edge_profiles: The core_profiles or edge_profiles IDS to be converted.
|
|
23
|
+
|
|
24
|
+
Keyword Args:
|
|
25
|
+
deepcopy: When True, performs a deep copy of all data. When False (default),
|
|
26
|
+
numpy arrays are not copied and the converted IDS shares the same underlying
|
|
27
|
+
data buffers.
|
|
28
|
+
"""
|
|
29
|
+
return _convert_to_plasma(core_or_edge_profiles, "profiles", deepcopy)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def convert_to_plasma_sources(
|
|
33
|
+
core_or_edge_sources: IDSToplevel, *, deepcopy: bool = False
|
|
34
|
+
) -> IDSToplevel:
|
|
35
|
+
"""Convert a core_sources or edge_sources IDS to a plasma_sources IDS.
|
|
36
|
+
|
|
37
|
+
The input IDS must use a Data Dictionary version for which the plasma_sources IDS
|
|
38
|
+
exists (3.42.0 or newer).
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
core_or_edge_sources: The core_sources or edge_sources IDS to be converted.
|
|
42
|
+
|
|
43
|
+
Keyword Args:
|
|
44
|
+
deepcopy: When True, performs a deep copy of all data. When False (default),
|
|
45
|
+
numpy arrays are not copied and the converted IDS shares the same underlying
|
|
46
|
+
data buffers.
|
|
47
|
+
"""
|
|
48
|
+
return _convert_to_plasma(core_or_edge_sources, "sources", deepcopy)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def convert_to_plasma_transport(
|
|
52
|
+
core_or_edge_transport: IDSToplevel, *, deepcopy: bool = False
|
|
53
|
+
) -> IDSToplevel:
|
|
54
|
+
"""Convert a core_transport or edge_transport IDS to a plasma_transport IDS.
|
|
55
|
+
|
|
56
|
+
The input IDS must use a Data Dictionary version for which the plasma_transport IDS
|
|
57
|
+
exists (3.42.0 or newer).
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
core_or_edge_transport: The core_transport or edge_transport IDS to be
|
|
61
|
+
converted.
|
|
62
|
+
|
|
63
|
+
Keyword Args:
|
|
64
|
+
deepcopy: When True, performs a deep copy of all data. When False (default),
|
|
65
|
+
numpy arrays are not copied and the converted IDS shares the same underlying
|
|
66
|
+
data buffers.
|
|
67
|
+
"""
|
|
68
|
+
return _convert_to_plasma(core_or_edge_transport, "transport", deepcopy)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class _CoreEdgePlasmaMap(DDVersionMap):
|
|
72
|
+
"""Subclass of DDVersionMap to generate an NBCPathMap that is suitable to copy
|
|
73
|
+
between a core/edge IDS and the corresponding plasma IDS."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, source, target, factory):
|
|
76
|
+
self.ids_name = source
|
|
77
|
+
self.old_version = factory._etree
|
|
78
|
+
self.new_version = factory._etree
|
|
79
|
+
self.version_old = Version(factory.version)
|
|
80
|
+
|
|
81
|
+
self.old_to_new = NBCPathMap()
|
|
82
|
+
self.new_to_old = NBCPathMap()
|
|
83
|
+
|
|
84
|
+
old_ids_object = factory._etree.find(f"IDS[@name='{source}']")
|
|
85
|
+
new_ids_object = factory._etree.find(f"IDS[@name='{target}']")
|
|
86
|
+
self._build_map(old_ids_object, new_ids_object)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _convert_to_plasma(source: IDSToplevel, suffix: str, deepcopy: bool) -> IDSToplevel:
|
|
90
|
+
# Sanity checks for input data
|
|
91
|
+
if not isinstance(source, IDSToplevel):
|
|
92
|
+
raise TypeError(
|
|
93
|
+
f"First argument to convert_to_plasma_{suffix} must be a core_{suffix} or "
|
|
94
|
+
f"edge_{suffix} of type IDSToplevel. Got a type {type(source)} instead."
|
|
95
|
+
)
|
|
96
|
+
if source.metadata.name not in [f"core_{suffix}", f"edge_{suffix}"]:
|
|
97
|
+
raise ValueError(
|
|
98
|
+
f"First argument to convert_to_plasma_{suffix} must be a core_{suffix} or "
|
|
99
|
+
f"edge_{suffix} IDS. Got a {source.metadata.name} IDS instead."
|
|
100
|
+
)
|
|
101
|
+
if source._lazy:
|
|
102
|
+
raise NotImplementedError(
|
|
103
|
+
"IDS conversion is not implemented for lazy-loaded IDSs"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# Construct target plasma_{suffix} IDS
|
|
107
|
+
factory: IDSFactory = source._parent
|
|
108
|
+
try:
|
|
109
|
+
target = factory.new(f"plasma_{suffix}")
|
|
110
|
+
except IDSNameError:
|
|
111
|
+
raise ValueError(
|
|
112
|
+
f"Cannot convert {source.metadata.name} IDS to plasma_{suffix}: the source "
|
|
113
|
+
f"IDS uses Data Dictionary version {factory.dd_version} which doesn't have "
|
|
114
|
+
f"a plasma_{suffix} IDS. Please convert the source IDS to a supported Data "
|
|
115
|
+
"Dictionary version using `imas.convert_ids` and try again."
|
|
116
|
+
) from None
|
|
117
|
+
|
|
118
|
+
# Leverage existing logic from ids_convert to do the copying
|
|
119
|
+
# First construct a map (to handle missing items in the target IDS)
|
|
120
|
+
data_map = _CoreEdgePlasmaMap(source.metadata.name, target.metadata.name, factory)
|
|
121
|
+
path_map = data_map.old_to_new # old = core/edge, new = plasma IDS
|
|
122
|
+
_copy_structure(source, target, deepcopy, path_map)
|
|
123
|
+
|
|
124
|
+
return target
|