imas-python 2.2.0__py3-none-any.whl → 2.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- imas/_to_xarray.py +20 -13
- imas/_version.py +9 -19
- imas/assets/ITER_134173_106_equilibrium.ids +0 -2
- imas/assets/equilibrium.ids +0 -2
- imas/backends/db_entry_impl.py +8 -0
- imas/backends/imas_core/al_context.py +11 -0
- imas/backends/imas_core/db_entry_al.py +13 -0
- imas/backends/imas_core/imas_interface.py +5 -0
- imas/backends/imas_core/mdsplus_model.py +11 -3
- imas/backends/netcdf/db_entry_nc.py +23 -6
- imas/backends/netcdf/ids2nc.py +12 -46
- imas/backends/netcdf/ids_tensorizer.py +69 -10
- imas/backends/netcdf/nc2ids.py +13 -14
- imas/db_entry.py +67 -4
- imas/ids_convert.py +66 -32
- imas/ids_coordinates.py +11 -10
- imas/ids_identifiers.py +28 -15
- imas/ids_primitive.py +3 -3
- imas/ids_structure.py +1 -1
- imas/test/test_convert_core_edge_plasma.py +1 -1
- imas/test/test_hash.py +1 -1
- imas/test/test_identifiers.py +18 -9
- imas/test/test_ids_convert.py +72 -0
- imas/test/test_list_filled_paths.py +101 -0
- imas/test/test_to_xarray.py +25 -2
- imas/training.py +11 -6
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/METADATA +1 -1
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/RECORD +32 -31
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/WHEEL +1 -1
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/entry_points.txt +0 -0
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/licenses/LICENSE.txt +0 -0
- {imas_python-2.2.0.dist-info → imas_python-2.3.1.dist-info}/top_level.txt +0 -0
imas/_to_xarray.py
CHANGED
|
@@ -8,7 +8,7 @@ from imas.backends.netcdf.ids_tensorizer import IDSTensorizer
|
|
|
8
8
|
from imas.ids_data_type import IDSDataType
|
|
9
9
|
|
|
10
10
|
fillvals = {
|
|
11
|
-
IDSDataType.INT: -(2**31) + 1,
|
|
11
|
+
IDSDataType.INT: numpy.int32(-(2**31) + 1),
|
|
12
12
|
IDSDataType.STR: "",
|
|
13
13
|
IDSDataType.FLT: numpy.nan,
|
|
14
14
|
IDSDataType.CPX: numpy.nan * (1 + 1j),
|
|
@@ -50,21 +50,28 @@ def to_xarray(ids: IDSToplevel, *paths: str) -> xarray.Dataset:
|
|
|
50
50
|
var_name = path.replace("/", ".")
|
|
51
51
|
metadata = ids.metadata[path]
|
|
52
52
|
if metadata.data_type in (IDSDataType.STRUCTURE, IDSDataType.STRUCT_ARRAY):
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
coordinates = tensorizer.filter_coordinates(path)
|
|
62
|
-
if coordinates:
|
|
63
|
-
coordinate_names.update(coordinates.split(" "))
|
|
64
|
-
attrs["coordinates"] = coordinates
|
|
53
|
+
# Metadata variables for (arrays of) structures
|
|
54
|
+
if paths and path not in paths:
|
|
55
|
+
continue
|
|
56
|
+
dimensions = ()
|
|
57
|
+
data = b""
|
|
58
|
+
else:
|
|
59
|
+
dimensions = tensorizer.get_dimensions(path)
|
|
60
|
+
data = tensorizer.tensorize(path, fillvals[metadata.data_type])
|
|
65
61
|
|
|
62
|
+
attrs = tensorizer.get_attributes(path, fillvals)
|
|
63
|
+
if "coordinates" in attrs:
|
|
64
|
+
coordinate_names.update(attrs["coordinates"].split(" "))
|
|
66
65
|
data_vars[var_name] = (dimensions, data, attrs)
|
|
67
66
|
|
|
67
|
+
# :shape array for sparse data
|
|
68
|
+
if path in tensorizer.shapes and metadata.ndim:
|
|
69
|
+
shape_name = f"{var_name}:shape"
|
|
70
|
+
dimensions = tensorizer.get_shape_dimensions(path)
|
|
71
|
+
data = tensorizer.shapes[path]
|
|
72
|
+
attrs = tensorizer.get_shape_attributes(var_name)
|
|
73
|
+
data_vars[shape_name] = (dimensions, data, attrs)
|
|
74
|
+
|
|
68
75
|
# Remove coordinates from data_vars and put in coordinates mapping:
|
|
69
76
|
coordinates = {}
|
|
70
77
|
for coordinate_name in coordinate_names:
|
imas/_version.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
3
4
|
|
|
4
5
|
__all__ = [
|
|
5
6
|
"__version__",
|
|
@@ -10,25 +11,14 @@ __all__ = [
|
|
|
10
11
|
"commit_id",
|
|
11
12
|
]
|
|
12
13
|
|
|
13
|
-
TYPE_CHECKING = False
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from typing import Tuple
|
|
16
|
-
from typing import Union
|
|
17
|
-
|
|
18
|
-
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
-
COMMIT_ID = Union[str, None]
|
|
20
|
-
else:
|
|
21
|
-
VERSION_TUPLE = object
|
|
22
|
-
COMMIT_ID = object
|
|
23
|
-
|
|
24
14
|
version: str
|
|
25
15
|
__version__: str
|
|
26
|
-
__version_tuple__:
|
|
27
|
-
version_tuple:
|
|
28
|
-
commit_id:
|
|
29
|
-
__commit_id__:
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
30
20
|
|
|
31
|
-
__version__ = version = '2.
|
|
32
|
-
__version_tuple__ = version_tuple = (2,
|
|
21
|
+
__version__ = version = '2.3.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 3, 1)
|
|
33
23
|
|
|
34
|
-
__commit_id__ = commit_id =
|
|
24
|
+
__commit_id__ = commit_id = 'gdc78cc36c'
|
imas/assets/equilibrium.ids
CHANGED
imas/backends/db_entry_impl.py
CHANGED
|
@@ -120,3 +120,11 @@ class DBEntryImpl(ABC):
|
|
|
120
120
|
@abstractmethod
|
|
121
121
|
def list_all_occurrences(self, ids_name: str) -> List[int]:
|
|
122
122
|
"""Implement DBEntry.list_all_occurrences()"""
|
|
123
|
+
|
|
124
|
+
@abstractmethod
|
|
125
|
+
def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]:
|
|
126
|
+
"""Implement DBEntry.list_filled_paths().
|
|
127
|
+
|
|
128
|
+
N.B. DD conversion is handled in DBEntry.list_filled_paths(), this method
|
|
129
|
+
returns the data paths as stored on-disk.
|
|
130
|
+
"""
|
|
@@ -174,6 +174,17 @@ class ALContext:
|
|
|
174
174
|
return list(occurrences)
|
|
175
175
|
return []
|
|
176
176
|
|
|
177
|
+
def list_filled_paths(self, path: str) -> List[str]:
|
|
178
|
+
"""List all filled paths in an IDS.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
path: IDS and occurrence as a string: <IDS>[/<occurrence>]
|
|
182
|
+
"""
|
|
183
|
+
status, result = ll_interface.list_filled_paths(self.ctx, path)
|
|
184
|
+
if status != 0:
|
|
185
|
+
raise LowlevelError(f"list filled paths for {path!r}", status)
|
|
186
|
+
return result
|
|
187
|
+
|
|
177
188
|
def close(self):
|
|
178
189
|
"""Close this ALContext."""
|
|
179
190
|
ll_interface.end_action(self.ctx)
|
|
@@ -364,6 +364,19 @@ class ALDBEntryImpl(DBEntryImpl):
|
|
|
364
364
|
) from None
|
|
365
365
|
return occurrence_list
|
|
366
366
|
|
|
367
|
+
def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]:
|
|
368
|
+
if self._db_ctx is None:
|
|
369
|
+
raise RuntimeError("Database entry is not open.")
|
|
370
|
+
ll_path = ids_name
|
|
371
|
+
if occurrence != 0:
|
|
372
|
+
ll_path += f"/{occurrence}"
|
|
373
|
+
paths = self._db_ctx.list_filled_paths(ll_path)
|
|
374
|
+
if not paths:
|
|
375
|
+
raise DataEntryException(
|
|
376
|
+
f"IDS {ids_name!r}, occurrence {occurrence} is empty."
|
|
377
|
+
)
|
|
378
|
+
return paths
|
|
379
|
+
|
|
367
380
|
def _check_uda_warnings(self, lazy: bool) -> None:
|
|
368
381
|
"""Various checks / warnings for the UDA backend."""
|
|
369
382
|
cache_mode = self._querydict.get("cache_mode")
|
|
@@ -166,6 +166,11 @@ class LowlevelInterface:
|
|
|
166
166
|
):
|
|
167
167
|
raise self._minimal_version("5.4")
|
|
168
168
|
|
|
169
|
+
# New method in AL 5.7
|
|
170
|
+
|
|
171
|
+
def list_filled_paths(self, ctx, path):
|
|
172
|
+
raise self._minimal_version("5.7")
|
|
173
|
+
|
|
169
174
|
|
|
170
175
|
# Dummy documentation for interface:
|
|
171
176
|
for funcname in dir(LowlevelInterface):
|
|
@@ -12,7 +12,7 @@ import tempfile
|
|
|
12
12
|
import time
|
|
13
13
|
import uuid
|
|
14
14
|
from pathlib import Path
|
|
15
|
-
from subprocess import CalledProcessError, check_output
|
|
15
|
+
from subprocess import STDOUT, CalledProcessError, check_output
|
|
16
16
|
from zlib import crc32
|
|
17
17
|
from packaging.version import Version
|
|
18
18
|
|
|
@@ -324,12 +324,19 @@ def create_mdsplus_model(cache_dir_path: Path) -> None:
|
|
|
324
324
|
"Could not determine CompileTree class in jTraverser.jar."
|
|
325
325
|
)
|
|
326
326
|
compiletree = compiletree_class.rpartition(".class")[0].replace("/", ".")
|
|
327
|
+
# only add option to suppress warning if the JVM supports it
|
|
328
|
+
native_access = ["--enable-native-access=ALL-UNNAMED"]
|
|
329
|
+
try:
|
|
330
|
+
check_output(["java", *native_access, "-version"], stderr=STDOUT)
|
|
331
|
+
except CalledProcessError:
|
|
332
|
+
native_access = []
|
|
327
333
|
check_output(
|
|
328
334
|
[
|
|
329
335
|
"java",
|
|
330
336
|
"-Xms1g", # what do these do?
|
|
331
337
|
"-Xmx8g", # what do these do?
|
|
332
338
|
"-XX:+UseG1GC", # what do these do?
|
|
339
|
+
*native_access,
|
|
333
340
|
"-cp",
|
|
334
341
|
jarfile,
|
|
335
342
|
compiletree,
|
|
@@ -339,13 +346,14 @@ def create_mdsplus_model(cache_dir_path: Path) -> None:
|
|
|
339
346
|
env={
|
|
340
347
|
"PATH": os.environ.get("PATH", ""),
|
|
341
348
|
"LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", ""),
|
|
342
|
-
|
|
349
|
+
# MDSplus locates models through '<tree>_path' (tree='ids')
|
|
350
|
+
"ids_path": str(cache_dir_path),
|
|
343
351
|
},
|
|
344
352
|
)
|
|
345
353
|
# Touch a file to show that we have finished the model
|
|
346
354
|
(cache_dir_path / "done.txt").touch()
|
|
347
355
|
except CalledProcessError as e:
|
|
348
|
-
logger.error("Error making MDSPlus model in
|
|
356
|
+
logger.error("Error making MDSPlus model in %s", cache_dir_path)
|
|
349
357
|
raise e
|
|
350
358
|
|
|
351
359
|
|
|
@@ -92,6 +92,14 @@ class NCDBEntryImpl(DBEntryImpl):
|
|
|
92
92
|
)
|
|
93
93
|
self._dataset.close()
|
|
94
94
|
|
|
95
|
+
def _get_group(self, ids_name: str, occurrence: int) -> "netCDF4.Group":
|
|
96
|
+
try:
|
|
97
|
+
return self._dataset[f"{ids_name}/{occurrence}"]
|
|
98
|
+
except LookupError as exc:
|
|
99
|
+
raise DataEntryException(
|
|
100
|
+
f"IDS {ids_name!r}, occurrence {occurrence} is not found."
|
|
101
|
+
) from exc
|
|
102
|
+
|
|
95
103
|
def get(
|
|
96
104
|
self,
|
|
97
105
|
ids_name: str,
|
|
@@ -110,12 +118,7 @@ class NCDBEntryImpl(DBEntryImpl):
|
|
|
110
118
|
raise NotImplementedError(f"`{func}` is not available for netCDF files.")
|
|
111
119
|
|
|
112
120
|
# Check if the IDS/occurrence exists, and obtain the group it is stored in
|
|
113
|
-
|
|
114
|
-
group = self._dataset[f"{ids_name}/{occurrence}"]
|
|
115
|
-
except KeyError:
|
|
116
|
-
raise DataEntryException(
|
|
117
|
-
f"IDS {ids_name!r}, occurrence {occurrence} is not found."
|
|
118
|
-
)
|
|
121
|
+
group = self._get_group(ids_name, occurrence)
|
|
119
122
|
|
|
120
123
|
# Load data into the destination IDS
|
|
121
124
|
if self._ds_factory.dd_version == destination._dd_version:
|
|
@@ -183,3 +186,17 @@ class NCDBEntryImpl(DBEntryImpl):
|
|
|
183
186
|
|
|
184
187
|
occurrence_list.sort()
|
|
185
188
|
return occurrence_list
|
|
189
|
+
|
|
190
|
+
def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]:
|
|
191
|
+
# Check if the IDS/occurrence exists, and obtain the group it is stored in
|
|
192
|
+
group = self._get_group(ids_name, occurrence)
|
|
193
|
+
|
|
194
|
+
result = []
|
|
195
|
+
for name, variable in group.variables.items():
|
|
196
|
+
if variable.ndim == 0 and variable.dtype == "S1":
|
|
197
|
+
continue # (Array of) Structure metadata node, no data
|
|
198
|
+
if name.endswith(":shape"):
|
|
199
|
+
continue # Shape data, not a DD path
|
|
200
|
+
result.append(name.replace(".", "/"))
|
|
201
|
+
|
|
202
|
+
return result
|
imas/backends/netcdf/ids2nc.py
CHANGED
|
@@ -47,7 +47,6 @@ class IDS2NC(IDSTensorizer):
|
|
|
47
47
|
|
|
48
48
|
def create_variables(self) -> None:
|
|
49
49
|
"""Create netCDF variables."""
|
|
50
|
-
get_dimensions = self.ncmeta.get_dimensions
|
|
51
50
|
for path in self.filled_data:
|
|
52
51
|
metadata = self.ids.metadata[path]
|
|
53
52
|
var_name = path.replace("/", ".")
|
|
@@ -75,54 +74,21 @@ class IDS2NC(IDSTensorizer):
|
|
|
75
74
|
if dtype is not dtypes[IDSDataType.CPX]: # Set fillvalue
|
|
76
75
|
kwargs.update(fill_value=default_fillvals[metadata.data_type])
|
|
77
76
|
# Create variable
|
|
78
|
-
dimensions = get_dimensions(path
|
|
77
|
+
dimensions = self.get_dimensions(path)
|
|
79
78
|
var = self.group.createVariable(var_name, dtype, dimensions, **kwargs)
|
|
80
79
|
|
|
81
80
|
# Fill metadata attributes
|
|
82
|
-
var.
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if metadata.data_type is not IDSDataType.STRUCT_ARRAY:
|
|
95
|
-
coordinates = self.filter_coordinates(path)
|
|
96
|
-
if coordinates:
|
|
97
|
-
var.coordinates = coordinates
|
|
98
|
-
|
|
99
|
-
# Sparsity and :shape array
|
|
100
|
-
if path in self.shapes:
|
|
101
|
-
if not metadata.ndim:
|
|
102
|
-
# Doesn't need a :shape array:
|
|
103
|
-
var.sparse = "Sparse data, missing data is filled with _FillValue"
|
|
104
|
-
var.sparse += f" ({default_fillvals[metadata.data_type]})"
|
|
105
|
-
|
|
106
|
-
else:
|
|
107
|
-
shape_name = f"{var_name}:shape"
|
|
108
|
-
var.sparse = f"Sparse data, data shapes are stored in {shape_name}"
|
|
109
|
-
|
|
110
|
-
# Create variable to store data shape
|
|
111
|
-
dimensions = get_dimensions(
|
|
112
|
-
self.ncmeta.aos.get(path), self.homogeneous_time
|
|
113
|
-
) + (f"{metadata.ndim}D",)
|
|
114
|
-
shape_var = self.group.createVariable(
|
|
115
|
-
shape_name,
|
|
116
|
-
SHAPE_DTYPE,
|
|
117
|
-
dimensions,
|
|
118
|
-
)
|
|
119
|
-
doc_indices = ",".join(chr(ord("i") + i) for i in range(3))
|
|
120
|
-
shape_var.documentation = (
|
|
121
|
-
f"Shape information for {var_name}.\n"
|
|
122
|
-
f"{shape_name}[{doc_indices},:] describes the shape of filled "
|
|
123
|
-
f"data of {var_name}[{doc_indices},...]. Data outside this "
|
|
124
|
-
"shape is unset (i.e. filled with _Fillvalue)."
|
|
125
|
-
)
|
|
81
|
+
var.setncatts(self.get_attributes(path, default_fillvals))
|
|
82
|
+
|
|
83
|
+
# :shape array for sparse data
|
|
84
|
+
if path in self.shapes and metadata.ndim:
|
|
85
|
+
shape_name = f"{var_name}:shape"
|
|
86
|
+
# Create variable to store data shape
|
|
87
|
+
dimensions = self.get_shape_dimensions(path)
|
|
88
|
+
shape_var = self.group.createVariable(
|
|
89
|
+
shape_name, SHAPE_DTYPE, dimensions
|
|
90
|
+
)
|
|
91
|
+
shape_var.setncatts(self.get_shape_attributes(var_name))
|
|
126
92
|
|
|
127
93
|
def store_data(self) -> None:
|
|
128
94
|
"""Store data in the netCDF variables"""
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"""Tensorization logic to convert IDSs to netCDF files and/or xarray Datasets."""
|
|
4
4
|
|
|
5
5
|
from collections import deque
|
|
6
|
-
from typing import List
|
|
6
|
+
from typing import List, Tuple, Dict
|
|
7
7
|
|
|
8
8
|
import numpy
|
|
9
9
|
|
|
@@ -47,13 +47,26 @@ class IDSTensorizer:
|
|
|
47
47
|
"""Map of IDS paths to filled data nodes."""
|
|
48
48
|
self.filled_variables = set()
|
|
49
49
|
"""Set of filled IDS variables"""
|
|
50
|
-
self.homogeneous_time = (
|
|
50
|
+
self.homogeneous_time = bool(
|
|
51
51
|
ids.ids_properties.homogeneous_time == IDS_TIME_MODE_HOMOGENEOUS
|
|
52
52
|
)
|
|
53
53
|
"""True iff the IDS time mode is homogeneous."""
|
|
54
54
|
self.shapes = {}
|
|
55
55
|
"""Map of IDS paths to data shape arrays."""
|
|
56
56
|
|
|
57
|
+
def get_dimensions(self, path: str) -> Tuple[str, ...]:
|
|
58
|
+
"""Get the dimensions for a netCDF variable.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
path: Data Dictionary path to the variable, e.g. ``ids_properties/comment``.
|
|
62
|
+
"""
|
|
63
|
+
return self.ncmeta.get_dimensions(path, self.homogeneous_time)
|
|
64
|
+
|
|
65
|
+
def get_shape_dimensions(self, path: str) -> Tuple[str, ...]:
|
|
66
|
+
"""Get dimensions names for shape array of the tensorized variable"""
|
|
67
|
+
ndim = self.ids.metadata[path].ndim
|
|
68
|
+
return self.get_dimensions(self.ncmeta.aos.get(path, "")) + (f"{ndim}D",)
|
|
69
|
+
|
|
57
70
|
def include_coordinate_paths(self) -> None:
|
|
58
71
|
"""Append all paths that are coordinates of self.paths_to_tensorize"""
|
|
59
72
|
# Use a queue so we can also take coordinates of coordinates into account
|
|
@@ -62,7 +75,7 @@ class IDSTensorizer:
|
|
|
62
75
|
for path in self.paths_to_tensorize:
|
|
63
76
|
while path:
|
|
64
77
|
path, _, _ = path.rpartition("/")
|
|
65
|
-
if self.
|
|
78
|
+
if self.get_dimensions(path):
|
|
66
79
|
queue.append(path)
|
|
67
80
|
|
|
68
81
|
self.paths_to_tensorize = []
|
|
@@ -82,7 +95,6 @@ class IDSTensorizer:
|
|
|
82
95
|
# Initialize dictionary with all paths that could exist in this IDS
|
|
83
96
|
filled_data = {path: {} for path in self.ncmeta.paths}
|
|
84
97
|
dimension_size = {}
|
|
85
|
-
get_dimensions = self.ncmeta.get_dimensions
|
|
86
98
|
|
|
87
99
|
if self.paths_to_tensorize:
|
|
88
100
|
# Restrict tensorization to provided paths
|
|
@@ -102,7 +114,7 @@ class IDSTensorizer:
|
|
|
102
114
|
ndim = node.metadata.ndim
|
|
103
115
|
if not ndim:
|
|
104
116
|
continue
|
|
105
|
-
dimensions = get_dimensions(path
|
|
117
|
+
dimensions = self.get_dimensions(path)
|
|
106
118
|
# We're only interested in the non-tensorized dimensions: [-ndim:]
|
|
107
119
|
for dim_name, size in zip(dimensions[-ndim:], node.shape):
|
|
108
120
|
dimension_size[dim_name] = max(dimension_size.get(dim_name, 0), size)
|
|
@@ -115,15 +127,13 @@ class IDSTensorizer:
|
|
|
115
127
|
|
|
116
128
|
def determine_data_shapes(self) -> None:
|
|
117
129
|
"""Determine tensorized data shapes and sparsity, save in :attr:`shapes`."""
|
|
118
|
-
get_dimensions = self.ncmeta.get_dimensions
|
|
119
|
-
|
|
120
130
|
for path, nodes_dict in self.filled_data.items():
|
|
121
131
|
metadata = self.ids.metadata[path]
|
|
122
132
|
# Structures don't have a size
|
|
123
133
|
if metadata.data_type is IDSDataType.STRUCTURE:
|
|
124
134
|
continue
|
|
125
135
|
ndim = metadata.ndim
|
|
126
|
-
dimensions = get_dimensions(path
|
|
136
|
+
dimensions = self.get_dimensions(path)
|
|
127
137
|
|
|
128
138
|
# node shape if it is completely filled
|
|
129
139
|
full_shape = tuple(self.dimension_size[dim] for dim in dimensions[-ndim:])
|
|
@@ -137,7 +147,7 @@ class IDSTensorizer:
|
|
|
137
147
|
|
|
138
148
|
else:
|
|
139
149
|
# Data is tensorized, determine if it is homogeneously shaped
|
|
140
|
-
aos_dims = get_dimensions(self.ncmeta.aos[path]
|
|
150
|
+
aos_dims = self.get_dimensions(self.ncmeta.aos[path])
|
|
141
151
|
shapes_shape = [self.dimension_size[dim] for dim in aos_dims]
|
|
142
152
|
if ndim:
|
|
143
153
|
shapes_shape.append(ndim)
|
|
@@ -168,6 +178,55 @@ class IDSTensorizer:
|
|
|
168
178
|
if coordinate in self.filled_variables
|
|
169
179
|
)
|
|
170
180
|
|
|
181
|
+
def get_attributes(self, path: str, fillvals: dict) -> Dict[str, str]:
|
|
182
|
+
"""Get metadata attributes of the tensorized variable"""
|
|
183
|
+
metadata = self.ids.metadata[path]
|
|
184
|
+
var_name = path.replace("/", ".")
|
|
185
|
+
|
|
186
|
+
assert metadata.documentation is not None
|
|
187
|
+
attrs = {"documentation": metadata.documentation}
|
|
188
|
+
if metadata.units:
|
|
189
|
+
attrs["units"] = metadata.units
|
|
190
|
+
|
|
191
|
+
ancillary_variables = " ".join(
|
|
192
|
+
error_var
|
|
193
|
+
for error_var in [f"{var_name}_error_upper", f"{var_name}_error_lower"]
|
|
194
|
+
if error_var in self.filled_variables
|
|
195
|
+
)
|
|
196
|
+
if ancillary_variables:
|
|
197
|
+
attrs["ancillary_variables"] = ancillary_variables
|
|
198
|
+
|
|
199
|
+
if metadata.data_type is not IDSDataType.STRUCT_ARRAY:
|
|
200
|
+
coordinates = self.filter_coordinates(path)
|
|
201
|
+
if coordinates:
|
|
202
|
+
attrs["coordinates"] = coordinates
|
|
203
|
+
|
|
204
|
+
# Sparsity
|
|
205
|
+
if path in self.shapes:
|
|
206
|
+
if not metadata.ndim:
|
|
207
|
+
# Doesn't need a :shape array
|
|
208
|
+
attrs["sparse"] = (
|
|
209
|
+
"Sparse data, missing data is filled with _FillValue"
|
|
210
|
+
f" ({fillvals[metadata.data_type]})"
|
|
211
|
+
)
|
|
212
|
+
else:
|
|
213
|
+
attrs["sparse"] = (
|
|
214
|
+
f"Sparse data, data shapes are stored in {var_name}:shape"
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
return attrs
|
|
218
|
+
|
|
219
|
+
def get_shape_attributes(self, var_name: str) -> Dict[str, str]:
|
|
220
|
+
"""Get attributes of the :shape variable corresponding to var_name"""
|
|
221
|
+
doc_indices = ",".join(chr(ord("i") + i) for i in range(3))
|
|
222
|
+
documentation = (
|
|
223
|
+
f"Shape information for {var_name}.\n"
|
|
224
|
+
f"{var_name}:shape[{doc_indices},:] describes the shape of filled "
|
|
225
|
+
f"data of {var_name}[{doc_indices},...]. Data outside this "
|
|
226
|
+
"shape is unset (i.e. filled with _Fillvalue)."
|
|
227
|
+
)
|
|
228
|
+
return {"documentation": documentation}
|
|
229
|
+
|
|
171
230
|
def tensorize(self, path, fillvalue):
|
|
172
231
|
"""
|
|
173
232
|
Tensorizes the data at the given path with the specified fill value.
|
|
@@ -180,7 +239,7 @@ class IDSTensorizer:
|
|
|
180
239
|
Returns:
|
|
181
240
|
A tensor filled with the data from the specified path.
|
|
182
241
|
"""
|
|
183
|
-
dimensions = self.
|
|
242
|
+
dimensions = self.get_dimensions(path)
|
|
184
243
|
shape = tuple(self.dimension_size[dim] for dim in dimensions)
|
|
185
244
|
|
|
186
245
|
# TODO: depending on the data, tmp_var may be HUGE, we may need a more
|
imas/backends/netcdf/nc2ids.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
|
-
from typing import Optional
|
|
3
|
+
from typing import Optional, Tuple
|
|
4
4
|
|
|
5
5
|
import netCDF4
|
|
6
6
|
import numpy as np
|
|
@@ -80,6 +80,14 @@ class NC2IDS:
|
|
|
80
80
|
)
|
|
81
81
|
self.homogeneous_time = var[()] == IDS_TIME_MODE_HOMOGENEOUS
|
|
82
82
|
|
|
83
|
+
def get_dimensions(self, path: str) -> Tuple[str, ...]:
|
|
84
|
+
"""Get the dimensions for a netCDF variable.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
path: Data Dictionary path to the variable, e.g. ``ids_properties/comment``.
|
|
88
|
+
"""
|
|
89
|
+
return self.ncmeta.get_dimensions(path, self.homogeneous_time)
|
|
90
|
+
|
|
83
91
|
def run(self, lazy: bool) -> None:
|
|
84
92
|
"""Load the data from the netCDF group into the IDS."""
|
|
85
93
|
self.variables.sort()
|
|
@@ -130,9 +138,7 @@ class NC2IDS:
|
|
|
130
138
|
|
|
131
139
|
else:
|
|
132
140
|
# FIXME: extract dimension name from nc file?
|
|
133
|
-
dim = self.
|
|
134
|
-
metadata.path_string, self.homogeneous_time
|
|
135
|
-
)[-1]
|
|
141
|
+
dim = self.get_dimensions(metadata.path_string)[-1]
|
|
136
142
|
size = self.group.dimensions[dim].size
|
|
137
143
|
for _, node in indexed_tree_iter(self.ids, target_metadata):
|
|
138
144
|
node.resize(size)
|
|
@@ -235,9 +241,7 @@ class NC2IDS:
|
|
|
235
241
|
raise variable_error(var, "data type", var.dtype, expected_dtype)
|
|
236
242
|
|
|
237
243
|
# Dimensions
|
|
238
|
-
expected_dims = self.
|
|
239
|
-
metadata.path_string, self.homogeneous_time
|
|
240
|
-
)
|
|
244
|
+
expected_dims = self.get_dimensions(metadata.path_string)
|
|
241
245
|
if var.dimensions != expected_dims:
|
|
242
246
|
raise variable_error(var, "dimensions", var.dimensions, expected_dims)
|
|
243
247
|
|
|
@@ -298,9 +302,7 @@ class NC2IDS:
|
|
|
298
302
|
return # Sparsity is stored with _Fillvalue, nothing to validate
|
|
299
303
|
|
|
300
304
|
# Dimensions
|
|
301
|
-
aos_dimensions = self.ncmeta.
|
|
302
|
-
self.ncmeta.aos.get(metadata.path_string), self.homogeneous_time
|
|
303
|
-
)
|
|
305
|
+
aos_dimensions = self.get_dimensions(self.ncmeta.aos.get(metadata.path_string))
|
|
304
306
|
shape_dimensions = shape_var.dimensions
|
|
305
307
|
if (
|
|
306
308
|
len(shape_dimensions) != len(aos_dimensions) + 1
|
|
@@ -331,7 +333,6 @@ class LazyContext:
|
|
|
331
333
|
|
|
332
334
|
Args:
|
|
333
335
|
child: The child IDS node which should be lazy loaded.
|
|
334
|
-
|
|
335
336
|
"""
|
|
336
337
|
metadata = child.metadata
|
|
337
338
|
path = metadata.path_string
|
|
@@ -347,9 +348,7 @@ class LazyContext:
|
|
|
347
348
|
size = nc2ids.group[var.name + ":shape"][self.index][0]
|
|
348
349
|
else:
|
|
349
350
|
# FIXME: extract dimension name from nc file?
|
|
350
|
-
dim = nc2ids.
|
|
351
|
-
metadata.path_string, nc2ids.homogeneous_time
|
|
352
|
-
)[-1]
|
|
351
|
+
dim = nc2ids.get_dimensions(metadata.path_string)[-1]
|
|
353
352
|
size = nc2ids.group.dimensions[dim].size
|
|
354
353
|
|
|
355
354
|
child._set_lazy_context(LazyArrayStructContext(nc2ids, self.index, size))
|
imas/db_entry.py
CHANGED
|
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
9
|
import pathlib
|
|
10
|
-
from typing import Any, Type, overload
|
|
10
|
+
from typing import Any, Type, overload, List
|
|
11
11
|
|
|
12
12
|
import numpy
|
|
13
13
|
|
|
@@ -113,7 +113,10 @@ class DBEntry:
|
|
|
113
113
|
object before you can use it for reading or writing data.
|
|
114
114
|
|
|
115
115
|
Args:
|
|
116
|
-
uri: URI to the data entry
|
|
116
|
+
uri: URI to the data entry (a path to a netCDF file ending with .nc,
|
|
117
|
+
or an `IMAS URI for one of IMAS-Core's backends
|
|
118
|
+
<https://imas-core.readthedocs.io/en/latest/user_guide/uris_guide.html>`__
|
|
119
|
+
).
|
|
117
120
|
mode: Mode to open the Data Entry in:
|
|
118
121
|
|
|
119
122
|
- ``"r"``: Open an existing data entry. Raises an error when the data
|
|
@@ -197,14 +200,14 @@ class DBEntry:
|
|
|
197
200
|
from imas.backends.imas_core.db_entry_al import ALDBEntryImpl as impl
|
|
198
201
|
return impl
|
|
199
202
|
|
|
200
|
-
def __enter__(self):
|
|
203
|
+
def __enter__(self) -> "DBEntry":
|
|
201
204
|
# Context manager protocol
|
|
202
205
|
if self._dbe_impl is None:
|
|
203
206
|
# Open if the DBEntry was not already opened or created
|
|
204
207
|
self.open()
|
|
205
208
|
return self
|
|
206
209
|
|
|
207
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
|
210
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
|
208
211
|
# Context manager protocol
|
|
209
212
|
self.close()
|
|
210
213
|
|
|
@@ -800,3 +803,63 @@ class DBEntry:
|
|
|
800
803
|
self.get(ids_name, occ, lazy=True)[node_path] for occ in occurrence_list
|
|
801
804
|
]
|
|
802
805
|
return occurrence_list, node_content_list
|
|
806
|
+
|
|
807
|
+
def list_filled_paths(
|
|
808
|
+
self, ids_name, occurrence: int = 0, *, autoconvert: bool = True
|
|
809
|
+
) -> List[str]:
|
|
810
|
+
"""Get a list of filled Data Dictionary paths from the backend.
|
|
811
|
+
|
|
812
|
+
Note that this is only supported by some backends (HDF5 and netCDF), and will
|
|
813
|
+
result in an error on unsupported backends.
|
|
814
|
+
|
|
815
|
+
Args:
|
|
816
|
+
ids_name: Name of the IDS to request filled data for.
|
|
817
|
+
occurrence: Occurrence number of the IDS to request filled data for.
|
|
818
|
+
|
|
819
|
+
Keyword Args:
|
|
820
|
+
autoconvert: If enabled (default), this method will take NBC renames into
|
|
821
|
+
account in the returned list of filled paths. This argument corresponds
|
|
822
|
+
to the :py:data:`~get.autoconvert` argument of :py:meth:`get`.
|
|
823
|
+
|
|
824
|
+
Returns:
|
|
825
|
+
List of paths which have some data filled in the backend. For example, when
|
|
826
|
+
``profiles_1d/ion/temperature`` is in this list, it means that there is at
|
|
827
|
+
least one ``ion`` in one ``profiles_1d`` entry for which the temperature is
|
|
828
|
+
filled.
|
|
829
|
+
|
|
830
|
+
The paths in this list may be ordered arbitrarily.
|
|
831
|
+
|
|
832
|
+
Example:
|
|
833
|
+
>>> with imas.DBEntry("imas:hdf5?path=./path/to/data", "r") as entry:
|
|
834
|
+
>>> print(entry.list_filled_paths("core_profiles"))
|
|
835
|
+
['ids_properties/comment', 'ids_properties/homogeneous_time',
|
|
836
|
+
'profiles_1d/grid/rho_tor_norm', 'profiles_1d/electrons/temperature',
|
|
837
|
+
'profiles_1d/ion/temperature', 'time']
|
|
838
|
+
"""
|
|
839
|
+
if self._dbe_impl is None:
|
|
840
|
+
raise RuntimeError("Database entry is not open.")
|
|
841
|
+
paths = self._dbe_impl.list_filled_paths(ids_name, occurrence)
|
|
842
|
+
if not autoconvert:
|
|
843
|
+
return paths
|
|
844
|
+
|
|
845
|
+
# DD conversion?
|
|
846
|
+
dd_version = self._dbe_impl.read_dd_version(ids_name, occurrence)
|
|
847
|
+
if dd_version == self._ids_factory.dd_version:
|
|
848
|
+
return paths # No conversion required
|
|
849
|
+
|
|
850
|
+
# Follow any NBC renames:
|
|
851
|
+
ddmap, source_is_older = dd_version_map_from_factories(
|
|
852
|
+
ids_name, IDSFactory(version=dd_version), self._ids_factory
|
|
853
|
+
)
|
|
854
|
+
nbc_map = ddmap.old_to_new if source_is_older else ddmap.new_to_old
|
|
855
|
+
|
|
856
|
+
converted_paths = []
|
|
857
|
+
for path in paths:
|
|
858
|
+
if path in nbc_map:
|
|
859
|
+
new_name = nbc_map.path[path]
|
|
860
|
+
if new_name is not None:
|
|
861
|
+
converted_paths.append(new_name)
|
|
862
|
+
else:
|
|
863
|
+
converted_paths.append(path)
|
|
864
|
+
|
|
865
|
+
return converted_paths
|