imas-python 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

imas/__init__.py

@@ -20,7 +20,6 @@ from .ids_identifiers import identifiers
 # Load the IMAS-Python IMAS AL/DD core
 from . import (
     db_entry,
-    dd_helpers,
     dd_zip,
     util,
 )

imas/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.0.0'
-__version_tuple__ = version_tuple = (2, 0, 0)
+__version__ = version = '2.0.1'
+__version_tuple__ = version_tuple = (2, 0, 1)

imas/backends/imas_core/al_context.py

@@ -1,7 +1,6 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Object-oriented interface to the IMAS lowlevel.
-"""
+"""Object-oriented interface to the IMAS lowlevel."""
 
 import logging
 import weakref
@@ -61,17 +60,21 @@ class ALContext:
     def __exit__(self, exc_type, exc_value, traceback) -> None:
         ll_interface.end_action(self.ctx)
 
-    def global_action(self, path: str, rwmode: int) -> "ALContext":
+    def global_action(self, path: str, rwmode: int, datapath: str = "") -> "ALContext":
         """Begin a new global action for use in a ``with`` context.
 
         Args:
             path: access layer path for this global action: ``<idsname>[/<occurrence>]``
             rwmode: read-only or read-write operation mode: ``READ_OP``/``WRITE_OP``
+            datapath: used by UDA backend to fetch only part of the data.
 
         Returns:
             The created context.
         """
-        status, ctx = ll_interface.begin_global_action(self.ctx, path, rwmode)
+        args = [self.ctx, path, rwmode]
+        if datapath:  # AL4 compatibility: datapath arg was added in AL5
+            args.append(datapath)
+        status, ctx = ll_interface.begin_global_action(*args)
         if status != 0:
             raise LowlevelError("global_action", status)
         return ALContext(ctx)

imas/backends/imas_core/db_entry_al.py

@@ -257,7 +257,8 @@ class ALDBEntryImpl(DBEntryImpl):
         if occurrence != 0:
             ll_path += f"/{occurrence}"
 
-        with self._db_ctx.global_action(ll_path, READ_OP) as read_ctx:
+        datapath = "ids_properties" if self.backend == "uda" else ""
+        with self._db_ctx.global_action(ll_path, READ_OP, datapath) as read_ctx:
             time_mode_path = "ids_properties/homogeneous_time"
             time_mode = read_ctx.read_data(time_mode_path, "", INTEGER_DATA, 0)
             # This is already checked by read_dd_version, but ensure:
@@ -314,7 +315,8 @@ class ALDBEntryImpl(DBEntryImpl):
         if occurrence != 0:
             ll_path += f"/{occurrence}"
 
-        with self._db_ctx.global_action(ll_path, READ_OP) as read_ctx:
+        datapath = "ids_properties" if self.backend == "uda" else ""
+        with self._db_ctx.global_action(ll_path, READ_OP, datapath) as read_ctx:
             time_mode_path = "ids_properties/homogeneous_time"
             time_mode = read_ctx.read_data(time_mode_path, "", INTEGER_DATA, 0)
             dd_version_path = "ids_properties/version_put/data_dictionary"

imas/backends/imas_core/mdsplus_model.py

@@ -12,7 +12,6 @@ import tempfile
 import time
 import uuid
 from pathlib import Path
-from saxonche import PySaxonProcessor
 from subprocess import CalledProcessError, check_output
 from zlib import crc32
 
@@ -244,11 +243,20 @@ def transform_with_xslt(xslt_processor, source, xslfile, output_file):
 
 def create_model_ids_xml(cache_dir_path, fname, version):
     """Use Saxon/C to compile an ids.xml suitable for creating an MDSplus model."""
+    try:
+        import saxonche
+    except ImportError:
+        raise RuntimeError(
+            "Building mdsplus models requires the 'saxonche' python package. "
+            "Please install this package (for example with 'pip install saxonche') "
+            "and try again."
+        )
+
     try:
         with as_file(files("imas") / "assets" / "IDSDef2MDSpreTree.xsl") as xslfile:
             output_file = Path(cache_dir_path) / "ids.xml"
 
-            with PySaxonProcessor(license=False) as proc:
+            with saxonche.PySaxonProcessor(license=False) as proc:
                 xslt_processor = proc.new_xslt30_processor()
                 xdm_ddgit = proc.make_string_value(str(version) or fname)
                 xslt_processor.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)

imas/dd_zip.py

@@ -1,103 +1,27 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-""" Extract DD versions from a zip file.
+"""Extract DD versions from the imas-data-dictionaries distribution."""
 
-The zip file contains files as
-* `data-dictionary/3.30.0.xml`
-* `data-dictionary/3.29.0.xml`
-
-multiple paths are checked. See `ZIPFILE_LOCATIONS`.
-First the environment variable IMAS_DDZIP is checked.
-If that exists and points to a file we will attempt to open it.
-Then, IDSDef.zip is searched in site-packages, the current folder,
-in .config/imas/ (`$$XDG_CONFIG_HOME`) and in
-the assets/ folder within the IMAS-Python package.
-
-1. `$$IMAS_DDZIP`
-2. The virtual environment
-3. USER_BASE`imas/IDSDef.zip`
-4. All `site-packages/imas/IDSDef.zip`
-5. `./IDSDef.zip`
-6. `~/.config/imas/IDSDef.zip`
-7. `__file__/../../imas/assets/IDSDef.zip`
-
-All files are checked, i.e. if your .config/imas/IDSDef.zip is outdated
-the IMAS-Python-packaged version will be used.
-
-The `assets/IDSDef.zip` provided with the package can be updated
-with the `python setup.py build_DD` command, which is also performed on install
-if you have access to the ITER data-dictionary git repo.
-Reinstalling imas thus also will give you access to the latest DD versions.
-"""
 import logging
 import os
-import re
 import xml.etree.ElementTree as ET
-from contextlib import contextmanager, nullcontext
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, Iterator, List, Tuple, Union
-from zipfile import ZipFile
-
-try:
-    from importlib.resources import as_file, files
-
-    try:
-        from importlib.resources.abc import Traversable
-    except ModuleNotFoundError:  # Python 3.9/3.10 support
-        from importlib.abc import Traversable
-
-except ImportError:  # Python 3.8 support
-    from importlib_resources import as_file, files
-    from importlib_resources.abc import Traversable
 
-from packaging.version import InvalidVersion, Version
+# These methods in imas_data_dictionaries used to be defined here. We import them here
+# for backwards compatibility:
+from imas_data_dictionaries import dd_identifiers  # noqa: F401
+from imas_data_dictionaries import get_dd_xml_crc  # noqa: F401
+from imas_data_dictionaries import get_identifier_xml  # noqa: F401
+from imas_data_dictionaries import dd_xml_versions, get_dd_xml, parse_dd_version
+from packaging.version import InvalidVersion
 
 import imas
-from imas.exception import UnknownDDVersion
+from imas.exception import UnknownDDVersion  # noqa: F401
 
 logger = logging.getLogger(__name__)
 
 
-def _get_xdg_config_dir():
-    """
-    Return the XDG config directory, according to the XDG base directory spec:
-
-    https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
-    """
-    return os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
-
-
-def _generate_zipfile_locations() -> Iterator[Union[Path, Traversable]]:
-    """Build a list of potential data dictionary locations.
-    We start with the path (if any) of the IMAS_DDZIP env var.
-    Then we look for IDSDef.zip in the current folder, in the
-    default XDG config dir (~/.config/imas/IDSDef.zip) and
-    finally in the assets distributed with this package.
-    """
-    zip_name = "IDSDef.zip"
-
-    environ = os.environ.get("IMAS_DDZIP")
-    if environ:
-        yield Path(environ).resolve()
-
-    yield Path(zip_name).resolve()
-    yield Path(_get_xdg_config_dir()).resolve() / "imas" / zip_name
-    yield files(imas) / "assets" / zip_name
-
-
-def parse_dd_version(version: str) -> Version:
-    try:
-        return Version(version)
-    except InvalidVersion:
-        # This is probably a dev build of the DD, of which the version is obtained with
-        # `git describe` in the format X.Y.Z-<ncommits>-g<hash> with X.Y.Z the previous
-        # released version: try again after converting the first dash to a + and treat
-        # it like a `local` version specifier, which is recognized as newer.
-        # https://packaging.python.org/en/latest/specifications/version-specifiers/
-        return Version(version.replace("-", "+", 1))
-
-
 # Expected use case is one, maximum two DD versions
 # Cache is bigger than that: in pytest we currently use the following DD versions:
 #   - 3.22.0
@@ -112,7 +36,6 @@ def parse_dd_version(version: str) -> Version:
 #   - IDS_minimal_struct_array.xml
 #   - IDS_minimal_types.xml
 _DD_CACHE_SIZE = 8
-ZIPFILE_LOCATIONS = list(_generate_zipfile_locations())
 
 
 def dd_etree(version=None, xml_path=None):
@@ -168,117 +91,6 @@ def _load_etree(version, xml_path):
     return tree
 
 
-@contextmanager
-def _open_zipfile(path: Union[Path, Traversable]) -> Iterator[ZipFile]:
-    """Open a zipfile, given a Path or Traversable."""
-    if isinstance(path, Path):
-        ctx = nullcontext(path)
-    else:
-        ctx = as_file(path)
-    with ctx as file:
-        with ZipFile(file) as zipfile:
-            yield zipfile
-
-
-@lru_cache
-def _read_dd_versions() -> Dict[str, Tuple[Union[Path, Traversable], str]]:
-    """Traverse all possible DD zip files and return a map of known versions.
-
-    Returns:
-        version_map: version -> (zipfile path, filename)
-    """
-    versions = {}
-    xml_re = re.compile(r"^data-dictionary/([0-9.]+)\.xml$")
-    for path in ZIPFILE_LOCATIONS:
-        if not path.is_file():
-            continue
-        with _open_zipfile(path) as zipfile:
-            for fname in zipfile.namelist():
-                match = xml_re.match(fname)
-                if match:
-                    version = match.group(1)
-                    if version not in versions:
-                        versions[version] = (path, fname)
-    if not versions:
-        raise RuntimeError(
-            "Could not find any data dictionary definitions. "
-            f"Looked in: {', '.join(map(repr, ZIPFILE_LOCATIONS))}."
-        )
-    return versions
-
-
-@lru_cache
-def _read_identifiers() -> Dict[str, Tuple[Union[Path, Traversable], str]]:
-    """Traverse all possible DD zip files and return a map of known identifiers.
-
-    Returns:
-        identifier_map: identifier -> (zipfile path, filename)
-    """
-    identifiers = {}
-    xml_re = re.compile(r"^identifiers/\w+/(\w+_identifier).xml$")
-    for path in ZIPFILE_LOCATIONS:
-        if not path.is_file():
-            continue
-        with _open_zipfile(path) as zipfile:
-            for fname in zipfile.namelist():
-                match = xml_re.match(fname)
-                if match:
-                    identifier_name = match.group(1)
-                    if identifier_name not in identifiers:
-                        identifiers[identifier_name] = (path, fname)
-    return identifiers
-
-
-@lru_cache
-def dd_xml_versions() -> List[str]:
-    """Parse IDSDef.zip to find version numbers available"""
-
-    def sort_key(version):
-        try:
-            return parse_dd_version(version)
-        except InvalidVersion:
-            # Don't fail when a malformatted version is present in the DD zip
-            logger.error(
-                f"Could not convert DD XML version {version} to a Version.", exc_info=1
-            )
-            return Version(0)
-
-    return sorted(_read_dd_versions(), key=sort_key)
-
-
-@lru_cache
-def dd_identifiers() -> List[str]:
-    """Parse IDSDef.zip to find available identifiers"""
-
-    return sorted(_read_identifiers())
-
-
-def get_dd_xml(version):
-    """Read XML file for the given data dictionary version."""
-    dd_versions = dd_xml_versions()
-    if version not in dd_versions:
-        raise UnknownDDVersion(version, dd_versions)
-    path, fname = _read_dd_versions()[version]
-    with _open_zipfile(path) as zipfile:
-        return zipfile.read(fname)
-
-
-def get_dd_xml_crc(version):
-    """Given a version string, return its CRC checksum"""
-    # Note, by this time get_dd_xml is already called, so we don't need to check if the
-    # version is known
-    path, fname = _read_dd_versions()[version]
-    with _open_zipfile(path) as zipfile:
-        return zipfile.getinfo(fname).CRC
-
-
-def get_identifier_xml(identifier_name):
-    """Get identifier XML for the given identifier name"""
-    path, fname = _read_identifiers()[identifier_name]
-    with _open_zipfile(path) as zipfile:
-        return zipfile.read(fname)
-
-
 def print_supported_version_warning(version):
     try:
         if parse_dd_version(version) < imas.OLDEST_SUPPORTED_VERSION:

imas/exception.py

@@ -1,11 +1,14 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Exception classes used in IMAS-Python.
-"""
+"""Exception classes used in IMAS-Python."""
 
 import difflib
 import logging
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
+
+# This exception from imas_data_dictionaries used to be defined here. We import it here
+# for backwards compatibility:
+from imas_data_dictionaries import UnknownDDVersion  # noqa: F401
 
 from imas.backends.imas_core import imas_interface as _imas_interface
 
@@ -23,20 +26,6 @@ else:
     ALException = None
 
 
-class UnknownDDVersion(ValueError):
-    """Error raised when an unknown DD version is specified."""
-
-    def __init__(self, version: str, available: List[str], note: str = "") -> None:
-        close_matches = difflib.get_close_matches(version, available, n=1)
-        if close_matches:
-            suggestions = f"Did you mean {close_matches[0]!r}?"
-        else:
-            suggestions = f"Available versions are {', '.join(reversed(available))}"
-        super().__init__(
-            f"Data dictionary version {version!r} cannot be found. {suggestions}{note}"
-        )
-
-
 class IDSNameError(ValueError):
     """Error raised by DBEntry.get(_slice) when providing an invalid IDS name."""
 

imas/ids_convert.py

@@ -1,36 +1,35 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Functionality for converting IDSToplevels between DD versions.
-"""
+"""Functionality for converting IDSToplevels between DD versions."""
 
 import copy
 import datetime
 import logging
-from functools import lru_cache
+from functools import lru_cache, partial
 from pathlib import Path
 from typing import Callable, Dict, Iterator, Optional, Set, Tuple
 from xml.etree.ElementTree import Element, ElementTree
 
 import numpy
 from packaging.version import InvalidVersion, Version
+from scipy.interpolate import interp1d
 
 import imas
 from imas.dd_zip import parse_dd_version
 from imas.ids_base import IDSBase
 from imas.ids_data_type import IDSDataType
+from imas.ids_defs import IDS_TIME_MODE_HETEROGENEOUS
 from imas.ids_factory import IDSFactory
 from imas.ids_path import IDSPath
-from imas.ids_primitive import (
-    IDSNumeric0D,
-    IDSNumericArray,
-    IDSPrimitive,
-    IDSString0D,
-)
+from imas.ids_primitive import IDSNumeric0D, IDSNumericArray, IDSPrimitive, IDSString0D
 from imas.ids_struct_array import IDSStructArray
 from imas.ids_structure import IDSStructure
 from imas.ids_toplevel import IDSToplevel
 
 logger = logging.getLogger(__name__)
+# Store for which paths we already emitted a warning that the target could not be found
+# to prevent polluting the output with lots of repeated items.
+_missing_paths_warning = set()
 
 
 def iter_parents(path: str) -> Iterator[str]:
@@ -334,12 +333,13 @@ class DDVersionMap:
         # Additional conversion rules for DDv3 to DDv4
         if self.version_old.major == 3 and new_version and new_version.major == 4:
             # Postprocessing for COCOS definition change:
-            xpath_query = ".//field[@cocos_label_transformation='psi_like']"
-            for old_item in old.iterfind(xpath_query):
-                old_path = old_item.get("path")
-                new_path = self.old_to_new.path.get(old_path, old_path)
-                self.new_to_old.post_process[new_path] = _cocos_change
-                self.old_to_new.post_process[old_path] = _cocos_change
+            for psi_like in ["psi_like", "dodpsi_like"]:
+                xpath_query = f".//field[@cocos_label_transformation='{psi_like}']"
+                for old_item in old.iterfind(xpath_query):
+                    old_path = old_item.get("path")
+                    new_path = self.old_to_new.path.get(old_path, old_path)
+                    self.new_to_old.post_process[new_path] = _cocos_change
+                    self.old_to_new.post_process[old_path] = _cocos_change
             # Definition change for pf_active circuit/connections
             if self.ids_name == "pf_active":
                 path = "circuit/connections"
@@ -474,32 +474,48 @@ def convert_ids(
             raise RuntimeError(
                 f"There is no IDS with name {ids_name} in DD version {version}."
             )
-        target_ids = factory.new(ids_name)
-    else:
-        target_ids = target
+        target = factory.new(ids_name)
 
     source_version = parse_dd_version(toplevel._version)
-    target_version = parse_dd_version(target_ids._version)
+    target_version = parse_dd_version(target._version)
     logger.info(
         "Starting conversion of IDS %s from version %s to version %s.",
         ids_name,
         source_version,
         target_version,
     )
+    global _missing_paths_warning
+    _missing_paths_warning = set()  # clear for which paths we emitted a warning
 
-    source_is_new = source_version > target_version
     source_tree = toplevel._parent._etree
-    target_tree = target_ids._parent._etree
-    if source_is_new:
+    target_tree = target._parent._etree
+    if source_version > target_version:
         version_map = _DDVersionMap(ids_name, target_tree, source_tree, target_version)
+        rename_map = version_map.new_to_old
     else:
         version_map = _DDVersionMap(ids_name, source_tree, target_tree, source_version)
+        rename_map = version_map.old_to_new
+
+    # Special case for DD3to4 pulse_schedule conversion
+    if (
+        toplevel.metadata.name == "pulse_schedule"
+        and toplevel.ids_properties.homogeneous_time == IDS_TIME_MODE_HETEROGENEOUS
+        and source_version < Version("3.40.0")
+        and target_version >= Version("3.40.0")
+    ):
+        try:
+            # Suppress "'.../time' does not exist in the target IDS." log messages.
+            logger.addFilter(_pulse_schedule_3to4_logfilter)
+            _pulse_schedule_3to4(toplevel, target, deepcopy, rename_map)
+        finally:
+            logger.removeFilter(_pulse_schedule_3to4_logfilter)
+    else:
+        _copy_structure(toplevel, target, deepcopy, rename_map)
 
-    _copy_structure(toplevel, target_ids, deepcopy, source_is_new, version_map)
     logger.info("Conversion of IDS %s finished.", ids_name)
     if provenance_origin_uri:
-        _add_provenance_entry(target_ids, toplevel._version, provenance_origin_uri)
-    return target_ids
+        _add_provenance_entry(target, toplevel._version, provenance_origin_uri)
+    return target
 
 
 def _add_provenance_entry(
@@ -541,12 +557,50 @@ def _add_provenance_entry(
         node.sources.append(source_txt)  # sources is a STR_1D (=list of strings)
 
 
+def _get_target_item(
+    item: IDSBase, target: IDSStructure, rename_map: NBCPathMap
+) -> Optional[IDSBase]:
+    """Find and return the corresponding target item if it exists.
+
+    This method follows NBC renames (as stored in the rename map). It returns None if
+    there is no corresponding target item in the target structure.
+    """
+    path = item.metadata.path_string
+
+    # Follow NBC renames:
+    if path in rename_map:
+        if rename_map.path[path] is None:
+            if path not in rename_map.ignore_missing_paths:
+                # Only warn the first time that we encounter this path:
+                if path not in _missing_paths_warning:
+                    if path in rename_map.type_change:
+                        msg = "Element %r changed type in the target IDS."
+                    else:
+                        msg = "Element %r does not exist in the target IDS."
+                    logger.warning(msg + " Data is not copied.", path)
+                    _missing_paths_warning.add(path)
+            return None
+        else:
+            return IDSPath(rename_map.path[path]).goto(target)
+
+    # No NBC renames:
+    try:
+        return target[item.metadata.name]
+    except AttributeError:
+        # In exceptional cases the item does not exist in the target. Example:
+        # neutron_diagnostic IDS between DD 3.40.1 and 3.41.0. has renamed
+        # synthetic_signals/fusion_power -> fusion_power. The synthetic_signals
+        # structure no longer exists but we need to descend into it to get the
+        # total_neutron_flux.
+        return target
+
+
 def _copy_structure(
     source: IDSStructure,
     target: IDSStructure,
     deepcopy: bool,
-    source_is_new: bool,
-    version_map: DDVersionMap,
+    rename_map: NBCPathMap,
+    callback: Optional[Callable] = None,
 ):
     """Recursively copy data, following NBC renames.
 
@@ -557,31 +611,14 @@ def _copy_structure(
         source_is_new: True iff the DD version of the source is newer than that of the
             target.
         version_map: Version map containing NBC renames.
+        callback: Optional callback that is called for every copied node.
     """
-    rename_map = version_map.new_to_old if source_is_new else version_map.old_to_new
     for item in source.iter_nonempty_():
         path = item.metadata.path_string
-        if path in rename_map:
-            if rename_map.path[path] is None:
-                if path not in rename_map.ignore_missing_paths:
-                    if path in rename_map.type_change:
-                        msg = "Element %r changed type in the target IDS."
-                    else:
-                        msg = "Element %r does not exist in the target IDS."
-                    logger.warning(msg + " Data is not copied.", path)
-                continue
-            else:
-                target_item = IDSPath(rename_map.path[path]).goto(target)
-        else:
-            try:
-                target_item = target[item.metadata.name]
-            except AttributeError:
-                # In exceptional cases the item does not exist in the target. Example:
-                # neutron_diagnostic IDS between DD 3.40.1 and 3.41.0. has renamed
-                # synthetic_signals/fusion_power -> fusion_power. The synthetic_signals
-                # structure no longer exists but we need to descend into it to get the
-                # total_neutron_flux.
-                target_item = target
+        target_item = _get_target_item(item, target, rename_map)
+        if target_item is None:
+            continue
+
         if path in rename_map.type_change:
             # Handle type change
             new_items = rename_map.type_change[path](item, target_item)
@@ -594,21 +631,17 @@ def _copy_structure(
             size = len(item)
             target_item.resize(size)
             for i in range(size):
-                _copy_structure(
-                    item[i], target_item[i], deepcopy, source_is_new, version_map
-                )
+                _copy_structure(item[i], target_item[i], deepcopy, rename_map, callback)
         elif isinstance(item, IDSStructure):
-            _copy_structure(item, target_item, deepcopy, source_is_new, version_map)
+            _copy_structure(item, target_item, deepcopy, rename_map, callback)
         else:
-            if deepcopy:
-                # No nested types are used as data, so a shallow copy is sufficient
-                target_item.value = copy.copy(item.value)
-            else:
-                target_item.value = item.value
+            target_item.value = copy.copy(item.value) if deepcopy else item.value
 
         # Post-process the node:
         if path in rename_map.post_process:
             rename_map.post_process[path](target_item)
+        if callback is not None:
+            callback(item, target_item)
 
 
 ########################################################################################
@@ -919,3 +952,82 @@ def _ids_properties_source(source: IDSString0D, provenance: IDSStructure) -> Non
     provenance.node.resize(1)
     provenance.node[0].reference.resize(1)
     provenance.node[0].reference[0].name = source.value
+
+
+def _pulse_schedule_3to4(
+    source: IDSStructure,
+    target: IDSStructure,
+    deepcopy: bool,
+    rename_map: NBCPathMap,
+):
+    """Recursively copy data, following NBC renames, and converting time bases for the
+    pulse_schedule IDS.
+
+    Args:
+        source: Source structure.
+        target: Target structure.
+        deepcopy: See :func:`convert_ids`.
+        rename_map: Map containing NBC renames.
+    """
+    # All prerequisites are checked before calling this function:
+    # - source and target are pulse_schedule IDSs
+    # - source has DD version < 3.40.0
+    # - target has DD version >= 4.0.0, < 5.0
+    # - IDS is using heterogeneous time
+
+    for item in source.iter_nonempty_():
+        name = item.metadata.name
+        target_item = _get_target_item(item, target, rename_map)
+        if target_item is None:
+            continue
+
+        # Special cases for non-dynamic stuff
+        if name in ["ids_properties", "code"]:
+            _copy_structure(item, target_item, deepcopy, rename_map)
+        elif name == "time":
+            target_item.value = item.value if not deepcopy else copy.copy(item.value)
+        elif name == "event":
+            size = len(item)
+            target_item.resize(size)
+            for i in range(size):
+                _copy_structure(item[i], target_item[i], deepcopy, rename_map)
+        else:
+            # Find all time bases
+            time_bases = [
+                node.value
+                for node in imas.util.tree_iter(item)
+                if node.metadata.name == "time"
+            ]
+            # Construct the common time base
+            timebase = numpy.unique(numpy.concatenate(time_bases)) if time_bases else []
+            target_item.time = timebase
+            # Do the conversion
+            callback = partial(_pulse_schedule_resample_callback, timebase)
+            _copy_structure(item, target_item, deepcopy, rename_map, callback)
+
+
+def _pulse_schedule_3to4_logfilter(logrecord: logging.LogRecord) -> bool:
+    """Suppress "'.../time' does not exist in the target IDS." log messages."""
+    return not (logrecord.args and str(logrecord.args[0]).endswith("/time"))
+
+
+def _pulse_schedule_resample_callback(timebase, item: IDSBase, target_item: IDSBase):
+    """Callback from _copy_structure to resample dynamic data on the new timebase"""
+    if item.metadata.ndim == 1 and item.metadata.coordinates[0].is_time_coordinate:
+        # Interpolate 1D dynamic quantities to the common time base
+        time = item.coordinates[0]
+        if len(item) != len(time):
+            raise ValueError(
+                f"Array {item} has a different size than its time base {time}."
+            )
+        is_integer = item.metadata.data_type is IDSDataType.INT
+        value = interp1d(
+            time.value,
+            item.value,
+            "previous" if is_integer else "linear",
+            copy=False,
+            bounds_error=False,
+            fill_value=(item[0], item[-1]),
+            assume_sorted=True,
+        )(timebase)
+        target_item.value = value.astype(numpy.int32) if is_integer else value

imas/test/test_ids_convert.py

@@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
 import numpy
+from numpy import array_equal
 import pytest
 
 from imas import identifiers
@@ -27,7 +28,7 @@ from imas.ids_defs import (
 from imas.ids_factory import IDSFactory
 from imas.ids_struct_array import IDSStructArray
 from imas.ids_structure import IDSStructure
-from imas.test.test_helpers import compare_children, open_dbentry
+from imas.test.test_helpers import compare_children, fill_consistent, open_dbentry
 
 UTC = timezone.utc
 
@@ -287,22 +288,22 @@ def test_3to4_repeat_children_first_point_conditional(dd4factory):
     for i in range(2):
         outline_inner = wall4.description_2d[0].vessel.unit[i].annular.outline_inner
         if i == 0:  # open outline, first point not repeated:
-            assert numpy.array_equal(outline_inner.r, [1.0, 2.0, 3.0])
-            assert numpy.array_equal(outline_inner.z, [-1.0, -2.0, -3.0])
+            assert array_equal(outline_inner.r, [1.0, 2.0, 3.0])
+            assert array_equal(outline_inner.z, [-1.0, -2.0, -3.0])
         else:  # closed outline, first point repeated:
-            assert numpy.array_equal(outline_inner.r, [1.0, 2.0, 3.0, 1.0])
-            assert numpy.array_equal(outline_inner.z, [-1.0, -2.0, -3.0, -1.0])
+            assert array_equal(outline_inner.r, [1.0, 2.0, 3.0, 1.0])
+            assert array_equal(outline_inner.z, [-1.0, -2.0, -3.0, -1.0])
 
     # Test conversion for case 2:
     assert len(wall4.description_2d[0].limiter.unit) == 2
     for i in range(2):
         unit = wall4.description_2d[0].limiter.unit[i]
         if i == 0:  # open outline, first point not repeated:
-            assert numpy.array_equal(unit.outline.r, [1.0, 2.0, 3.0])
-            assert numpy.array_equal(unit.outline.z, [-1.0, -2.0, -3.0])
+            assert array_equal(unit.outline.r, [1.0, 2.0, 3.0])
+            assert array_equal(unit.outline.z, [-1.0, -2.0, -3.0])
         else:  # closed outline, first point repeated:
-            assert numpy.array_equal(unit.outline.r, [1.0, 2.0, 3.0, 1.0])
-            assert numpy.array_equal(unit.outline.z, [-1.0, -2.0, -3.0, -1.0])
+            assert array_equal(unit.outline.r, [1.0, 2.0, 3.0, 1.0])
+            assert array_equal(unit.outline.z, [-1.0, -2.0, -3.0, -1.0])
 
     # Test conversion for case 3:
     assert len(wall4.description_2d[0].mobile.unit) == 2
@@ -310,11 +311,11 @@ def test_3to4_repeat_children_first_point_conditional(dd4factory):
         unit = wall4.description_2d[0].mobile.unit[i]
         for j in range(3):
             if i == 0:  # open outline, first point not repeated:
-                assert numpy.array_equal(unit.outline[j].r, [1.0, 2.0, 3.0])
-                assert numpy.array_equal(unit.outline[j].z, [-1.0, -2.0, -3.0])
+                assert array_equal(unit.outline[j].r, [1.0, 2.0, 3.0])
+                assert array_equal(unit.outline[j].z, [-1.0, -2.0, -3.0])
             else:  # closed outline, first point repeated:
-                assert numpy.array_equal(unit.outline[j].r, [1.0, 2.0, 3.0, 1.0])
-                assert numpy.array_equal(unit.outline[j].z, [-1.0, -2.0, -3.0, -1.0])
+                assert array_equal(unit.outline[j].r, [1.0, 2.0, 3.0, 1.0])
+                assert array_equal(unit.outline[j].z, [-1.0, -2.0, -3.0, -1.0])
             assert unit.outline[j].time == pytest.approx(j / 5)
 
     # Test conversion for case 4:
@@ -322,9 +323,9 @@ def test_3to4_repeat_children_first_point_conditional(dd4factory):
     for i in range(2):
         thickness = wall4.description_2d[1].vessel.unit[i].annular.thickness
         if i == 0:  # open outline, there was one value too many, drop the last one
-            assert numpy.array_equal(thickness, [1, 0.9])
+            assert array_equal(thickness, [1, 0.9])
         else:  # closed outline, thickness values kept
-            assert numpy.array_equal(thickness, [1, 0.9, 0.9])
+            assert array_equal(thickness, [1, 0.9, 0.9])
 
     # Test conversion back
     wall3 = convert_ids(wall4, "3.39.0")
@@ -340,8 +341,8 @@ def test_3to4_repeat_children_first_point(dd4factory):
 
     iron_core4 = convert_ids(iron_core, None, factory=dd4factory)
     geometry = iron_core4.segment[0].geometry
-    assert numpy.array_equal(geometry.outline.r, [1.0, 2.0, 3.0, 1.0])
-    assert numpy.array_equal(geometry.outline.z, [-1.0, -2.0, -3.0, -1.0])
+    assert array_equal(geometry.outline.r, [1.0, 2.0, 3.0, 1.0])
+    assert array_equal(geometry.outline.z, [-1.0, -2.0, -3.0, -1.0])
 
     iron_core3 = convert_ids(iron_core4, "3.39.0")
     compare_children(iron_core, iron_core3)
@@ -356,11 +357,11 @@ def test_3to4_cocos_change(dd4factory):
     cp.profiles_1d[0].grid.psi = numpy.linspace(10, 20, 11)
 
     cp4 = convert_ids(cp, None, factory=dd4factory)
-    assert numpy.array_equal(
+    assert array_equal(
         cp4.profiles_1d[0].grid.rho_tor_norm,
         cp.profiles_1d[0].grid.rho_tor_norm,
     )
-    assert numpy.array_equal(
+    assert array_equal(
         cp4.profiles_1d[0].grid.psi,
         -cp.profiles_1d[0].grid.psi,
     )
@@ -368,6 +369,26 @@ def test_3to4_cocos_change(dd4factory):
     cp3 = convert_ids(cp4, "3.39.0")
     compare_children(cp, cp3)
 
+    eq = IDSFactory("3.39.0").equilibrium()
+    eq.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
+    eq.time = [1.0]
+    eq.time_slice.resize(1)
+    eq.time_slice[0].profiles_1d.psi = numpy.linspace(0, 1, 11)
+    eq.time_slice[0].profiles_1d.dpressure_dpsi = numpy.linspace(1, 2, 11)
+
+    eq4 = convert_ids(eq, None, factory=dd4factory)
+    assert array_equal(
+        eq4.time_slice[0].profiles_1d.psi,
+        -eq.time_slice[0].profiles_1d.psi,
+    )
+    assert array_equal(
+        eq4.time_slice[0].profiles_1d.dpressure_dpsi,
+        -eq.time_slice[0].profiles_1d.dpressure_dpsi,
+    )
+
+    eq3 = convert_ids(eq4, "3.39.0")
+    compare_children(eq, eq3)
+
 
 def test_3to4_circuit_connections(dd4factory, caplog):
     pfa = IDSFactory("3.39.0").pf_active()
@@ -380,7 +401,7 @@ def test_3to4_circuit_connections(dd4factory, caplog):
     ]
 
     pfa4 = convert_ids(pfa, None, factory=dd4factory)
-    assert numpy.array_equal(
+    assert array_equal(
         pfa4.circuit[0].connections, [[-1, 0, 1], [0, 1, -1], [1, -1, 0]]
     )
 
@@ -397,7 +418,7 @@ def test_3to4_circuit_connections(dd4factory, caplog):
     with caplog.at_level(logging.ERROR):
         pfa4 = convert_ids(pfa, None, factory=dd4factory)
     # Incorrect shape, data is not converted:
-    assert numpy.array_equal(pfa.circuit[0].connections, pfa4.circuit[0].connections)
+    assert array_equal(pfa.circuit[0].connections, pfa4.circuit[0].connections)
     # Check that a message with ERROR severity was logged
     assert len(caplog.record_tuples) == 1
     assert caplog.record_tuples[0][1] == logging.ERROR
@@ -410,7 +431,53 @@ def test_3to4_cocos_magnetics_workaround(dd4factory):
     mag.flux_loop[0].flux.data = [1.0, 2.0]
 
     mag4 = convert_ids(mag, None, factory=dd4factory)
-    assert numpy.array_equal(mag4.flux_loop[0].flux.data, [-1.0, -2.0])
+    assert array_equal(mag4.flux_loop[0].flux.data, [-1.0, -2.0])
 
     mag3 = convert_ids(mag4, "3.39.0")
     compare_children(mag, mag3)
+
+
+def test_3to4_pulse_schedule():
+    ps = IDSFactory("3.39.0").pulse_schedule()
+    ps.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
+
+    ps.ec.launcher.resize(3)
+    ps.ec.launcher[0].power.reference.data = [1.0, 2.0, 3.0]
+    ps.ec.launcher[0].power.reference.time = [1.0, 2.0, 3.0]
+    ps.ec.launcher[1].power.reference.data = [0.0, 2.0, 5.0]
+    ps.ec.launcher[1].power.reference.time = [0.0, 2.0, 5.0]
+    ps.ec.launcher[2].power.reference.data = [1.0, 1.5]
+    ps.ec.launcher[2].power.reference.time = [1.0, 1.5]
+
+    ps.ec.mode.data = [1, 2, 5]
+    ps.ec.mode.time = [1.0, 2.0, 5.0]
+
+    ps4 = convert_ids(ps, "4.0.0")
+    assert array_equal(ps4.ec.time, [0.0, 1.0, 1.5, 2.0, 3.0, 5.0])
+    item = "power_launched/reference"
+    assert array_equal(ps4.ec.beam[0][item], [1.0, 1.0, 1.5, 2.0, 3.0, 3.0])
+    assert array_equal(ps4.ec.beam[1][item], [0.0, 1.0, 1.5, 2.0, 3.0, 5.0])
+    assert array_equal(ps4.ec.beam[2][item], [1.0, 1.0, 1.5, 1.5, 1.5, 1.5])
+    assert array_equal(ps4.ec.mode, [1, 1, 1, 2, 2, 5])
+
+
+def test_3to4_pulse_schedule_exceptions():
+    ps = IDSFactory("3.39.0").pulse_schedule()
+    ps.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
+
+    ps.ec.launcher.resize(3)
+    ps.ec.launcher[0].power.reference.data = [1.0, 2.0, 3.0]
+    with pytest.raises(ValueError):  # missing time base
+        convert_ids(ps, "4.0.0")
+
+    ps.ec.launcher[0].power.reference.time = [1.0, 2.0]
+    with pytest.raises(ValueError):  # incorrect size of time base
+        convert_ids(ps, "4.0.0")
+
+
+def test_3to4_pulse_schedule_fuzz():
+    ps = IDSFactory("3.39.0").pulse_schedule()
+    ps.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
+
+    fill_consistent(ps)
+    convert_ids(ps, "4.0.0")

imas/test/test_lazy_loading.py

@@ -78,7 +78,7 @@ def test_lazy_loading_distributions_random_netcdf(tmp_path):
 
 def run_lazy_loading_distributions_random(dbentry):
     ids = IDSFactory().new("distributions")
-    fill_consistent(ids)
+    fill_consistent(ids, skip_complex=True)
     dbentry.put(ids)
 
     def iterate(structure):

{imas_python-2.0.0.dist-info → imas_python-2.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: imas-python
-Version: 2.0.0
+Version: 2.0.1
 Summary: Pythonic wrappers for the IMAS Access Layer
 Author: ITER Organization
 Author-email: Karel Lucas van de Plassche <karelvandeplassche@gmail.com>, Daan van Vugt <dvanvugt@ignitioncomputing.com>, Maarten Sebregts <msebregts@ignitioncomputing.com>, imas-support@iter.org
@@ -205,15 +205,14 @@ Classifier: Topic :: Utilities
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: numpy<2,>=1.15.4
+Requires-Dist: numpy>=1.15.4
 Requires-Dist: rich
 Requires-Dist: scipy
 Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
-Requires-Dist: saxonche
-Requires-Dist: gitpython
+Requires-Dist: imas_data_dictionaries
 Provides-Extra: all
 Requires-Dist: imas-python[docs,test]; extra == "all"
 Provides-Extra: docs
@@ -227,6 +226,8 @@ Provides-Extra: h5py
 Requires-Dist: h5py; extra == "h5py"
 Provides-Extra: xarray
 Requires-Dist: xarray; extra == "xarray"
+Provides-Extra: saxonche
+Requires-Dist: saxonche; extra == "saxonche"
 Provides-Extra: test
 Requires-Dist: pytest>=5.4.1; extra == "test"
 Requires-Dist: pytest-cov>=0.6; extra == "test"
@@ -236,7 +237,8 @@ Requires-Dist: matplotlib; extra == "test"
 Requires-Dist: asv==0.6.1; extra == "test"
 Requires-Dist: virtualenv; extra == "test"
 Requires-Dist: pint; extra == "test"
-Requires-Dist: imas-python[h5py,netcdf,xarray]; extra == "test"
+Requires-Dist: imas-python[h5py,netcdf,saxonche,xarray]; extra == "test"
+Dynamic: license-file
 
 # IMAS-Python
 
@@ -248,13 +250,20 @@ Data Model.
 
 ## Install
 
-Install steps are described in the documentation generated from `/docs/source/installing.rst`.
+Simply install IMAS-Python with ``pip``:
+```bash
+pip install imas-python
+```
+or with optional dependencies for netCDF and xarray support:
+```bash
+pip install imas-python[netcdf,xarray]
+```
 
-Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
+More details are described in the documentation generated from `/docs/source/installing.rst`.
+The documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
 and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
-The documentation can be manually generated by installing sphinx and running:
-
+To generate the documentation yourself, install the ``docs`` optional dependencies and do:
 ```bash
 make -C docs html
 ```

{imas_python-2.0.0.dist-info → imas_python-2.0.1.dist-info}/RECORD

@@ -1,14 +1,13 @@
-imas/__init__.py,sha256=LyuR1kEQUkXdHae3VTBBZCHLtFhrJ2fMPY_j6fX7dhc,878
+imas/__init__.py,sha256=lJPQRtGz6WP4e298IbK_qeaslaipe2GzT0mS6h3pPKU,862
 imas/__main__.py,sha256=nei61YSa3KE4V601qcApNQYgv9hKMenoMJyZLWmsHRQ,418
 imas/_to_xarray.py,sha256=l3Vc5tE0zcc9elt5RCRJjqzjmvvnNLCjkzqXQoep4zU,2713
 imas/_util.py,sha256=3CwP5yoyXNipyaUdMb-IwzVDlqV35VjTFxfaronO1AI,9484
-imas/_version.py,sha256=2thmcF9DS_Zp1zHI3N0kjBeMAuCP9mdDGnL0clqQpS8,511
+imas/_version.py,sha256=0PAtIHDEagdGmwgz1nnz5zUKeyYjsxtV8Nbsv7dXl8w,511
 imas/db_entry.py,sha256=GgzNXbU-4gL5ac2vVo28LtnF3JIKIqyQzg1MIhyMkkQ,31755
-imas/dd_helpers.py,sha256=MmD_ZSwW-c-7eq2lKwbCWvAZoW0nFAuL0T70dd-yZSk,6195
-imas/dd_zip.py,sha256=k0JFUqWBMXwuQmSOzukEujOJV21PF77jCBsNJQDcpGc,10322
-imas/exception.py,sha256=0IoVTpFZJPZVE-eeWQfR7lGX_v6ZYjgiT9S7ilBjEYM,3750
+imas/dd_zip.py,sha256=Fiu81NjIyQ-4tdmpaUNJtgKohiNYWIzYcNz96S9zrZ4,3988
+imas/exception.py,sha256=rX-dOIRzPEassnMCT3xXxK1NBMcBbjzpPEfYiVjyIzM,3354
 imas/ids_base.py,sha256=1jfnpRis-mlS3VbbaarBQ1HVyD3kwsjxhHkyPthlH_Q,5439
-imas/ids_convert.py,sha256=jU86M-svO3xca6pOknne3IS4GiSh1T_1rGIeiUBGb8g,38601
+imas/ids_convert.py,sha256=62NACbsBBjbQgPiUCzZtuCMnJfcddB0pP66py-DHYTY,43235
 imas/ids_coordinates.py,sha256=OT_9V9-0_mVZqutj6uuddnrhdgZ6_YocrdgeRWY-oyc,16955
 imas/ids_data_type.py,sha256=N3nQXmIht0q8BbqE99sPsTmZ62CTUHv10Pfj_rtYi54,3302
 imas/ids_defs.py,sha256=WRLnhHRlePdXey6jm_05w-z2qmZPHOW39rvHs7ifJco,5417
@@ -23,7 +22,6 @@ imas/ids_toplevel.py,sha256=MQVFFjgKxF-7tC5PLPCJEtJCjzOvwRxflrO6d0DThXs,14915
 imas/setup_logging.py,sha256=0v4_haGLsUJr8fHkl46UmTEgMImBzvfL2xU5kudB9ac,1872
 imas/training.py,sha256=QM447EzkfN1o6OKcvquh99omIU_4LbxEzW_HynSPdwI,1576
 imas/util.py,sha256=up6VH1IAfkn5WKmJ-1aQqaefbp2l8aDkOWEEPDds5zA,20763
-imas/assets/IDSDef.zip,sha256=4jCydzzVoCf59glCNUq3Vwul3DUTs1RHvDkaXD1zp10,27383258
 imas/assets/IDSDef2MDSpreTree.xsl,sha256=V9eDmDnxe6oYhDP81gmdYip8cF2yST9dGxmZBaATLdA,7296
 imas/assets/IDS_fake_toplevel.xml,sha256=TJhukQuDHHVTW2U7fK_-KmTQiYOFZMY-ECZXBHQE2mk,18683
 imas/assets/IDS_minimal.xml,sha256=4V8a7sYKyQnUeJQsWGIG1W8kWaTgZ0LjTGCvd9dkOlc,1740
@@ -38,11 +36,11 @@ imas/assets/equilibrium.ids,sha256=uYpVwTAyGVcWCrBUdSR3427qDY5zx1WXeX4Hd-o_2Jo,3
 imas/backends/__init__.py,sha256=mRL7PcVmJYdAjTEYwrNBQxgDR_BCjCdrVJeZDxqmhJU,469
 imas/backends/db_entry_impl.py,sha256=jGZDCxHnN9m2haUqp2RIPD1kZDnv4gFI5nQojf-m4X4,3995
 imas/backends/imas_core/__init__.py,sha256=Xdbe5WbZzPEfl2zgPzrAvU6k_tB6N70D1Qey9Gvoljg,190
-imas/backends/imas_core/al_context.py,sha256=zQYZqK5JJF1S6awjreXMZ-sdifb5gqDBR_QIIQG7hYk,14533
-imas/backends/imas_core/db_entry_al.py,sha256=-qPK8qBtG59oJGbEj0xuuT4mHhVoKt2qmBLEPHYTmHs,14893
+imas/backends/imas_core/al_context.py,sha256=i9GRd6ZHBhCfRug2UWvDJGoSqjjfyCvTVnduSF_seEA,14756
+imas/backends/imas_core/db_entry_al.py,sha256=ZtK5x-Nw9EGjKoXLAVeR2CsVI2nR-YlLvFai9DhGHbU,15051
 imas/backends/imas_core/db_entry_helpers.py,sha256=fjvb3gEBiAzexDCyIsoGTwEcWuvPwhjZ0epXa-p_9hA,7883
 imas/backends/imas_core/imas_interface.py,sha256=Ml28BrN-6MOUgxXgjUvm7tixhgkWwUWygKaHeI98cCk,7980
-imas/backends/imas_core/mdsplus_model.py,sha256=pBo_pOMkD852ISBqyLBIFpE_9S-8E6toUwcFXaPyS7c,14756
+imas/backends/imas_core/mdsplus_model.py,sha256=NdZ5fbBksU3ipk3iTMWUK5uPJxby2FoPsWXlwLq_I5U,15016
 imas/backends/imas_core/uda_support.py,sha256=lxHzby6Fkkpn5j6yWpQrlqlwng-gA9O_hVK03Wf_Ij8,1770
 imas/backends/netcdf/__init__.py,sha256=T1xOq4F7fJtFKYtzQQwVOV_9r9plNH6kgpUsDx6vDN0,191
 imas/backends/netcdf/db_entry_nc.py,sha256=0y44SkvapH7QRJOa_wVMCEE7g6PpVH120MonPseBLmE,7260
@@ -59,7 +57,6 @@ imas/command/timer.py,sha256=kyYGOGVTzeYzq8FWAvLqlGS-vVM7lO6hTozh-0aBUBA,2328
 imas/test/test_all_dd_versions.py,sha256=8T9q9S7Vhwa96y6KUDv7HoeuOPlc6RovEhPZEseDoDs,790
 imas/test/test_cli.py,sha256=ZSQdO5t4n0wa2ohJNsRQEVftJOxyDIHsj_IUZxKuszA,1677
 imas/test/test_dbentry.py,sha256=fq49bh7fa2O7ocLiXiDSB5EEVII6Cx3zeoiU5uzjyo8,4025
-imas/test/test_dd_helpers.py,sha256=dV5m6J5EMXGij0UOtXDxD2St9sKrfGhixk-QlH1v2-E,1691
 imas/test/test_dd_zip.py,sha256=SgKPufIvk3tK_Q4JpHKq57g0lKthEj1IgcdZ2c-IzwI,820
 imas/test/test_deepcopy.py,sha256=vYTwpEqRpi-G-xVSRWVIwLL_jBY7f6v31QPOU0e_FKs,606
 imas/test/test_exception.py,sha256=bL_JlWsdtT-emKkSHx1yFbDKLqWOx8piCFBNHJ4osig,408
@@ -69,7 +66,7 @@ imas/test/test_helpers.py,sha256=ynhwAW5lotJfbscub_S2cppHWyruxDiT304ELyjjFIA,146
 imas/test/test_identifiers.py,sha256=UXjgZJBRYQVnHejpdug6umNcnutdgQkAnWuTNduxgig,3895
 imas/test/test_ids2nc.py,sha256=zLp0F6GW04gU6HYp63bWivFS3Let_dwMohYy6Rgbis0,6040
 imas/test/test_ids_ascii_data.py,sha256=apah_SmGXKuATDEq-ADePQejh9qW4gXphMbCGKA8X34,1116
-imas/test/test_ids_convert.py,sha256=H0sc6Cz6i2q-PXxhlg2o-P1Eifdj_w6JUQho7z-BtpM,16172
+imas/test/test_ids_convert.py,sha256=syB6YPN0uqFPPownoXvbN8kbCedsDliOQ7Ar7mw5i7U,18511
 imas/test/test_ids_coordinates.py,sha256=d6o5wxNE-V_JFZ0Qq6Vz3UGC2KuhuVwVah06REnnbcI,6002
 imas/test/test_ids_data_type.py,sha256=VGQnJ-aeHzStJJQsnv3B2jr4rs33vRKKkOJxG-Szk-Y,1443
 imas/test/test_ids_factory.py,sha256=5izNPvMmpt64wgOxh2ResvzsRgE2ZVtaNZlYuU_27YY,1062
@@ -84,7 +81,7 @@ imas/test/test_ids_validate.py,sha256=0tRwmId61JFLlFmz-17DRKxntQtsKn_kRZULznt4d-
 imas/test/test_latest_dd_autofill.py,sha256=3SZD9J4jm_NQznJYT7ypnrmJCRAMV2iM8IJZnymrfJQ,2052
 imas/test/test_latest_dd_manual.py,sha256=XINCYCw3ZtbRch6Ug_gSPRyYXIg0dAorM7Y270kSYvI,1428
 imas/test/test_latest_dd_resample.py,sha256=ziPV-vReTiMr3is7l9-UHqVEyeHqinJ7PbXfK_0KO70,2859
-imas/test/test_lazy_loading.py,sha256=8cPDAeTnwrpwvr-7Fg1ycxVoA_tpVWdrDHvXCTsPvDo,8424
+imas/test/test_lazy_loading.py,sha256=gOtIMSG3XVo95Bb6qyR1Zu8tog0cTLCiZPECWVNkBrk,8443
 imas/test/test_list_occurrences.py,sha256=Fdzf2ap8vODkg1xdpUPZ2_k1RxlBvcE0pih0FcX-W48,2986
 imas/test/test_minimal.py,sha256=NWfzH_zWOGzNmQ8lVTr-FyLTdadrnI82L6eRZJ26eWQ,1419
 imas/test/test_minimal_conversion.py,sha256=4wufUlbLZzAgh57VJVSgpjqd2ybrcaxcmgsmL7aJV4M,1678
@@ -105,9 +102,9 @@ imas/test/test_time_slicing.py,sha256=FwbdggRJ75Suc268-89daCSaVXhzl8pZZm3Od13EiR
 imas/test/test_to_xarray.py,sha256=9eXRRNq0M-tF2OOQzQxG-mXTsOX1JfPVTEM5JIixyrY,2867
 imas/test/test_uda_support.py,sha256=AmakuexebZfJ2-vk_DHfNRPwHmNM2RnwzXqT0eWwWJ0,338
 imas/test/test_util.py,sha256=Dbl07LhaBZiabm7yh9cfkSS4VkEFsBNoo3fmhQPAC1g,6814
-imas_python-2.0.0.dist-info/LICENSE.txt,sha256=RzxtV7MG0zbF6dDvQsaZ1OKsHZ7CRyeILenjUIol5rs,7651
-imas_python-2.0.0.dist-info/METADATA,sha256=WS_iO0u3qUEvicGX2oTyzSjjlAW9Ti_VW41yi9bouTc,13877
-imas_python-2.0.0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-imas_python-2.0.0.dist-info/entry_points.txt,sha256=e8oXj2SLwyFEiBeJRaAXRc9EYQ8eHWWn7NWdjixjeDM,99
-imas_python-2.0.0.dist-info/top_level.txt,sha256=_WFOlA1rmpYm0T73sNo5Ntvs7S3x4cR7UE8Oy5I5eJA,5
-imas_python-2.0.0.dist-info/RECORD,,
+imas_python-2.0.1.dist-info/licenses/LICENSE.txt,sha256=RzxtV7MG0zbF6dDvQsaZ1OKsHZ7CRyeILenjUIol5rs,7651
+imas_python-2.0.1.dist-info/METADATA,sha256=7pMSO_q4pKcVkNEdqrGhuQDnP2OOoF6rIhx2UjtAa28,14168
+imas_python-2.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+imas_python-2.0.1.dist-info/entry_points.txt,sha256=PI0yiL4MmzUS-sIQjt4gSEnBCdL-rB_XHGASKNsTf6g,46
+imas_python-2.0.1.dist-info/top_level.txt,sha256=_WFOlA1rmpYm0T73sNo5Ntvs7S3x4cR7UE8Oy5I5eJA,5
+imas_python-2.0.1.dist-info/RECORD,,

{imas_python-2.0.0.dist-info → imas_python-2.0.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

imas_python-2.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+imas = imas.command.cli:cli

imas/dd_helpers.py

@@ -1,167 +0,0 @@
-# This file is part of IMAS-Python.
-# You should have received the IMAS-Python LICENSE file with this project.
-"""Helper functions to build IDSDef.xml"""
-
-import logging
-import os
-import shutil
-from pathlib import Path
-from typing import Tuple
-from zipfile import ZIP_DEFLATED, ZipFile
-
-from packaging.version import Version as V
-from saxonche import PySaxonProcessor
-
-logger = logging.getLogger(__name__)
-
-_idsdef_zip_relpath = Path("imas/assets/IDSDef.zip")
-_build_dir = Path("build")
-
-
-def prepare_data_dictionaries():
-    """Build IMAS IDSDef.xml files for each tagged version in the DD repository
-    1. Use saxonche for transformations
-    2. Clone the DD repository (ask for user/pass unless ssh key access is available)
-    3. Generate IDSDef.xml and rename to IDSDef_${version}.xml
-    4. Zip all these IDSDefs together and include in wheel
-    """
-    from git import Repo
-
-    repo: Repo = get_data_dictionary_repo()
-    if repo:
-        newest_version_and_tag = (V("0"), None)
-        for tag in repo.tags:
-            version_and_tag = (V(str(tag)), tag)
-            if V(str(tag)) > V("3.21.1"):
-                newest_version_and_tag = max(newest_version_and_tag, version_and_tag)
-                logger.debug("Building data dictionary version %s", tag)
-                build_data_dictionary(repo, tag)
-
-        logger.info("Creating zip file of DD versions")
-
-        if _idsdef_zip_relpath.is_file():
-            logger.warning("Overwriting '%s'", _idsdef_zip_relpath)
-
-        with ZipFile(
-            _idsdef_zip_relpath,
-            mode="w",  # this needs w, since zip can have multiple same entries
-            compression=ZIP_DEFLATED,
-        ) as dd_zip:
-            for filename in _build_dir.glob("[0-9]*.xml"):
-                arcname = Path("data-dictionary").joinpath(*filename.parts[1:])
-                dd_zip.write(filename, arcname=arcname)
-            # Include identifiers from latest tag in zip file
-            repo.git.checkout(newest_version_and_tag[1], force=True)
-            # DD layout <= 4.0.0
-            for filename in Path("data-dictionary").glob("*/*identifier.xml"):
-                arcname = Path("identifiers").joinpath(*filename.parts[1:])
-                dd_zip.write(filename, arcname=arcname)
-            # DD layout > 4.0.0
-            for filename in Path("data-dictionary").glob("schemas/*/*identifier.xml"):
-                arcname = Path("identifiers").joinpath(*filename.parts[2:])
-                dd_zip.write(filename, arcname=arcname)
-
-
-def get_data_dictionary_repo() -> Tuple[bool, bool]:
-    try:
-        import git  # Import git here, the user might not have it!
-    except ModuleNotFoundError:
-        raise RuntimeError(
-            "Could not find 'git' module, try 'pip install gitpython'. \
-            Will not build Data Dictionaries!"
-        )
-
-        # We need the actual source code (for now) so grab it from ITER
-    dd_repo_path = "data-dictionary"
-
-    if "DD_DIRECTORY" in os.environ:
-        logger.info("Found DD_DIRECTORY, copying")
-        try:
-            shutil.copytree(os.environ["DD_DIRECTORY"], dd_repo_path)
-        except FileExistsError:
-            pass
-    else:
-        logger.info("Trying to pull data dictionary git repo from ITER")
-
-    # Set up a bare repo and fetch the data-dictionary repository in it
-    os.makedirs(dd_repo_path, exist_ok=True)
-    try:
-        repo = git.Repo(dd_repo_path)
-    except git.exc.InvalidGitRepositoryError:
-        repo = git.Repo.init(dd_repo_path)
-    logger.info("Set up local git repository {!s}".format(repo))
-
-    try:
-        origin = repo.remote()
-    except ValueError:
-        dd_repo_url = "https://github.com/iterorganization/imas-data-dictionary.git"
-        origin = repo.create_remote("origin", url=dd_repo_url)
-    logger.info("Set up remote '{!s}' linking to '{!s}'".format(origin, origin.url))
-
-    try:
-        origin.fetch(tags=True)
-    except git.exc.GitCommandError as ee:
-        logger.warning(
-            "Could not fetch tags from %s. Git reports:\n %s." "\nTrying to continue",
-            list(origin.urls),
-            ee,
-        )
-    else:
-        logger.info("Remote tags fetched")
-    return repo
-
-
-def _run_xsl_transformation(
-    xsd_file: Path, xsl_file: Path, tag: str, output_file: Path
-) -> None:
-    """
-    This function performs an XSL transformation using Saxon-HE (saxonche)
-    with the provided XSD file,  XSL file, tag, and output file.
-
-    Args:
-        xsd_file (Path): XML Schema Definition (XSD) file
-        xsl_file (Path): The `xsl_file` parameter
-        tag (str): tag name to provide to 'DD_GIT_DESCRIBE' parameter
-        output_file (Path): The `output_file` parameter for resulting xml
-    """
-    with PySaxonProcessor(license=False) as proc:
-        logger.debug("Initializing Saxon Processor")
-        xsltproc = proc.new_xslt30_processor()
-        xdm_ddgit = proc.make_string_value(tag)
-        xsltproc.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)
-        xsltproc.transform_to_file(
-            source_file=str(xsd_file),
-            stylesheet_file=str(xsl_file),
-            output_file=str(output_file),
-        )
-
-
-def build_data_dictionary(repo, tag: str, rebuild=False) -> None:
-    """Build a single version of the data dictionary given by the tag argument
-    if the IDS does not already exist.
-
-    In the data-dictionary repository sometimes IDSDef.xml is stored
-    directly, in which case we do not call make.
-
-    Args:
-        repo: Repository object containing the DD source code
-        tag: The DD version tag that will be build
-        rebuild: If true, overwrites existing pre-build tagged DD version
-    """
-    _build_dir.mkdir(exist_ok=True)
-    result_xml = _build_dir / f"{tag}.xml"
-
-    if result_xml.exists() and not rebuild:
-        logger.debug(f"XML for tag '{tag}' already exists, skipping")
-        return
-
-    repo.git.checkout(tag, force=True)
-
-    # Perform the XSL transformation with saxonche
-    dd_xsd = Path("data-dictionary/dd_data_dictionary.xml.xsd")
-    dd_xsl = Path("data-dictionary/dd_data_dictionary.xml.xsl")
-    _run_xsl_transformation(dd_xsd, dd_xsl, tag.name, result_xml)
-
-
-if __name__ == "__main__":
-    prepare_data_dictionaries()

imas/test/test_dd_helpers.py

@@ -1,53 +0,0 @@
-from pathlib import Path
-import shutil
-import pytest
-import os
-import zipfile
-
-from imas.dd_helpers import prepare_data_dictionaries, _idsdef_zip_relpath, _build_dir
-
-_idsdef_unzipped_relpath = Path("idsdef_unzipped")
-
-
-@pytest.mark.skip(reason="skipping IDSDef.zip generation")
-def test_prepare_data_dictionaries():
-    prepare_data_dictionaries()
-    assert os.path.exists(
-        _idsdef_zip_relpath
-    ), f"IDSDef.zip file does not exist at path: {_idsdef_zip_relpath}"
-
-    expected_xml_files = [
-        _build_dir / "3.40.0.xml",
-        _build_dir / "3.41.0.xml",
-        _build_dir / "3.42.0.xml",
-        _build_dir / "4.0.0.xml",
-    ]
-
-    for xml_file in expected_xml_files:
-        assert os.path.exists(xml_file), f"{xml_file} does not exist"
-
-    with zipfile.ZipFile(_idsdef_zip_relpath, "r") as zip_ref:
-        zip_ref.extractall(_idsdef_unzipped_relpath)
-
-    expected_ids_directories = [
-        _idsdef_unzipped_relpath / "data-dictionary" / "3.40.0.xml",
-        _idsdef_unzipped_relpath / "data-dictionary" / "3.41.0.xml",
-        _idsdef_unzipped_relpath / "data-dictionary" / "3.42.0.xml",
-        _idsdef_unzipped_relpath / "data-dictionary" / "4.0.0.xml",
-        _idsdef_unzipped_relpath
-        / "identifiers"
-        / "core_sources"
-        / "core_source_identifier.xml",
-        _idsdef_unzipped_relpath
-        / "identifiers"
-        / "equilibrium"
-        / "equilibrium_profiles_2d_identifier.xml",
-    ]
-
-    for file_path in expected_ids_directories:
-        assert os.path.exists(
-            file_path
-        ), f"Expected_ids_directories {file_path} does not exist"
-
-    if _build_dir.exists():
-        shutil.rmtree(_idsdef_unzipped_relpath)

imas_python-2.0.0.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-build_DD = imas.dd_helpers:prepare_data_dictionaries
-imas = imas.command.cli:cli