imas-python 2.0.0.post1__tar.gz → 2.0.1__tar.gz

imas_python-2.0.1/.github/CODEOWNERS

@@ -0,0 +1,10 @@
+# This file allows setting automatically reviewers for pull requests.
+# Each line is a file pattern followed by one or more owners.
+# The last match takes precedence over previous ones.
+# Do not edit unless specifically mandated to do so.
+
+# Global/fallback and technical modifications.
+*         @maarten-ic @prasad-sawantdesai @olivhoenen
+
+# Modifications to CODEOWNERS and action workflows 
+.github/  @SimonPinches @olivhoenen

{imas_python-2.0.0.post1 → imas_python-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: imas-python
-Version: 2.0.0.post1
+Version: 2.0.1
 Summary: Pythonic wrappers for the IMAS Access Layer
 Author: ITER Organization
 Author-email: Karel Lucas van de Plassche <karelvandeplassche@gmail.com>, Daan van Vugt <dvanvugt@ignitioncomputing.com>, Maarten Sebregts <msebregts@ignitioncomputing.com>, imas-support@iter.org
@@ -205,15 +205,14 @@ Classifier: Topic :: Utilities
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: numpy<2,>=1.15.4
+Requires-Dist: numpy>=1.15.4
 Requires-Dist: rich
 Requires-Dist: scipy
 Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
-Requires-Dist: saxonche
-Requires-Dist: gitpython
+Requires-Dist: imas_data_dictionaries
 Provides-Extra: all
 Requires-Dist: imas-python[docs,test]; extra == "all"
 Provides-Extra: docs
@@ -227,6 +226,8 @@ Provides-Extra: h5py
 Requires-Dist: h5py; extra == "h5py"
 Provides-Extra: xarray
 Requires-Dist: xarray; extra == "xarray"
+Provides-Extra: saxonche
+Requires-Dist: saxonche; extra == "saxonche"
 Provides-Extra: test
 Requires-Dist: pytest>=5.4.1; extra == "test"
 Requires-Dist: pytest-cov>=0.6; extra == "test"
@@ -236,7 +237,8 @@ Requires-Dist: matplotlib; extra == "test"
 Requires-Dist: asv==0.6.1; extra == "test"
 Requires-Dist: virtualenv; extra == "test"
 Requires-Dist: pint; extra == "test"
-Requires-Dist: imas-python[h5py,netcdf,xarray]; extra == "test"
+Requires-Dist: imas-python[h5py,netcdf,saxonche,xarray]; extra == "test"
+Dynamic: license-file
 
 # IMAS-Python
 
@@ -248,13 +250,20 @@ Data Model.
 
 ## Install
 
-Install steps are described in the documentation generated from `/docs/source/installing.rst`.
+Simply install IMAS-Python with ``pip``:
+```bash
+pip install imas-python
+```
+or with optional dependencies for netCDF and xarray support:
+```bash
+pip install imas-python[netcdf,xarray]
+```
 
-Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
+More details are described in the documentation generated from `/docs/source/installing.rst`.
+The documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
 and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
-The documentation can be manually generated by installing sphinx and running:
-
+To generate the documentation yourself, install the ``docs`` optional dependencies and do:
 ```bash
 make -C docs html
 ```

{imas_python-2.0.0.post1 → imas_python-2.0.1}/README.md

@@ -8,13 +8,20 @@ Data Model.
 
 ## Install
 
-Install steps are described in the documentation generated from `/docs/source/installing.rst`.
+Simply install IMAS-Python with ``pip``:
+```bash
+pip install imas-python
+```
+or with optional dependencies for netCDF and xarray support:
+```bash
+pip install imas-python[netcdf,xarray]
+```
 
-Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
+More details are described in the documentation generated from `/docs/source/installing.rst`.
+The documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
 and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
-The documentation can be manually generated by installing sphinx and running:
-
+To generate the documentation yourself, install the ``docs`` optional dependencies and do:
 ```bash
 make -C docs html
 ```

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/changelog.rst

@@ -3,6 +3,21 @@
 Changelog
 =========
 
+What's new in IMAS-Python 2.0.1
+-------------------------------
+
+Improvements
+''''''''''''
+
+- improve DD3-->DD4 conversion (sign conversion to dodpsi_like)
+- improve conversion of pulse_schedule IDS >= 3.39.0
+- numpy 2 compatibility
+- improve UDA data fetch
+- improve documentation
+- new dependency on `imas-data-dictionaries package <https://pypi.org/project/imas-data-dictionaries/>`__ (remove internal build via saxonche, except for the optional MDSplus models)
+- full compatibility of tests with netCDF<1.7 (no complex numbers)
+
+
 What's new in IMAS-Python 2.0.0
 -------------------------------
 

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/courses/advanced/dd_versions.rst

@@ -60,7 +60,7 @@ Bundled Data Dictionary definitions
 
 IMAS-Python comes bundled [#DDdefs]_ with many versions of the Data Dictionary definitions.
 You can find out which versions are available by calling
-:py:meth:`imas.dd_zip.dd_xml_versions`.
+``imas.dd_zip.dd_xml_versions``.
 
 
 Converting an IDS between Data Dictionary versions
@@ -290,6 +290,6 @@ build, you can use them like you normally would.
 
 .. rubric:: Footnotes
 
-.. [#DDdefs] To be more precise, the Data Dictionary definitions are generated when the
-    IMAS-Python package is created. See :ref:`this reference <DD background>` for more
-    details.
+.. [#DDdefs] To be more precise, the Data Dictionary definitions are provided by the
+    `IMAS Data Dictionaries <http://pypi.org/project/imas-data-dictionaries/>`__
+    package.

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/courses/basic/analyze.rst

@@ -246,3 +246,20 @@ Exercise 5
             A plot of :math:`T_e` vs :math:`t`.
 
 .. seealso:: :ref:`Lazy loading`
+
+
+Explore the DBEntry and occurrences
+'''''''''''''''''''''''''''''''''''
+
+You may not know a priori which types of IDSs are available within an IMAS database entry.
+It can also happen that several IDSs objects of the same type are stored within
+this entry, in that case each IDS is stored as a separate `occurrence`
+(occurrences are identified with an integer value, 0 being the default).
+
+In IMAS-Python, the function :meth:`~imas.db_entry.DBEntry.list_all_occurrences()` will
+help you finding which occurrences are available in a given database entry and for a given
+IDS type.
+
+The following snippet shows how to list the available IDSs in a given database entry:
+
+.. literalinclude:: imas_snippets/explore_data_entry.py

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/courses/basic/explore.rst

@@ -7,8 +7,8 @@ In this part of the training, we will learn how to use Python to explore data
 saved in IDSs.
 
 
-Explore which IDSs are available
---------------------------------
+Explore which IDS structures are available
+------------------------------------------
 
 Most codes will touch multiple IDSs inside a single IMAS data entry. For example
 a heating code using a magnetic equilibrium from the ``equilibrium`` IDS with a

imas_python-2.0.1/docs/source/courses/basic/imas_snippets/explore_data_entry.py

@@ -0,0 +1,11 @@
+import imas
+
+# Open input data entry 
+entry = imas.DBEntry("imas:hdf5?path=<...>", "r")
+
+# Print the list of available IDSs with their occurrence
+for idsname in imas.IDSFactory().ids_names():
+    for occ in entry.list_all_occurrences(idsname):
+        print(idsname, occ)
+
+entry.close()

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/imas_architecture.rst

@@ -72,11 +72,7 @@ Data Dictionary building and loading
 The following submodules are responsible for building the Data Dictionary and loading DD
 definitions at runtime.
 
--   :py:mod:`imas.dd_helpers` handles building the ``IDSDef.zip`` file, containing all
-    versions of the Data Dictionary since ``3.22.0``.
-
 -   :py:mod:`imas.dd_zip` handles loading the Data Dictionary definitions at run time.
-    These definitions can be loaded from an ``IDSDef.zip`` or from a custom XML file.
 
 
 .. _imas_architecture/IDS_nodes:

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/installing.rst

@@ -13,6 +13,29 @@ To get started, you can install it from `pypi.org <https://pypi.org/project/imas
 
     pip install imas-python
 
+You can also install optional dependencies (e.g. netCDF and xarray):
+
+.. code-block:: bash
+
+    pip install imas-python[netcdf,xarray]
+
+
+List of optional dependencies
+-----------------------------
+
+- ``netcdf``: enables storing/loading of IDS to/from netCDF files
+- ``xarray``: enables loading IDS (entire IDS or part of it) into an xarray ``dataset``
+- ``h5py``: enables ``analyze-db`` CLI option 
+- ``docs``: installs required packages to build the Sphinx documentation
+- ``test``: installs required packages to run the tests with ``pytest`` and ``asv``
+- ``saxonche``: installs saxonche to enable creation of MDSplus models for the selected versions of the IMAS Data Dictionary (only relevant when working with ``imas_core``)
+
+.. note::
+
+    Some tests will be skipped unless you also have ``imas_core`` installed
+    (it is not yet available on PyPI, so you will need to install it from sources
+    if you have access to them at https://git.iter.org/projects/IMAS/repos/al-core) 
+
 
 Local installation from sources
 -------------------------------

{imas_python-2.0.0.post1 → imas_python-2.0.1}/docs/source/multi-dd.rst

@@ -146,6 +146,7 @@ explicit conversion mechanisms.
   Changed definition of open/closed contours, Yes, No
   Changed definition of ``space/coordinates_type`` in GGD grids, Yes, No
   Migrate obsolescent ``ids_properties/source`` to ``ids_properties/provenance``, Yes, No
+  Convert the multiple time-bases in the ``pulse_schedule`` IDS [#ps3to4]_, Yes, No
 
 .. [#rename] Quantities which have been renamed between the two DD versions. For
   example, the ``ec/beam`` Array of Structures in the ``pulse_schedule`` IDS,
@@ -175,6 +176,15 @@ explicit conversion mechanisms.
 .. [#ignore_type_change] These type changes are not supported. Quantities in the
     destination IDS will remain empty.
 
+.. [#ps3to4] In Data Dictionary 3.39.0 and older, all dynamic quantities in the
+    ``pulse_schedule`` IDS had their own time array. In DD 4.0.0 this was
+    restructured to one time array per component (for example `ec/time
+    <https://imas-data-dictionary.readthedocs.io/en/latest/generated/ids/pulse_schedule.html#pulse_schedule-ec-time>`__).
+    This migration constructs a common time base per subgroup, and interpolates
+    the dynamic quantities within the group to the new time base. Resampling
+    uses `previous neighbour` interpolation for integer quantities, and linear
+    interpolation otherwise. See also:
+    https://github.com/iterorganization/IMAS-Python/issues/21.
 
 .. _`DD background`:
 
@@ -197,21 +207,14 @@ Automated tests have been provided that check the loading of all of the DD
 versions tagged in the data-dictionary git repository.
 
 
-Extending the DD set
-''''''''''''''''''''
+Data Dictionary definitions
+'''''''''''''''''''''''''''
 
-Use the command ``python setup.py build_DD`` to build a new ``IDSDef.zip``. This
-fetches all tags from the data dictionary git repository and builds the ``IDSDef.zip``.
+The Data Dictionary definitions used by IMAS-Python are provided by the `IMAS Data
+Dictionaries <http://pypi.org/project/imas-data-dictionaries/>`__ package.
+Please update this package if you need a more recent version of the data dictionary. For
+example, using ``pip``:
 
-IMAS-Python searches for an ``IDSDef.zip`` in the following locations:
+.. code-block:: bash
 
-1.  The environment variable ``$IMAS_DDZIP`` (path to a zip file)
-2.  The file ``./IDSDef.zip`` in the current working directory
-3.  In the local configuration folder: ``~/.config/imas/IDSDef.zip``, or
-    ``$XDG_CONFIG_DIR/imas/IDSDef.zip`` (if the environment variable
-    ``$XDG_CONFIG_DIR`` is set)
-4.  The zipfile bundled with the IMAS-Python installation: ``assets/IDSDef.zip``
-
-All paths are searched in order when loading the definitions of a specific data
-dictionary version: the first zip file that contains the definitions of the requested
-version is used.
+  pip install --upgrade imas-data-dictionaries

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/__init__.py

@@ -20,7 +20,6 @@ from .ids_identifiers import identifiers
 # Load the IMAS-Python IMAS AL/DD core
 from . import (
     db_entry,
-    dd_helpers,
     dd_zip,
     util,
 )

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.0.0.post1'
-__version_tuple__ = version_tuple = (2, 0, 0, 'post1')
+__version__ = version = '2.0.1'
+__version_tuple__ = version_tuple = (2, 0, 1)

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/backends/imas_core/al_context.py

@@ -1,7 +1,6 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Object-oriented interface to the IMAS lowlevel.
-"""
+"""Object-oriented interface to the IMAS lowlevel."""
 
 import logging
 import weakref
@@ -61,17 +60,21 @@ class ALContext:
     def __exit__(self, exc_type, exc_value, traceback) -> None:
         ll_interface.end_action(self.ctx)
 
-    def global_action(self, path: str, rwmode: int) -> "ALContext":
+    def global_action(self, path: str, rwmode: int, datapath: str = "") -> "ALContext":
         """Begin a new global action for use in a ``with`` context.
 
         Args:
             path: access layer path for this global action: ``<idsname>[/<occurrence>]``
             rwmode: read-only or read-write operation mode: ``READ_OP``/``WRITE_OP``
+            datapath: used by UDA backend to fetch only part of the data.
 
         Returns:
             The created context.
         """
-        status, ctx = ll_interface.begin_global_action(self.ctx, path, rwmode)
+        args = [self.ctx, path, rwmode]
+        if datapath:  # AL4 compatibility: datapath arg was added in AL5
+            args.append(datapath)
+        status, ctx = ll_interface.begin_global_action(*args)
         if status != 0:
             raise LowlevelError("global_action", status)
         return ALContext(ctx)

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/backends/imas_core/db_entry_al.py

@@ -257,7 +257,8 @@ class ALDBEntryImpl(DBEntryImpl):
         if occurrence != 0:
             ll_path += f"/{occurrence}"
 
-        with self._db_ctx.global_action(ll_path, READ_OP) as read_ctx:
+        datapath = "ids_properties" if self.backend == "uda" else ""
+        with self._db_ctx.global_action(ll_path, READ_OP, datapath) as read_ctx:
             time_mode_path = "ids_properties/homogeneous_time"
             time_mode = read_ctx.read_data(time_mode_path, "", INTEGER_DATA, 0)
             # This is already checked by read_dd_version, but ensure:
@@ -314,7 +315,8 @@ class ALDBEntryImpl(DBEntryImpl):
         if occurrence != 0:
             ll_path += f"/{occurrence}"
 
-        with self._db_ctx.global_action(ll_path, READ_OP) as read_ctx:
+        datapath = "ids_properties" if self.backend == "uda" else ""
+        with self._db_ctx.global_action(ll_path, READ_OP, datapath) as read_ctx:
             time_mode_path = "ids_properties/homogeneous_time"
             time_mode = read_ctx.read_data(time_mode_path, "", INTEGER_DATA, 0)
             dd_version_path = "ids_properties/version_put/data_dictionary"

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/backends/imas_core/mdsplus_model.py

@@ -12,7 +12,6 @@ import tempfile
 import time
 import uuid
 from pathlib import Path
-from saxonche import PySaxonProcessor
 from subprocess import CalledProcessError, check_output
 from zlib import crc32
 
@@ -244,11 +243,20 @@ def transform_with_xslt(xslt_processor, source, xslfile, output_file):
 
 def create_model_ids_xml(cache_dir_path, fname, version):
     """Use Saxon/C to compile an ids.xml suitable for creating an MDSplus model."""
+    try:
+        import saxonche
+    except ImportError:
+        raise RuntimeError(
+            "Building mdsplus models requires the 'saxonche' python package. "
+            "Please install this package (for example with 'pip install saxonche') "
+            "and try again."
+        )
+
     try:
         with as_file(files("imas") / "assets" / "IDSDef2MDSpreTree.xsl") as xslfile:
             output_file = Path(cache_dir_path) / "ids.xml"
 
-            with PySaxonProcessor(license=False) as proc:
+            with saxonche.PySaxonProcessor(license=False) as proc:
                 xslt_processor = proc.new_xslt30_processor()
                 xdm_ddgit = proc.make_string_value(str(version) or fname)
                 xslt_processor.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)

imas_python-2.0.1/imas/dd_zip.py

@@ -0,0 +1,109 @@
+# This file is part of IMAS-Python.
+# You should have received the IMAS-Python LICENSE file with this project.
+"""Extract DD versions from the imas-data-dictionaries distribution."""
+
+import logging
+import os
+import xml.etree.ElementTree as ET
+from functools import lru_cache
+from pathlib import Path
+
+# These methods in imas_data_dictionaries used to be defined here. We import them here
+# for backwards compatibility:
+from imas_data_dictionaries import dd_identifiers  # noqa: F401
+from imas_data_dictionaries import get_dd_xml_crc  # noqa: F401
+from imas_data_dictionaries import get_identifier_xml  # noqa: F401
+from imas_data_dictionaries import dd_xml_versions, get_dd_xml, parse_dd_version
+from packaging.version import InvalidVersion
+
+import imas
+from imas.exception import UnknownDDVersion  # noqa: F401
+
+logger = logging.getLogger(__name__)
+
+
+# Expected use case is one, maximum two DD versions
+# Cache is bigger than that: in pytest we currently use the following DD versions:
+#   - 3.22.0
+#   - 3.25.0
+#   - 3.28.0
+#   - 3.39.0
+#   - 4.0.0 (if available)
+#   - Environment default
+#   - IDS_fake_toplevel.xml
+#   - IDS_minimal.xml
+#   - IDS_minimal_2.xml
+#   - IDS_minimal_struct_array.xml
+#   - IDS_minimal_types.xml
+_DD_CACHE_SIZE = 8
+
+
+def dd_etree(version=None, xml_path=None):
+    """Return the DD element tree corresponding to the provided dd_version or xml_file.
+
+    By default (``dd_version`` and ``dd_xml`` are not supplied), this will attempt
+    to get the version from the environment (``IMAS_VERSION``) and use the latest
+    available version as fallback.
+
+    You can also specify a specific DD version to use (e.g. "3.38.1") or point to a
+    specific data-dictionary XML file. These options are exclusive.
+
+    Args:
+        version: DD version string, e.g. "3.38.1".
+        xml_path: XML file containing data dictionary definition.
+    """
+    if version and xml_path:
+        raise ValueError("version and xml_path cannot be provided both.")
+    if not version and not xml_path:
+        # Figure out which DD version to use
+        if "IMAS_VERSION" in os.environ:
+            imas_version = os.environ["IMAS_VERSION"]
+            if imas_version in dd_xml_versions():
+                # Use bundled DD version when available
+                version = imas_version
+            elif "IMAS_PREFIX" in os.environ:
+                # Try finding the IDSDef.xml in this installation
+                imas_prefix = Path(os.environ["IMAS_PREFIX"]).resolve()
+                xml_file = imas_prefix / "include" / "IDSDef.xml"
+                if xml_file.exists():
+                    xml_path = str(xml_file)
+            if not version and not xml_path:
+                logger.warning(
+                    "Unable to load IMAS version %s, falling back to latest version.",
+                    imas_version,
+                )
+    if not version and not xml_path:
+        # Use latest available from
+        version = latest_dd_version()
+    # Do the actual loading in a cached method:
+    return _load_etree(version, xml_path)
+
+
+@lru_cache(_DD_CACHE_SIZE)
+def _load_etree(version, xml_path):
+    if xml_path:
+        logger.info("Parsing data dictionary from file: %s", xml_path)
+        tree = ET.parse(xml_path)
+    else:
+        xml = get_dd_xml(version)
+        logger.info("Parsing data dictionary version %s", version)
+        tree = ET.ElementTree(ET.fromstring(xml))
+    return tree
+
+
+def print_supported_version_warning(version):
+    try:
+        if parse_dd_version(version) < imas.OLDEST_SUPPORTED_VERSION:
+            logger.warning(
+                "Version %s is below lowest supported version of %s.\
+                Proceed at your own risk.",
+                version,
+                imas.OLDEST_SUPPORTED_VERSION,
+            )
+    except InvalidVersion:
+        logging.warning("Ignoring version parsing error.", exc_info=1)
+
+
+def latest_dd_version():
+    """Find the latest version in data-dictionary/IDSDef.zip"""
+    return dd_xml_versions()[-1]

{imas_python-2.0.0.post1 → imas_python-2.0.1}/imas/exception.py

@@ -1,11 +1,14 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Exception classes used in IMAS-Python.
-"""
+"""Exception classes used in IMAS-Python."""
 
 import difflib
 import logging
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
+
+# This exception from imas_data_dictionaries used to be defined here. We import it here
+# for backwards compatibility:
+from imas_data_dictionaries import UnknownDDVersion  # noqa: F401
 
 from imas.backends.imas_core import imas_interface as _imas_interface
 
@@ -23,20 +26,6 @@ else:
     ALException = None
 
 
-class UnknownDDVersion(ValueError):
-    """Error raised when an unknown DD version is specified."""
-
-    def __init__(self, version: str, available: List[str], note: str = "") -> None:
-        close_matches = difflib.get_close_matches(version, available, n=1)
-        if close_matches:
-            suggestions = f"Did you mean {close_matches[0]!r}?"
-        else:
-            suggestions = f"Available versions are {', '.join(reversed(available))}"
-        super().__init__(
-            f"Data dictionary version {version!r} cannot be found. {suggestions}{note}"
-        )
-
-
 class IDSNameError(ValueError):
     """Error raised by DBEntry.get(_slice) when providing an invalid IDS name."""