imas-python 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

imas/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.1'
-__version_tuple__ = version_tuple = (2, 0, 1)
+__version__ = version = '2.1.0'
+__version_tuple__ = version_tuple = (2, 1, 0)
+
+__commit_id__ = commit_id = None

imas/backends/imas_core/al_context.py

@@ -71,10 +71,7 @@ class ALContext:
         Returns:
             The created context.
         """
-        args = [self.ctx, path, rwmode]
-        if datapath:  # AL4 compatibility: datapath arg was added in AL5
-            args.append(datapath)
-        status, ctx = ll_interface.begin_global_action(*args)
+        status, ctx = ll_interface.begin_global_action(self.ctx, path, rwmode, datapath)
         if status != 0:
             raise LowlevelError("global_action", status)
         return ALContext(ctx)

imas/backends/imas_core/db_entry_al.py

@@ -5,30 +5,29 @@ import getpass
 import logging
 import os
 from collections import deque
+import re
 from typing import Any, Deque, List, Optional, Union
 from urllib.parse import urlparse
 
+from packaging.version import Version
+
+import imas
 from imas.backends.db_entry_impl import GetSampleParameters, GetSliceParameters
 from imas.db_entry import DBEntryImpl
 from imas.exception import DataEntryException, LowlevelError
 from imas.ids_convert import NBCPathMap, dd_version_map_from_factories
 from imas.ids_defs import (
-    ASCII_BACKEND,
     CHAR_DATA,
     CLOSE_PULSE,
     CREATE_PULSE,
     ERASE_PULSE,
     FORCE_CREATE_PULSE,
     FORCE_OPEN_PULSE,
-    HDF5_BACKEND,
     IDS_TIME_MODE_UNKNOWN,
     IDS_TIME_MODES,
     INTEGER_DATA,
-    MDSPLUS_BACKEND,
-    MEMORY_BACKEND,
     OPEN_PULSE,
     READ_OP,
-    UDA_BACKEND,
     UNDEFINED_INTERP,
     UNDEFINED_TIME,
     WRITE_OP,
@@ -40,16 +39,9 @@ from imas.ids_toplevel import IDSToplevel
 from .al_context import ALContext, LazyALContext
 from .db_entry_helpers import delete_children, get_children, put_children
 from .imas_interface import LLInterfaceError, has_imas, ll_interface
-from .mdsplus_model import ensure_data_dir, mdsplus_model_dir
+from .mdsplus_model import mdsplus_model_dir
 from .uda_support import extract_idsdef, get_dd_version_from_idsdef_xml
 
-_BACKEND_NAME = {
-    ASCII_BACKEND: "ascii",
-    HDF5_BACKEND: "hdf5",
-    MEMORY_BACKEND: "memory",
-    MDSPLUS_BACKEND: "mdsplus",
-    UDA_BACKEND: "uda",
-}
 _OPEN_MODES = {
     "r": OPEN_PULSE,
     "a": FORCE_OPEN_PULSE,
@@ -71,13 +63,26 @@ def require_imas_available():
 class ALDBEntryImpl(DBEntryImpl):
     """DBEntry implementation using imas_core as a backend."""
 
-    """Map to the expected open_pulse (AL4) / begin_dataentry_action (AL5) argument."""
+    def __init__(self, uri: str, mode: int, factory: IDSFactory):
+        # Setup backend and lowlevel Access Layer:
+        backend = urlparse(uri).path.lower().lstrip("/")
+        self._setup_backend(backend, mode, factory)
+        status, ctx = ll_interface.begin_dataentry_action(uri, mode)
+        if status != 0:
+            raise LowlevelError("opening/creating data entry", status)
 
-    def __init__(self, backend: str, ctx: ALContext, factory: IDSFactory):
         self.backend = backend
-        self._db_ctx = ctx
+        self._db_ctx = ALContext(ctx)
         self._ids_factory = factory
         self._lazy_ctx_cache: Deque[ALContext] = deque()
+        self._uri = uri
+
+        # Parse query options, mimic logic in AL-Core instead of using
+        # urllib.parse.parse_qs(..). See https://github.com/jholloc/simple-uri-parser
+        self._querydict = {}
+        for option in re.split("[&;?]", urlparse(self._uri).query):
+            name, _, value = option.partition("=")
+            self._querydict[name] = value
 
     @classmethod
     def from_uri(cls, uri: str, mode: str, factory: IDSFactory) -> "ALDBEntryImpl":
@@ -85,7 +90,7 @@ class ALDBEntryImpl(DBEntryImpl):
         if mode not in _OPEN_MODES:
             modes = list(_OPEN_MODES)
             raise ValueError(f"Unknown mode {mode!r}, was expecting any of {modes}")
-        return cls._from_uri(uri, _OPEN_MODES[mode], factory)
+        return cls(uri, _OPEN_MODES[mode], factory)
 
     @classmethod
     def from_pulse_run(
@@ -108,60 +113,18 @@ class ALDBEntryImpl(DBEntryImpl):
         data_version = data_version or factory.dd_version
         options = options if options else ""
 
-        if ll_interface._al_version.major >= 5:
-            # We need a URI for AL 5 or later, construct from legacy parameters
-            status, uri = ll_interface.build_uri_from_legacy_parameters(
-                backend_id, pulse, run, user_name, db_name, data_version, options
-            )
-            if status != 0:
-                raise LowlevelError("build URI from legacy parameters", status)
-
-            return cls._from_uri(uri, mode, factory)
-
-        else:
-            # AL4 legacy support:
-            backend = _BACKEND_NAME.get(backend_id, "")
-            cls._setup_backend(backend, mode, factory, user_name, db_name, run)
-
-            status, ctx = ll_interface.begin_pulse_action(
-                backend_id, pulse, run, user_name, db_name, data_version
-            )
-            if status != 0:
-                raise LowlevelError("begin pulse action", status)
-
-            status = ll_interface.open_pulse(ctx, mode, options)
-            if status != 0:
-                raise LowlevelError("opening/creating data entry", status)
-
-            return cls(backend, ALContext(ctx), factory)
-
-    @classmethod
-    def _from_uri(cls, uri: str, mode: int, factory: IDSFactory) -> "ALDBEntryImpl":
-        """Helper method to actually open/create the dataentry."""
-        backend = urlparse(uri).path.lower().lstrip("/")
-        cls._setup_backend(backend, mode, factory)
-
-        status, ctx = ll_interface.begin_dataentry_action(uri, mode)
+        # Construct URI from legacy parameters
+        status, uri = ll_interface.build_uri_from_legacy_parameters(
+            backend_id, pulse, run, user_name, db_name, data_version, options
+        )
         if status != 0:
-            raise LowlevelError("opening/creating data entry", status)
+            raise LowlevelError("build URI from legacy parameters", status)
 
-        return cls(backend, ALContext(ctx), factory)
+        return cls(uri, mode, factory)
 
     @classmethod
-    def _setup_backend(
-        cls,
-        backend: str,
-        mode: int,
-        factory: IDSFactory,
-        user_name: str = "",
-        db_name="",
-        run=1,
-    ) -> None:
-        """Custom logic for preparing some backends.
-
-        Note: user_name, db_name and run are only used for AL 4.x, they can be
-        omitted when using AL 5 or later.
-        """
+    def _setup_backend(cls, backend: str, mode: int, factory: IDSFactory) -> None:
+        """Custom logic for preparing some backends."""
         if backend == "mdsplus":
             # MDSplus models:
             if mode != OPEN_PULSE:
@@ -170,22 +133,6 @@ class ALDBEntryImpl(DBEntryImpl):
                 if ids_path:
                     os.environ["ids_path"] = ids_path
 
-            if ll_interface._al_version.major == 4:
-                # Ensure the data directory exists
-                # Note: MDSPLUS model directory only uses the major version component of
-                # IMAS_VERSION, so we'll take the first character of IMAS_VERSION:
-                version = factory.version[0]
-                ensure_data_dir(user_name, db_name, version, run)
-
-        elif backend == "hdf5":
-            pass  # nothing to set up
-
-        elif backend == "memory":
-            pass  # nothing to set up
-
-        elif backend == "ascii":
-            pass  # nothing to set up
-
         elif backend == "uda":
             # Set IDSDEF_PATH to point the UDA backend to the selected DD version
             idsdef_path = None
@@ -193,7 +140,6 @@ class ALDBEntryImpl(DBEntryImpl):
             if factory._xml_path is not None:
                 # Factory was constructed with an explicit XML path, point UDA to that:
                 idsdef_path = factory._xml_path
-
             elif "IMAS_PREFIX" in os.environ:
                 # Check if UDA can use the IDSDef.xml stored in $IMAS_PREFIX/include/
                 idsdef_path = os.environ["IMAS_PREFIX"] + "/include/IDSDef.xml"
@@ -203,15 +149,9 @@ class ALDBEntryImpl(DBEntryImpl):
             if idsdef_path is None:
                 # Extract XML from the DD zip and point UDA to it
                 idsdef_path = extract_idsdef(factory.version)
-
             os.environ["IDSDEF_PATH"] = idsdef_path
-            logger.warning(
-                "The UDA backend is not tested with "
-                "IMAS-Python and may not work properly. "
-                "Please raise any issues you find."
-            )
 
-        elif backend == "flexbuffers":
+        elif backend in ["hdf5", "memory", "ascii", "flexbuffers"]:
             pass  # nothing to set up
 
         else:
@@ -249,6 +189,8 @@ class ALDBEntryImpl(DBEntryImpl):
             raise RuntimeError("Database entry is not open.")
         if lazy and self.backend == "ascii":
             raise RuntimeError("Lazy loading is not supported by the ASCII backend.")
+        if self.backend == "uda":
+            self._check_uda_warnings(lazy)
 
         # Mixing contexts can be problematic, ensure all lazy contexts are closed:
         self._clear_lazy_ctx_cache()
@@ -339,6 +281,15 @@ class ALDBEntryImpl(DBEntryImpl):
         # Create a version conversion map, if needed
         nbc_map = None
         if ids._version != self._ids_factory._version:
+            if ids._version.split(".")[0] != self._ids_factory._version.split(".")[0]:
+                raise RuntimeError(
+                    f"Provided IDS uses DD {ids._version} which has a different major "
+                    f"version than the Data Entry ({self._ids_factory._version}). "
+                    "IMAS-Python will not automatically convert this data for you."
+                    "See the documentation for more details and fixes: "
+                    f"{imas.PUBLISHED_DOCUMENTATION_ROOT}"
+                    "/multi-dd.html#storing-idss-with-a-different-major-version"
+                )
             ddmap, source_is_older = dd_version_map_from_factories(
                 ids_name, ids._parent, self._ids_factory
             )
@@ -405,3 +356,28 @@ class ALDBEntryImpl(DBEntryImpl):
                 "Access Layer 5.1 or newer is required."
             ) from None
         return occurrence_list
+
+    def _check_uda_warnings(self, lazy: bool) -> None:
+        """Various checks / warnings for the UDA backend."""
+        cache_mode = self._querydict.get("cache_mode")
+        if lazy and cache_mode != "none":
+            # cache_mode=none requires imas core 5.5.1 or newer, and a recent enough UDA
+            # server plugin (which we cannot check...)
+            cache_mode_hint = ""
+            if ll_interface._al_version >= Version("5.5.1"):
+                cache_mode_hint = (
+                    "\nYou may add the parameter ';cache_mode=none' to the IMAS URI "
+                    "to avoid loading all of the data from the remote server."
+                )
+            logger.warning(
+                "The UDA backend will load all IDS data from the remote server. "
+                "Lazy loading with the UDA backend may therefore still be slow.%s",
+                cache_mode_hint,
+            )
+
+        if cache_mode == "none" and ll_interface._al_version < Version("5.5.1"):
+            logger.warning(
+                "UDA option 'cache_mode=None' may not work correctly with "
+                "IMAS Core version %s.",
+                ll_interface._al_version,
+            )

imas/backends/imas_core/imas_interface.py

@@ -6,6 +6,7 @@ Helper module for providing a version-independent interface to the Access Layer.
 This module tries to abstract away most API incompatibilities between the supported
 Access Layer versions (for example the rename of _ual_lowlevel to _al_lowlevel).
 """
+
 import inspect
 import logging
 
@@ -31,7 +32,7 @@ except ImportError as exc:
     has_imas = False
     imasdef = None
     lowlevel = None
-    logger.critical(
+    logger.warning(
         "Could not import 'imas_core': %s. Some functionality is not available.",
         exc,
     )
@@ -61,9 +62,6 @@ class LowlevelInterface:
 
     - If the lowlevel drops methods, we need to update the implementation fo the method
       to provide a proper error message or a workaround.
-    - Renamed methods (if this will ever happen) are perhaps best handled in the
-      ``__init__`` by providing a mapping of new to old name, so far this was only
-      relevant for the ``ual_`` to ``al_`` rename.
     """
 
     def __init__(self, lowlevel):
@@ -84,23 +82,13 @@ class LowlevelInterface:
             # Introduced after 5.0.0
             self._al_version_str = self._lowlevel.get_al_version()
             self._al_version = Version(self._al_version_str)
-        elif hasattr(lowlevel, "al_read_data"):
-            # In AL 5.0.0, all `ual_` methods were renamed to `al_`
+        else:
             self._al_version_str = "5.0.0"
             self._al_version = Version(self._al_version_str)
-        else:
-            # AL 4, don't try to determine in more detail
-            self._al_version_str = "4.?.?"
-            self._al_version = Version("4")
-            public_methods.remove("close_pulse")
 
-        if self._al_version < Version("5"):
-            method_prefix = "ual_"
-        else:
-            method_prefix = "al_"
         # Overwrite all of our methods that are implemented in the lowlevel
         for method in public_methods:
-            ll_method = getattr(lowlevel, method_prefix + method, None)
+            ll_method = getattr(lowlevel, f"al_{method}", None)
             if ll_method is not None:
                 setattr(self, method, ll_method)
 
@@ -115,24 +103,10 @@ class LowlevelInterface:
             f"but the current version is {self._al_version_str}"
         )
 
-    # AL 4 lowlevel API
-
-    def begin_pulse_action(self, backendID, shot, run, user, tokamak, version):
-        # Removed in AL5, compatibility handled in DBEntry
-        raise LLInterfaceError(f"{__name__} is not implemented")
-
-    def open_pulse(self, pulseCtx, mode, options):
-        # Removed in AL5, compatibility handled in DBEntry
-        raise LLInterfaceError(f"{__name__} is not implemented")
-
     def close_pulse(self, pulseCtx, mode):
-        # options argument (mandatory in AL4) was removed in AL5
-        # This method is overwritten in AL5, but for AL4 we need to do this:
-        return lowlevel.ual_close_pulse(pulseCtx, mode, None)
+        raise LLInterfaceError(f"{__name__} is not implemented")
 
-    def begin_global_action(self, pulseCtx, dataobjectname, rwmode, datapath=""):
-        # datapath was added in AL5 to support more efficient partial_get in the
-        # UDA backend. TODO: figure out if this is useful for lazy loading.
+    def begin_global_action(self, pulseCtx, dataobjectname, rwmode, datapath):
         raise LLInterfaceError(f"{__name__} is not implemented")
 
     def begin_slice_action(self, pulseCtx, dataobjectname, rwmode, time, interpmode):

imas/backends/imas_core/mdsplus_model.py

@@ -364,25 +364,3 @@ def jTraverser_jar() -> Path:
         return jar_path
     else:
         raise MDSPlusModelError("jTraverser.jar not found. Is MDSplus-Java available?")
-
-
-def ensure_data_dir(user: str, tokamak: str, version: str, run: int) -> None:
-    """Ensure that a data dir exists with a similar algorithm that
-    the MDSplus backend uses to set the data path.
-    See also mdsplus_backend.cpp:751 (setDataEnv)"""
-    if user == "public":
-        if "IMAS_HOME" not in os.environ:
-            raise RuntimeError(
-                "Environment variable IMAS_HOME must be set to access "
-                "the public database."
-            )
-        dbdir = Path(os.environ["IMAS_HOME"]) / "shared" / "imasdb" / tokamak / version
-    elif user[0] == "/":
-        dbdir = Path(user) / tokamak / version
-    else:
-        dbdir = Path.home() / "public" / "imasdb" / tokamak / version
-
-    # Check subfolder based on run
-    assert 0 <= run <= 99_999
-    index = run // 10_000
-    (dbdir / str(index)).mkdir(parents=True, exist_ok=True)

imas/backends/netcdf/nc2ids.py

@@ -366,9 +366,12 @@ class LazyContext:
 
             if value is not None:
                 if isinstance(value, np.ndarray):
-                    # Convert the numpy array to a read-only view
-                    value = value.view()
-                    value.flags.writeable = False
+                    if value.ndim == 0:  # Unpack 0D numpy arrays:
+                        value = value.item()
+                    else:
+                        # Convert the numpy array to a read-only view
+                        value = value.view()
+                        value.flags.writeable = False
                 # NOTE: bypassing IDSPrimitive.value.setter logic
                 child._IDSPrimitive__value = value
 

imas/command/cli.py

@@ -102,7 +102,6 @@ def print_ids(uri, ids, occurrence, print_all):
     ids         Name of the IDS to print (e.g. "core_profiles").
     occurrence  Which occurrence to print (defaults to 0).
     """
-    min_version_guard(Version("5.0"))
     setup_rich_log_handler(False)
 
     with DBEntry(uri, "r") as dbentry:

imas/command/db_analysis.py

@@ -7,6 +7,8 @@ import platform
 import re
 import readline
 import sys
+from csv import writer as csvwriter
+from collections import Counter, defaultdict
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional
@@ -139,12 +141,70 @@ def ids_info(idsfile: Path):
     }
 
 
+@dataclass
+class _PathUsage:
+    num_occurrences: int = 0
+    path_counter: Counter = field(default_factory=Counter)
+
+
+def _write_usage_stats_to_csv(
+    writer, usage_per_entry, usage_per_occurrence, num_entries
+):
+    """Write usage statistics to csv file.
+
+    Args:
+        writer: an instance of csv.writer
+        usage_per_entry: path usage statistics per data entry
+        usage_per_occurrence: path usage statistics per occurrence
+        num_entries: number of data entries
+    """
+    # Write header
+    writer.writerow(
+        [
+            "IDS",
+            "Path in IDS",
+            "Uses errorbar",
+            "Frequency (without occurrences)",
+            "Frequency (with occurences)",
+        ]
+    )
+
+    for ids_name in sorted(usage_per_entry):
+        entry_usage = usage_per_entry[ids_name]
+        occurrence_usage = usage_per_occurrence[ids_name]
+
+        # Usage statistics of the IDS (# entries with this IDS / # entries)
+        freq = entry_usage.num_occurrences / num_entries
+        writer.writerow([ids_name, "", "", freq, ""])
+
+        for path, entry_count in sorted(entry_usage.path_counter.items()):
+            if "_error_" in path:
+                continue  # Skip error nodes
+            occurrence_count = occurrence_usage.path_counter[path]
+
+            uses_error = f"{path}_error_upper" in entry_usage.path_counter
+            # Frequency without occurrences, see GH#84 for details
+            freq1 = entry_count / entry_usage.num_occurrences
+            # Frequency with occurences
+            freq2 = occurrence_count / occurrence_usage.num_occurrences
+
+            # Write data row
+            writer.writerow([ids_name, path, "X" if uses_error else "", freq1, freq2])
+
+
+_csv_help_text = (
+    "Write analysis output to the provided CSV file. For details, "
+    "see https://github.com/iterorganization/IMAS-Python/issues/84."
+)
+
+
 @click.command("process-db-analysis")
 @click.argument(
     "infiles", metavar="INPUT_FILES...", nargs=-1, type=infile_path, required=True
 )
 @click.option("--show-empty-ids", is_flag=True, help="Show empty IDSs in the overview.")
-def process_db_analysis(infiles, show_empty_ids):
+@click.option("--csv", type=outfile_path, help=_csv_help_text)
+def process_db_analysis(infiles, show_empty_ids, csv):
     """Process supplied Data Entry analyses, and display statistics.
 
     \b
@@ -153,9 +213,10 @@ def process_db_analysis(infiles, show_empty_ids):
     """
     setup_rich_log_handler(False)
 
-    factory = imas.IDSFactory()
-    filled_per_ids = {ids_name: set() for ids_name in factory.ids_names()}
-    logger.info("Using Data Dictionary version %s.", factory.dd_version)
+    usage_per_entry = defaultdict(_PathUsage)
+    usage_per_occurrence = defaultdict(_PathUsage)
+    num_entries = 0
+
     logger.info("Reading %d input files...", len(infiles))
 
     # Read input data and collate usage info per IDS
@@ -164,17 +225,51 @@ def process_db_analysis(infiles, show_empty_ids):
             data = json.load(file)
 
         for entry in data:
+            usage_for_this_entry = defaultdict(_PathUsage)
             for ids_info in entry["ids_info"]:
-                fill_info = filled_per_ids[ids_info["name"]]
-                fill_info.update(ids_info["filled_data"])
+                ids_name = ids_info["name"]
+                filled_paths = ids_info["filled_data"]
+                # Update counters for this entry
+                usage_for_this_entry[ids_name].path_counter.update(filled_paths)
+                # Update counters for all occurrecnes
+                usage_per_occurrence[ids_name].num_occurrences += 1
+                usage_per_occurrence[ids_name].path_counter.update(filled_paths)
+            # Update data entry usage
+            for ids_name, usage in usage_for_this_entry.items():
+                usage_per_entry[ids_name].num_occurrences += 1
+                usage_per_entry[ids_name].path_counter.update(usage.path_counter.keys())
+            num_entries += 1
 
     logger.info("Done reading input files.")
+
+    if csv is not None:
+        # Output to CSV file
+        logger.info("Writing output to CSV file: %s", csv)
+        with open(csv, "w") as csvfile:
+            writer = csvwriter(csvfile)
+            _write_usage_stats_to_csv(
+                writer, usage_per_entry, usage_per_occurrence, num_entries
+            )
+        logger.info("Done.")
+        return
+
     logger.info("Analyzing filled data...")
+    factory = imas.IDSFactory()
+    logger.info("Using Data Dictionary version %s.", factory.dd_version)
 
     # Construct AnalysisNodes per IDS
     analysis_nodes: Dict[str, _AnalysisNode] = {}
-    for ids_name, filled in filled_per_ids.items():
+    for ids_name, usage in usage_per_occurrence.items():
+        if ids_name not in factory.ids_names():
+            logger.warning(
+                "Founds IDS %s in data files, but this IDS is not available "
+                "in DD version %s. Statistics will not be tracked.",
+                ids_name,
+                factory.dd_version,
+            )
+            continue
         metadata = factory.new(ids_name).metadata
+        filled = set(usage.path_counter.keys())
         ids_analysis_node = _AnalysisNode("")
 
         def walk_metadata_tree(metadata: IDSMetadata, node: _AnalysisNode):

imas/command/helpers.py

@@ -36,7 +36,7 @@ def min_version_guard(al_version: Version):
         al_version: Minimum imas_core version required for this command.
     """
     used_version = ll_interface._al_version
-    if used_version >= al_version:
+    if used_version and used_version >= al_version:
         return
     click.echo(
         f"This command requires at least version {al_version} of the Access Layer."