imas-python 0.1.dev1629__tar.gz → 0.1.dev1631__tar.gz

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/.gitignore

@@ -91,17 +91,12 @@ ENV/
 *.swo
 
 # SCM setuptools
-imas/version.py
-
-# Saxon symlink or downloaded file
-saxon*.jar
+imas/_version.py
 
 # IMAS DD
 data-dictionary
-access-layer
 containers/arch/imas/
 containers/arch/data-dictionary/
-containers/arch/access-layer/
 imas/assets/IDSDef.zip
 
 # IDS files

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: imas-python
-Version: 0.1.dev1629
+Version: 0.1.dev1631
 Summary: Pythonic wrappers for the IMAS Access Layer
 Author: imas-python developers
 Author-email: Olivier Hoenen <olivier.hoenen@iter.org>
@@ -212,6 +212,8 @@ Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
+Requires-Dist: saxonche
+Requires-Dist: gitpython
 Provides-Extra: all
 Requires-Dist: imas[docs,h5py,imas-core,netcdf,test]; extra == "all"
 Provides-Extra: docs
@@ -247,7 +249,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/README.md

@@ -11,7 +11,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/benchmarks/technical.py

@@ -1,5 +1,4 @@
 import imas
-import imas
 
 
 def timeraw_create_default_imas_factory():
@@ -16,12 +15,6 @@ def timeraw_import_imas():
     """
 
 
-def timeraw_import_imas():
-    return """
-    import imas
-    """
-
-
 # It would be nice if we could track these, but unfortunately it breaks things like
 # `asv compare` :(
 """

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/benchmarks/utils.py

@@ -6,10 +6,6 @@ from pathlib import Path
 import imas
 import imas.exception
 
-# Don't directly import imas: code analyzers break on the huge code base
-imas = importlib.import_module("imas")
-
-
 # Backend constants
 HDF5 = "HDF5"
 MDSPLUS = "MDSplus"
@@ -60,13 +56,11 @@ available_slicing_backends = [
     backend for backend in available_backends if backend not in [ASCII, NETCDF]
 ]
 
-hlis = ["imas", "imas"]
+hlis = ["imas"]
 DBEntry = {
     "imas": imas.DBEntry,
-    "imas": imas.DBEntry,
 }
 factory = {
-    "imas": imas,
     "imas": imas.IDSFactory(),
 }
 available_serializers = [imas.ids_defs.ASCII_SERIALIZER_PROTOCOL]
@@ -74,9 +68,6 @@ available_serializers = [imas.ids_defs.ASCII_SERIALIZER_PROTOCOL]
 
 def create_dbentry(hli, backend):
     if backend == NETCDF:
-        if hli == "imas":
-            # Raising NotImplementedError will skip the benchmarks for this combination
-            raise NotImplementedError("AL-Python HLI doesn't implement netCDF.")
         if hli == "imas":  # check if netcdf backend is available
             try:
                 assert (

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/docs/source/ci_config.rst

@@ -76,8 +76,8 @@ Deployment projects
 There are two Bamboo deployment projects for imas-python:
 
 `Deploy imas-python-doc <https://ci.iter.org/deploy/viewDeploymentProjectEnvironments.action?id=1784709122>`_
-    Deploy the documentation created in the `Build docs and dists` job to `Sharepoint
-    <https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/imas-python-doc/index.html>`_.
+    Deploy the documentation created in the `Build docs and dists` job to `readthedocs
+    <https://imas-python.readthedocs.io/en/latest/>`_.
 
     This deployment project runs for after each successful CI build of the imas-python main
     branch.

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas/__init__.py

@@ -25,10 +25,7 @@ from . import (
     util,
 )
 
-PUBLISHED_DOCUMENTATION_ROOT = (
-    "https://sharepoint.iter.org/departments/POP/CM/IMDesign/"
-    "Code%20Documentation/imas-python-doc"
-)
+PUBLISHED_DOCUMENTATION_ROOT = "https://imas-python.readthedocs.io/en/latest/"
 """URL to the published documentation."""
 OLDEST_SUPPORTED_VERSION = _V("3.22.0")
 """Oldest Data Dictionary version that is supported by imas-python."""

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.dev1629'
-__version_tuple__ = version_tuple = (0, 1, 'dev1629')
+__version__ = version = '0.1.dev1631'
+__version_tuple__ = version_tuple = (0, 1, 'dev1631')

imas_python-0.1.dev1631/imas/dd_helpers.py

@@ -0,0 +1,169 @@
+# This file is part of imas-python.
+# You should have received the imas-python LICENSE file with this project.
+"""Helper functions to build IDSDef.xml"""
+
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Tuple
+from zipfile import ZIP_DEFLATED, ZipFile
+
+from packaging.version import Version as V
+from saxonche import PySaxonProcessor
+
+logger = logging.getLogger(__name__)
+
+_idsdef_zip_relpath = Path("imas/assets/IDSDef.zip")
+_build_dir = Path("build")
+
+
+def prepare_data_dictionaries():
+    """Build IMAS IDSDef.xml files for each tagged version in the DD repository
+    1. Use saxonche for transformations
+    2. Clone the DD repository (ask for user/pass unless ssh key access is available)
+    3. Generate IDSDef.xml and rename to IDSDef_${version}.xml
+    4. Zip all these IDSDefs together and include in wheel
+    """
+    print("prepare_data_dictionaries")
+    from git import Repo
+
+    repo: Repo = get_data_dictionary_repo()
+    if repo:
+        newest_version_and_tag = (V("0"), None)
+        for tag in repo.tags:
+            version_and_tag = (V(str(tag)), tag)
+            if V(str(tag)) > V("3.21.1"):
+                newest_version_and_tag = max(newest_version_and_tag, version_and_tag)
+                logger.debug("Building data dictionary version %s", tag)
+                build_data_dictionary(repo, tag)
+
+        logger.info("Creating zip file of DD versions")
+
+        if _idsdef_zip_relpath.is_file():
+            logger.warning("Overwriting '%s'", _idsdef_zip_relpath)
+
+        with ZipFile(
+            _idsdef_zip_relpath,
+            mode="w",  # this needs w, since zip can have multiple same entries
+            compression=ZIP_DEFLATED,
+        ) as dd_zip:
+            for filename in _build_dir.glob("[0-9]*.xml"):
+                arcname = Path("data-dictionary").joinpath(*filename.parts[1:])
+                dd_zip.write(filename, arcname=arcname)
+            # Include identifiers from latest tag in zip file
+            repo.git.checkout(newest_version_and_tag[1], force=True)
+            # DD layout <= 4.0.0
+            for filename in Path("data-dictionary").glob("*/*identifier.xml"):
+                arcname = Path("identifiers").joinpath(*filename.parts[1:])
+                dd_zip.write(filename, arcname=arcname)
+            # DD layout > 4.0.0
+            for filename in Path("data-dictionary").glob("schemas/*/*identifier.xml"):
+                arcname = Path("identifiers").joinpath(*filename.parts[2:])
+                dd_zip.write(filename, arcname=arcname)
+
+
+def get_data_dictionary_repo() -> Tuple[bool, bool]:
+    try:
+        import git  # Import git here, the user might not have it!
+    except ModuleNotFoundError:
+        raise RuntimeError(
+            "Could not find 'git' module, try 'pip install gitpython'. \
+            Will not build Data Dictionaries!"
+        )
+
+        # We need the actual source code (for now) so grab it from ITER
+    dd_repo_path = "data-dictionary"
+
+    if "DD_DIRECTORY" in os.environ:
+        logger.info("Found DD_DIRECTORY, copying")
+        try:
+            shutil.copytree(os.environ["DD_DIRECTORY"], dd_repo_path)
+        except FileExistsError:
+            pass
+    else:
+        logger.info("Trying to pull data dictionary git repo from ITER")
+
+    # Set up a bare repo and fetch the data-dictionary repository in it
+    os.makedirs(dd_repo_path, exist_ok=True)
+    try:
+        repo = git.Repo(dd_repo_path)
+    except git.exc.InvalidGitRepositoryError:
+        repo = git.Repo.init(dd_repo_path)
+    logger.info("Set up local git repository {!s}".format(repo))
+
+    try:
+        origin = repo.remote()
+    except ValueError:
+        dd_repo_url = "https://github.com/iterorganization/imas-data-dictionary.git"
+        origin = repo.create_remote("origin", url=dd_repo_url)
+    logger.info("Set up remote '{!s}' linking to '{!s}'".format(origin, origin.url))
+
+    try:
+        origin.fetch(tags=True)
+    except git.exc.GitCommandError as ee:
+        logger.warning(
+            "Could not fetch tags from %s. Git reports:\n %s." "\nTrying to continue",
+            list(origin.urls),
+            ee,
+        )
+    else:
+        logger.info("Remote tags fetched")
+    return repo
+
+
+def _run_xsl_transformation(
+    xsd_file: Path, xsl_file: Path, tag: str, output_file: Path
+) -> None:
+    """
+    This function performs an XSL transformation using Saxon-HE (saxonche)
+    with the provided XSD file,  XSL file, tag, and output file.
+
+    Args:
+        xsd_file (Path): XML Schema Definition (XSD) file
+        xsl_file (Path): The `xsl_file` parameter
+        tag (str): tag name to provide to 'DD_GIT_DESCRIBE' parameter
+        output_file (Path): The `output_file` parameter for resulting xml
+    """
+    with PySaxonProcessor(license=False) as proc:
+        logger.debug("Initializing Saxon Processor")
+        xsltproc = proc.new_xslt30_processor()
+        xdm_ddgit = proc.make_string_value(tag)
+        xsltproc.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)
+        xsltproc.transform_to_file(
+            source_file=str(xsd_file),
+            stylesheet_file=str(xsl_file),
+            output_file=str(output_file),
+        )
+        logger.info("Transformation complete: %s -> %s", xsd_file, output_file)
+
+
+def build_data_dictionary(repo, tag: str, rebuild=False) -> None:
+    """Build a single version of the data dictionary given by the tag argument
+    if the IDS does not already exist.
+
+    In the data-dictionary repository sometimes IDSDef.xml is stored
+    directly, in which case we do not call make.
+
+    Args:
+        repo: Repository object containing the DD source code
+        tag: The DD version tag that will be build
+        rebuild: If true, overwrites existing pre-build tagged DD version
+    """
+    _build_dir.mkdir(exist_ok=True)
+    result_xml = _build_dir / f"{tag}.xml"
+
+    if result_xml.exists() and not rebuild:
+        logger.debug(f"XML for tag '{tag}' already exists, skipping")
+        return
+
+    repo.git.checkout(tag, force=True)
+
+    # Perform the XSL transformation with saxonche
+    dd_xsd = Path("data-dictionary/dd_data_dictionary.xml.xsd")
+    dd_xsl = Path("data-dictionary/dd_data_dictionary.xml.xsl")
+    _run_xsl_transformation(dd_xsd, dd_xsl, tag.name, result_xml)
+
+
+if __name__ == "__main__":
+    prepare_data_dictionaries()

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas/ids_structure.py

@@ -252,8 +252,8 @@ class IDSStructure(IDSBase):
                 "Iterating over non-empty nodes of a lazy loaded IDS will skip nodes "
                 "that are not loaded. Set accept_lazy=True to continue. "
                 "See the documentation for more information: "
-                "https://sharepoint.iter.org/departments/POP/CM/IMDesign/"
-                "Code%20Documentation/imas-python-doc/generated/imas.ids_structure."
+                "https://imas-python.readthedocs.io/en/latest"
+                "/generated/imas.ids_structure."
                 "IDSStructure.html#imas.ids_structure.IDSStructure.iter_nonempty_"
             )
         for child in self._children:

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: imas-python
-Version: 0.1.dev1629
+Version: 0.1.dev1631
 Summary: Pythonic wrappers for the IMAS Access Layer
 Author: imas-python developers
 Author-email: Olivier Hoenen <olivier.hoenen@iter.org>
@@ -212,6 +212,8 @@ Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
+Requires-Dist: saxonche
+Requires-Dist: gitpython
 Provides-Extra: all
 Requires-Dist: imas[docs,h5py,imas-core,netcdf,test]; extra == "all"
 Provides-Extra: docs
@@ -247,7 +249,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas_python.egg-info/SOURCES.txt

@@ -14,7 +14,7 @@ conftest.py
 pyproject.toml
 setup.cfg
 setup.py
-.github/workflows/liniting.yml
+.github/workflows/linting.yml
 .github/workflows/publish.yml
 benchmarks/__init__.py
 benchmarks/core_profiles.py

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/imas_python.egg-info/requires.txt

@@ -4,6 +4,8 @@ scipy
 click
 packaging
 xxhash>=2
+saxonche
+gitpython
 
 [:python_version < "3.9"]
 importlib_resources

{imas_python-0.1.dev1629 → imas_python-0.1.dev1631}/pyproject.toml

@@ -1,8 +1,7 @@
 [build-system]
 # Minimum requirements for the build system to execute.
 # Keep this on a single line for the grep magic of build scripts to work
-requires = ["setuptools>=61", "wheel", "numpy", "gitpython", "packaging", "tomli;python_version<'3.11'", "setuptools_scm>8"]
-# needs saxon9he.jar in CLASSPATH
+requires = ["setuptools>=61", "wheel", "numpy", "gitpython", "saxonche","packaging", "tomli;python_version<'3.11'", "setuptools_scm>8"]
 
 build-backend = "setuptools.build_meta"
 
@@ -64,6 +63,8 @@ dependencies = [
     "importlib_resources;python_version<'3.9'",
     "packaging",
     "xxhash >= 2",
+    "saxonche",
+    "gitpython"
 ]
 
 [project.optional-dependencies]