imas-python 0.1.dev1628__tar.gz → 0.1.dev1630__tar.gz

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/.gitignore

@@ -91,10 +91,7 @@ ENV/
 *.swo
 
 # SCM setuptools
-imas/version.py
-
-# Saxon symlink or downloaded file
-saxon*.jar
+imas/_version.py
 
 # IMAS DD
 data-dictionary

imas_python-0.1.dev1630/.readthedocs.yml

@@ -0,0 +1,23 @@
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+  jobs:
+    post_checkout:
+      - git fetch --unshallow || true
+    
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs
+        - netcdf
+        - h5py
+
+sphinx:
+  builder: html
+  configuration: docs/source/conf.py
+  fail_on_warning: false

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.2
 Name: imas-python
-Version: 0.1.dev1628
+Version: 0.1.dev1630
 Summary: Pythonic wrappers for the IMAS Access Layer
-Author: imas-python Developers
+Author: imas-python developers
 Author-email: Olivier Hoenen <olivier.hoenen@iter.org>
 License:                   GNU LESSER GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -212,6 +212,8 @@ Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
+Requires-Dist: saxonche
+Requires-Dist: gitpython
 Provides-Extra: all
 Requires-Dist: imas[docs,h5py,imas-core,netcdf,test]; extra == "all"
 Provides-Extra: docs
@@ -247,7 +249,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/README.md

@@ -11,7 +11,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/docs/source/ci_config.rst

@@ -76,8 +76,8 @@ Deployment projects
 There are two Bamboo deployment projects for imas-python:
 
 `Deploy imas-python-doc <https://ci.iter.org/deploy/viewDeploymentProjectEnvironments.action?id=1784709122>`_
-    Deploy the documentation created in the `Build docs and dists` job to `Sharepoint
-    <https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/imas-python-doc/index.html>`_.
+    Deploy the documentation created in the `Build docs and dists` job to `readthedocs
+    <https://imas-python.readthedocs.io/en/latest/>`_.
 
     This deployment project runs for after each successful CI build of the imas-python main
     branch.

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/docs/source/imas_architecture.rst

@@ -1,4 +1,4 @@
-imas-python Architecture
+imas-python architecture
 ========================
 
 This document provides a brief overview of the components of imas-python, grouped into

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/docs/source/index.rst

@@ -3,7 +3,7 @@
    define TOC here, but it'll be put in the sidebar by the theme
 
 ==================
-imas-python Manual
+imas-python manual
 ==================
 
 imas-python is a pure-python library to handle arbitrarily nested

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas/__init__.py

@@ -25,10 +25,7 @@ from . import (
     util,
 )
 
-PUBLISHED_DOCUMENTATION_ROOT = (
-    "https://sharepoint.iter.org/departments/POP/CM/IMDesign/"
-    "Code%20Documentation/imas-python-doc"
-)
+PUBLISHED_DOCUMENTATION_ROOT = "https://imas-python.readthedocs.io/en/latest/"
 """URL to the published documentation."""
 OLDEST_SUPPORTED_VERSION = _V("3.22.0")
 """Oldest Data Dictionary version that is supported by imas-python."""

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.dev1628'
-__version_tuple__ = version_tuple = (0, 1, 'dev1628')
+__version__ = version = '0.1.dev1630'
+__version_tuple__ = version_tuple = (0, 1, 'dev1630')

imas_python-0.1.dev1630/imas/dd_helpers.py

@@ -0,0 +1,169 @@
+# This file is part of imas-python.
+# You should have received the imas-python LICENSE file with this project.
+"""Helper functions to build IDSDef.xml"""
+
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Tuple
+from zipfile import ZIP_DEFLATED, ZipFile
+
+from packaging.version import Version as V
+from saxonche import PySaxonProcessor
+
+logger = logging.getLogger(__name__)
+
+_idsdef_zip_relpath = Path("imas/assets/IDSDef.zip")
+_build_dir = Path("build")
+
+
+def prepare_data_dictionaries():
+    """Build IMAS IDSDef.xml files for each tagged version in the DD repository
+    1. Use saxonche for transformations
+    2. Clone the DD repository (ask for user/pass unless ssh key access is available)
+    3. Generate IDSDef.xml and rename to IDSDef_${version}.xml
+    4. Zip all these IDSDefs together and include in wheel
+    """
+    print("prepare_data_dictionaries")
+    from git import Repo
+
+    repo: Repo = get_data_dictionary_repo()
+    if repo:
+        newest_version_and_tag = (V("0"), None)
+        for tag in repo.tags:
+            version_and_tag = (V(str(tag)), tag)
+            if V(str(tag)) > V("3.21.1"):
+                newest_version_and_tag = max(newest_version_and_tag, version_and_tag)
+                logger.debug("Building data dictionary version %s", tag)
+                build_data_dictionary(repo, tag)
+
+        logger.info("Creating zip file of DD versions")
+
+        if _idsdef_zip_relpath.is_file():
+            logger.warning("Overwriting '%s'", _idsdef_zip_relpath)
+
+        with ZipFile(
+            _idsdef_zip_relpath,
+            mode="w",  # this needs w, since zip can have multiple same entries
+            compression=ZIP_DEFLATED,
+        ) as dd_zip:
+            for filename in _build_dir.glob("[0-9]*.xml"):
+                arcname = Path("data-dictionary").joinpath(*filename.parts[1:])
+                dd_zip.write(filename, arcname=arcname)
+            # Include identifiers from latest tag in zip file
+            repo.git.checkout(newest_version_and_tag[1], force=True)
+            # DD layout <= 4.0.0
+            for filename in Path("data-dictionary").glob("*/*identifier.xml"):
+                arcname = Path("identifiers").joinpath(*filename.parts[1:])
+                dd_zip.write(filename, arcname=arcname)
+            # DD layout > 4.0.0
+            for filename in Path("data-dictionary").glob("schemas/*/*identifier.xml"):
+                arcname = Path("identifiers").joinpath(*filename.parts[2:])
+                dd_zip.write(filename, arcname=arcname)
+
+
+def get_data_dictionary_repo() -> Tuple[bool, bool]:
+    try:
+        import git  # Import git here, the user might not have it!
+    except ModuleNotFoundError:
+        raise RuntimeError(
+            "Could not find 'git' module, try 'pip install gitpython'. \
+            Will not build Data Dictionaries!"
+        )
+
+        # We need the actual source code (for now) so grab it from ITER
+    dd_repo_path = "data-dictionary"
+
+    if "DD_DIRECTORY" in os.environ:
+        logger.info("Found DD_DIRECTORY, copying")
+        try:
+            shutil.copytree(os.environ["DD_DIRECTORY"], dd_repo_path)
+        except FileExistsError:
+            pass
+    else:
+        logger.info("Trying to pull data dictionary git repo from ITER")
+
+    # Set up a bare repo and fetch the data-dictionary repository in it
+    os.makedirs(dd_repo_path, exist_ok=True)
+    try:
+        repo = git.Repo(dd_repo_path)
+    except git.exc.InvalidGitRepositoryError:
+        repo = git.Repo.init(dd_repo_path)
+    logger.info("Set up local git repository {!s}".format(repo))
+
+    try:
+        origin = repo.remote()
+    except ValueError:
+        dd_repo_url = "https://github.com/iterorganization/imas-data-dictionary.git"
+        origin = repo.create_remote("origin", url=dd_repo_url)
+    logger.info("Set up remote '{!s}' linking to '{!s}'".format(origin, origin.url))
+
+    try:
+        origin.fetch(tags=True)
+    except git.exc.GitCommandError as ee:
+        logger.warning(
+            "Could not fetch tags from %s. Git reports:\n %s." "\nTrying to continue",
+            list(origin.urls),
+            ee,
+        )
+    else:
+        logger.info("Remote tags fetched")
+    return repo
+
+
+def _run_xsl_transformation(
+    xsd_file: Path, xsl_file: Path, tag: str, output_file: Path
+) -> None:
+    """
+    This function performs an XSL transformation using Saxon-HE (saxonche)
+    with the provided XSD file,  XSL file, tag, and output file.
+
+    Args:
+        xsd_file (Path): XML Schema Definition (XSD) file
+        xsl_file (Path): The `xsl_file` parameter
+        tag (str): tag name to provide to 'DD_GIT_DESCRIBE' parameter
+        output_file (Path): The `output_file` parameter for resulting xml
+    """
+    with PySaxonProcessor(license=False) as proc:
+        logger.debug("Initializing Saxon Processor")
+        xsltproc = proc.new_xslt30_processor()
+        xdm_ddgit = proc.make_string_value(tag)
+        xsltproc.set_parameter("DD_GIT_DESCRIBE", xdm_ddgit)
+        xsltproc.transform_to_file(
+            source_file=str(xsd_file),
+            stylesheet_file=str(xsl_file),
+            output_file=str(output_file),
+        )
+        logger.info("Transformation complete: %s -> %s", xsd_file, output_file)
+
+
+def build_data_dictionary(repo, tag: str, rebuild=False) -> None:
+    """Build a single version of the data dictionary given by the tag argument
+    if the IDS does not already exist.
+
+    In the data-dictionary repository sometimes IDSDef.xml is stored
+    directly, in which case we do not call make.
+
+    Args:
+        repo: Repository object containing the DD source code
+        tag: The DD version tag that will be build
+        rebuild: If true, overwrites existing pre-build tagged DD version
+    """
+    _build_dir.mkdir(exist_ok=True)
+    result_xml = _build_dir / f"{tag}.xml"
+
+    if result_xml.exists() and not rebuild:
+        logger.debug(f"XML for tag '{tag}' already exists, skipping")
+        return
+
+    repo.git.checkout(tag, force=True)
+
+    # Perform the XSL transformation with saxonche
+    dd_xsd = Path("data-dictionary/dd_data_dictionary.xml.xsd")
+    dd_xsl = Path("data-dictionary/dd_data_dictionary.xml.xsl")
+    _run_xsl_transformation(dd_xsd, dd_xsl, tag.name, result_xml)
+
+
+if __name__ == "__main__":
+    prepare_data_dictionaries()

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas/ids_structure.py

@@ -252,8 +252,8 @@ class IDSStructure(IDSBase):
                 "Iterating over non-empty nodes of a lazy loaded IDS will skip nodes "
                 "that are not loaded. Set accept_lazy=True to continue. "
                 "See the documentation for more information: "
-                "https://sharepoint.iter.org/departments/POP/CM/IMDesign/"
-                "Code%20Documentation/imas-python-doc/generated/imas.ids_structure."
+                "https://imas-python.readthedocs.io/en/latest"
+                "/generated/imas.ids_structure."
                 "IDSStructure.html#imas.ids_structure.IDSStructure.iter_nonempty_"
             )
         for child in self._children:

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas_python.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.2
 Name: imas-python
-Version: 0.1.dev1628
+Version: 0.1.dev1630
 Summary: Pythonic wrappers for the IMAS Access Layer
-Author: imas-python Developers
+Author: imas-python developers
 Author-email: Olivier Hoenen <olivier.hoenen@iter.org>
 License:                   GNU LESSER GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
@@ -212,6 +212,8 @@ Requires-Dist: click
 Requires-Dist: importlib_resources; python_version < "3.9"
 Requires-Dist: packaging
 Requires-Dist: xxhash>=2
+Requires-Dist: saxonche
+Requires-Dist: gitpython
 Provides-Extra: all
 Requires-Dist: imas[docs,h5py,imas-core,netcdf,test]; extra == "all"
 Provides-Extra: docs
@@ -247,7 +249,7 @@ Data Model.
 Install steps are described in the documentation generated from `/docs/source/installing.rst`.
 
 Documentation is autogenerated from the source using [Sphinx](http://sphinx-doc.org/)
-and can be found at the [ITER sharepoint](https://sharepoint.iter.org/departments/POP/CM/IMDesign/Code%20Documentation/IMAS-doc/index.html)
+and can be found at the [readthedocs](https://imas-python.readthedocs.io/en/latest/)
 
 The documentation can be manually generated by installing sphinx and running:
 

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas_python.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@
 .gitattributes
 .gitignore
 .pylintrc
+.readthedocs.yml
 CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 LICENSE.txt

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/imas_python.egg-info/requires.txt

@@ -4,6 +4,8 @@ scipy
 click
 packaging
 xxhash>=2
+saxonche
+gitpython
 
 [:python_version < "3.9"]
 importlib_resources

{imas_python-0.1.dev1628 → imas_python-0.1.dev1630}/pyproject.toml

@@ -1,15 +1,14 @@
 [build-system]
 # Minimum requirements for the build system to execute.
 # Keep this on a single line for the grep magic of build scripts to work
-requires = ["setuptools>=61", "wheel", "numpy", "gitpython", "packaging", "tomli;python_version<'3.11'", "setuptools_scm>8"]
-# needs saxon9he.jar in CLASSPATH
+requires = ["setuptools>=61", "wheel", "numpy", "gitpython", "saxonche","packaging", "tomli;python_version<'3.11'", "setuptools_scm>8"]
 
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "imas-python"
 authors = [
-  {name = "imas-python Developers"},
+  {name = "imas-python developers"},
   {name = "Olivier Hoenen", email = "olivier.hoenen@iter.org"},
 ]
 description = "Pythonic wrappers for the IMAS Access Layer"
@@ -64,6 +63,8 @@ dependencies = [
     "importlib_resources;python_version<'3.9'",
     "packaging",
     "xxhash >= 2",
+    "saxonche",
+    "gitpython"
 ]
 
 [project.optional-dependencies]
@@ -78,6 +79,7 @@ docs = [
     "sphinx-click",
 ]
 
+# TODO enable when imas-core is available on pypi
 # imas-core = [ "imas-core@git+https://github.com/iterorganization/imas-core.git@main" ]
 
 netcdf = [