pynxtools 0.2.0__tar.gz → 0.3.0__tar.gz

{pynxtools-0.2.0 → pynxtools-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pynxtools
-Version: 0.2.0
+Version: 0.3.0
 Summary: Extend NeXus for experiments and characterization in Materials Science and Materials Engineering and serve as a NOMAD parser implementation for NeXus.
 Author: The NOMAD Authors
 License: 
@@ -228,6 +228,9 @@ Requires-Dist: pandas>=1.3.2
 Requires-Dist: ase>=3.19.0
 Requires-Dist: mergedeep
 Requires-Dist: importlib-metadata
+Requires-Dist: lxml>=4.9.1
+Requires-Dist: anytree
+Requires-Dist: pydantic
 Provides-Extra: docs
 Requires-Dist: mkdocs; extra == "docs"
 Requires-Dist: mkdocs-material; extra == "docs"
@@ -235,7 +238,7 @@ Requires-Dist: mkdocs-material-extensions; extra == "docs"
 Requires-Dist: mkdocs-macros-plugin; extra == "docs"
 Provides-Extra: dev
 Requires-Dist: mypy; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
+Requires-Dist: ruff==0.3.4; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-timeout; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -246,7 +249,11 @@ Requires-Dist: types-requests; extra == "dev"
 Requires-Dist: pip-tools; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Provides-Extra: convert
-Requires-Dist: pynxtools[mpes,stm,xps,xrd]; extra == "convert"
+Requires-Dist: pynxtools[apm,ellips,em,mpes,stm,xps,xrd]; extra == "convert"
+Provides-Extra: apm
+Requires-Dist: pynxtools-apm; extra == "apm"
+Provides-Extra: em
+Requires-Dist: pynxtools-em; extra == "em"
 Provides-Extra: mpes
 Requires-Dist: pynxtools-mpes; extra == "mpes"
 Provides-Extra: xps
@@ -255,6 +262,8 @@ Provides-Extra: stm
 Requires-Dist: pynxtools-stm; extra == "stm"
 Provides-Extra: xrd
 Requires-Dist: pynxtools-xrd; extra == "xrd"
+Provides-Extra: ellips
+Requires-Dist: pynxtools-ellips; extra == "ellips"
 
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 ![](https://github.com/FAIRmat-NFDI/pynxtools/actions/workflows/pytest.yml/badge.svg)
@@ -310,28 +319,6 @@ data into the NeXus standard and visualising the files content.
 # Documentation
 Documentation for the different tools can be found [here](https://fairmat-nfdi.github.io/pynxtools/).
 
-# Plugins
-There are a number of plugins available for pynxtools. These are extensions of pynxtools used for reading data of specialized experimental techniques.
-- [**pynxtools-mpes**](https://github.com/FAIRmat-NFDI/pynxtools-mpes): A reader for multi-dimensional photoelectron spectroscopy data.
-- [**pynxtools-stm**](https://github.com/FAIRmat-NFDI/pynxtools-stm): A reader for scanning tunneling microscopy (SPM) and spectroscopy (STS) data.
-- [**pynxtools-xps**](https://github.com/FAIRmat-NFDI/pynxtools-xps): A reader for X-ray photoelectron spectroscopy (XPS) data.
-
-Respective readers for the research fields of electron microscopy and atom probe are currently refactored into pynxtools plugins.
-Until this refactoring will have become completed, users are advised to use the apm and em readers via pynxtools<=0.1.1.
-- [**pynxtools-apm**](https://github.com/FAIRmat-NFDI/pynxtools-apm): A reader for atom probe as well as related field ion microscopy data.
-- [**pynxtools-em**](https://github.com/FAIRmat-NFDI/pynxtools-em): A reader for electron microscopy data.
-
-You can install each of the plugins together with `pynxtools` by passing the name of the plugin as an extra to the pip install call. For example, for the `pynxtools-mpes` plugin:
-```shell
-pip install pynxtools[mpes]
-```
-
-In addition, you can also install all of the `pynxtools` reader plugins which are maintained by FAIRmat by passing the `[convert]` extra to the pip install call:
-```shell
-pip install pynxtools[convert]
-```
-There is also a [cookiecutter template](https://github.com/FAIRmat-NFDI/pynxtools-plugin-template) available for creating your own pynxtools plugin.
-
 # Contributing
 
 ## Development install

{pynxtools-0.2.0 → pynxtools-0.3.0}/README.md

@@ -52,28 +52,6 @@ data into the NeXus standard and visualising the files content.
 # Documentation
 Documentation for the different tools can be found [here](https://fairmat-nfdi.github.io/pynxtools/).
 
-# Plugins
-There are a number of plugins available for pynxtools. These are extensions of pynxtools used for reading data of specialized experimental techniques.
-- [**pynxtools-mpes**](https://github.com/FAIRmat-NFDI/pynxtools-mpes): A reader for multi-dimensional photoelectron spectroscopy data.
-- [**pynxtools-stm**](https://github.com/FAIRmat-NFDI/pynxtools-stm): A reader for scanning tunneling microscopy (SPM) and spectroscopy (STS) data.
-- [**pynxtools-xps**](https://github.com/FAIRmat-NFDI/pynxtools-xps): A reader for X-ray photoelectron spectroscopy (XPS) data.
-
-Respective readers for the research fields of electron microscopy and atom probe are currently refactored into pynxtools plugins.
-Until this refactoring will have become completed, users are advised to use the apm and em readers via pynxtools<=0.1.1.
-- [**pynxtools-apm**](https://github.com/FAIRmat-NFDI/pynxtools-apm): A reader for atom probe as well as related field ion microscopy data.
-- [**pynxtools-em**](https://github.com/FAIRmat-NFDI/pynxtools-em): A reader for electron microscopy data.
-
-You can install each of the plugins together with `pynxtools` by passing the name of the plugin as an extra to the pip install call. For example, for the `pynxtools-mpes` plugin:
-```shell
-pip install pynxtools[mpes]
-```
-
-In addition, you can also install all of the `pynxtools` reader plugins which are maintained by FAIRmat by passing the `[convert]` extra to the pip install call:
-```shell
-pip install pynxtools[convert]
-```
-There is also a [cookiecutter template](https://github.com/FAIRmat-NFDI/pynxtools-plugin-template) available for creating your own pynxtools plugin.
-
 # Contributing
 
 ## Development install

{pynxtools-0.2.0 → pynxtools-0.3.0}/dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml
@@ -8,9 +8,9 @@ ase==3.22.1
     # via pynxtools (pyproject.toml)
 babel==2.14.0
     # via mkdocs-material
-build==1.0.3
+build==1.1.1
     # via pip-tools
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -26,51 +26,49 @@ click-default-group==1.2.4
     # via pynxtools (pyproject.toml)
 colorama==0.4.6
     # via mkdocs-material
-contourpy==1.1.1
+contourpy==1.2.0
     # via matplotlib
-coverage[toml]==7.3.2
-    # via
-    #   coverage
-    #   pytest-cov
+coverage[toml]==7.4.4
+    # via pytest-cov
 cycler==0.12.1
     # via matplotlib
-distlib==0.3.7
+distlib==0.3.8
     # via virtualenv
-exceptiongroup==1.2.0
-    # via pytest
-filelock==3.13.1
+filelock==3.13.3
     # via virtualenv
-fonttools==4.45.1
+fonttools==4.50.0
     # via matplotlib
 ghp-import==2.1.0
     # via mkdocs
 h5py==3.10.0
     # via pynxtools (pyproject.toml)
-identify==2.5.33
+identify==2.5.35
     # via pre-commit
 idna==3.6
     # via requests
-importlib-metadata==6.8.0
+importlib-metadata==7.1.0
     # via pynxtools (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   mkdocs
     #   mkdocs-macros-plugin
     #   mkdocs-material
 kiwisolver==1.4.5
     # via matplotlib
-markdown==3.5.2
+lxml==5.1.0
+    # via pynxtools (pyproject.toml)
+markdown==3.6
     # via
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
-markupsafe==2.1.3
+markupsafe==2.1.5
     # via
     #   jinja2
     #   mkdocs
-matplotlib==3.7.4
+matplotlib==3.8.3
     # via ase
 mergedeep==1.3.4
     # via
@@ -83,19 +81,19 @@ mkdocs==1.5.3
     #   pynxtools (pyproject.toml)
 mkdocs-macros-plugin==1.0.5
     # via pynxtools (pyproject.toml)
-mkdocs-material==9.5.10
+mkdocs-material==9.5.15
     # via pynxtools (pyproject.toml)
 mkdocs-material-extensions==1.3.1
     # via
     #   mkdocs-material
     #   pynxtools (pyproject.toml)
-mypy==1.7.1
+mypy==1.9.0
     # via pynxtools (pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
 nodeenv==1.8.0
     # via pre-commit
-numpy==1.24.4
+numpy==1.26.4
     # via
     #   ase
     #   contourpy
@@ -105,7 +103,7 @@ numpy==1.24.4
     #   pynxtools (pyproject.toml)
     #   scipy
     #   xarray
-packaging==23.2
+packaging==24.0
     # via
     #   build
     #   matplotlib
@@ -114,48 +112,50 @@ packaging==23.2
     #   xarray
 paginate==0.5.6
     # via mkdocs-material
-pandas==2.0.3
+pandas==2.2.1
     # via
     #   pynxtools (pyproject.toml)
     #   xarray
 pathspec==0.12.1
     # via mkdocs
-pillow==10.0.1
+pillow==10.2.0
     # via matplotlib
-pip-tools==7.3.0
+pip-tools==7.4.1
     # via pynxtools (pyproject.toml)
-platformdirs==4.0.0
+platformdirs==4.2.0
     # via
     #   mkdocs
     #   virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
-pre-commit==3.5.0
+pre-commit==3.7.0
     # via pynxtools (pyproject.toml)
 pygments==2.17.2
     # via mkdocs-material
-pymdown-extensions==10.7
+pymdown-extensions==10.7.1
     # via mkdocs-material
-pyparsing==3.1.1
+pyparsing==3.1.2
     # via matplotlib
 pyproject-hooks==1.0.0
-    # via build
-pytest==7.4.3
+    # via
+    #   build
+    #   pip-tools
+pytest==8.1.1
     # via
     #   pynxtools (pyproject.toml)
     #   pytest-cov
     #   pytest-timeout
-pytest-cov==4.1.0
+pytest-cov==5.0.0
     # via pynxtools (pyproject.toml)
-pytest-timeout==2.2.0
+pytest-timeout==2.3.1
     # via pynxtools (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   ghp-import
     #   matplotlib
     #   mkdocs-macros-plugin
     #   pandas
-pytz==2023.3.post1
+pytz==2024.1
     # via pandas
 pyyaml==6.0.1
     # via
@@ -171,47 +171,39 @@ regex==2023.12.25
     # via mkdocs-material
 requests==2.31.0
     # via mkdocs-material
-ruff==0.1.7
+ruff==0.3.4
     # via pynxtools (pyproject.toml)
-scipy==1.9.1
+scipy==1.12.0
     # via ase
 six==1.16.0
     # via python-dateutil
-structlog==23.2.0
+structlog==24.1.0
     # via pynxtools (pyproject.toml)
 termcolor==2.4.0
     # via mkdocs-macros-plugin
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-types-pytz==2023.3.1.1
+types-pytz==2024.1.0.20240203
     # via pynxtools (pyproject.toml)
-types-pyyaml==6.0.12.12
+types-pyyaml==6.0.12.20240311
     # via pynxtools (pyproject.toml)
-types-requests==2.31.0.10
+types-requests==2.31.0.20240311
     # via pynxtools (pyproject.toml)
-typing-extensions==4.8.0
+typing-extensions==4.10.0
     # via mypy
-tzdata==2023.3
+tzdata==2024.1
     # via pandas
-urllib3==2.1.0
+urllib3==2.2.1
     # via
     #   requests
     #   types-requests
-virtualenv==20.25.0
+virtualenv==20.25.1
     # via pre-commit
 watchdog==4.0.0
     # via mkdocs
-wheel==0.42.0
+wheel==0.43.0
     # via pip-tools
-xarray==2023.1.0
+xarray==2024.2.0
     # via pynxtools (pyproject.toml)
-zipp==3.17.0
+zipp==3.18.1
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

{pynxtools-0.2.0 → pynxtools-0.3.0}/pynxtools/__init__.py

@@ -1,6 +1,3 @@
-"""init file
-
-"""
 #
 # Copyright The NOMAD Authors.
 #
@@ -18,11 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 import os
 import re
 from datetime import datetime
-from glob import glob
-from typing import Union
 
 from pynxtools._build_wrapper import get_vcs_version
 from pynxtools.definitions.dev_tools.globals.nxdl import get_nxdl_version

{pynxtools-0.2.0 → pynxtools-0.3.0}/pynxtools/_build_wrapper.py

@@ -2,6 +2,7 @@
 Build wrapper for setuptools to create a nexus-version.txt file
 containing the nexus definitions verison.
 """
+
 import os
 from subprocess import CalledProcessError, run
 from typing import Optional

pynxtools-0.3.0/pynxtools/dataconverter/__init__.py

@@ -0,0 +1,3 @@
+from pynxtools.dataconverter import helpers, validation
+
+helpers.validate_data_dict = validation.validate_data_dict  # type: ignore

{pynxtools-0.2.0 → pynxtools-0.3.0}/pynxtools/dataconverter/convert.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 """This script runs the conversion routine using a selected reader and write out a NeXus file."""
+
 import glob
 import importlib.machinery
 import importlib.util
@@ -23,18 +24,20 @@ import json
 import logging
 import os
 import sys
-import xml.etree.ElementTree as ET
 from gettext import gettext
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import List, Literal, Optional, Tuple
 
 import click
+import lxml.etree as ET
 import yaml
 from click_default_group import DefaultGroup
 
 from pynxtools.dataconverter import helpers
+from pynxtools.dataconverter.nexus_tree import generate_tree_from
 from pynxtools.dataconverter.readers.base.reader import BaseReader
 from pynxtools.dataconverter.template import Template
+from pynxtools.dataconverter.validation import validate_dict_against
 from pynxtools.dataconverter.writer import Writer
 from pynxtools.nexus import nexus
 
@@ -98,61 +101,11 @@ def get_names_of_all_readers() -> List[str]:
     return sorted(all_readers + plugins)
 
 
-def get_nxdl_root_and_path(nxdl: str):
-    """Get xml root element and file path from nxdl name e.g. NXapm.
-
-    Parameters
-    ----------
-    nxdl: str
-        Name of nxdl file e.g. NXapm from NXapm.nxdl.xml.
-
-    Returns
-    -------
-    ET.root
-        Root element of nxdl file.
-    str
-        Path of nxdl file.
-
-    Raises
-    ------
-    FileNotFoundError
-        Error if no file with the given nxdl name is found.
-    """
-    # Reading in the NXDL and generating a template
-    definitions_path = nexus.get_nexus_definitions_path()
-    if nxdl == "NXtest":
-        nxdl_f_path = os.path.join(
-            f"{os.path.abspath(os.path.dirname(__file__))}/../../",
-            "tests",
-            "data",
-            "dataconverter",
-            "NXtest.nxdl.xml",
-        )
-    elif nxdl == "NXroot":
-        nxdl_f_path = os.path.join(definitions_path, "base_classes", "NXroot.nxdl.xml")
-    else:
-        nxdl_f_path = os.path.join(
-            definitions_path, "contributed_definitions", f"{nxdl}.nxdl.xml"
-        )
-        if not os.path.exists(nxdl_f_path):
-            nxdl_f_path = os.path.join(
-                definitions_path, "applications", f"{nxdl}.nxdl.xml"
-            )
-        if not os.path.exists(nxdl_f_path):
-            nxdl_f_path = os.path.join(
-                definitions_path, "base_classes", f"{nxdl}.nxdl.xml"
-            )
-        if not os.path.exists(nxdl_f_path):
-            raise FileNotFoundError(f"The nxdl file, {nxdl}, was not found.")
-
-    return ET.parse(nxdl_f_path).getroot(), nxdl_f_path
-
-
 def transfer_data_into_template(
     input_file,
     reader,
     nxdl_name,
-    nxdl_root: Optional[ET.Element] = None,
+    nxdl_root: Optional[ET._Element] = None,
     skip_verify: bool = False,
     **kwargs,
 ):
@@ -181,7 +134,7 @@ def transfer_data_into_template(
 
     """
     if nxdl_root is None:
-        nxdl_root, _ = get_nxdl_root_and_path(nxdl=nxdl_name)
+        nxdl_root, _ = helpers.get_nxdl_root_and_path(nxdl=nxdl_name)
 
     template = Template()
     helpers.generate_template_from_nxdl(nxdl_root, template)
@@ -203,11 +156,24 @@ def transfer_data_into_template(
             "The chosen NXDL isn't supported by the selected reader."
         )
 
+    if "ignore_undocumented" in kwargs:
+        ignore_undocumented = kwargs["ignore_undocumented"]
+        del kwargs["ignore_undocumented"]
+    else:
+        ignore_undocumented = False
+
     data = data_reader().read(  # type: ignore[operator]
         template=Template(template), file_paths=input_file, **kwargs
     )
+    entry_names = data.get_all_entry_names()
+    for entry_name in entry_names:
+        helpers.write_nexus_def_to_entry(data, entry_name, nxdl_name)
     if not skip_verify:
-        helpers.validate_data_dict(template, data, nxdl_root)
+        validate_dict_against(
+            nxdl_name,
+            data,
+            ignore_undocumented=ignore_undocumented,
+        )
     return data
 
 
@@ -250,7 +216,7 @@ def convert(
     None.
     """
 
-    nxdl_root, nxdl_f_path = get_nxdl_root_and_path(nxdl)
+    nxdl_root, nxdl_f_path = helpers.get_nxdl_root_and_path(nxdl)
 
     data = transfer_data_into_template(
         input_file=input_file,
@@ -362,6 +328,12 @@ def main_cli():
     default=False,
     help="Shows a log output for all undocumented fields",
 )
+@click.option(
+    "--ignore-undocumented",
+    is_flag=True,
+    default=False,
+    help="Ignore all undocumented fields during validation.",
+)
 @click.option(
     "--skip-verify",
     is_flag=True,
@@ -382,6 +354,7 @@ def convert_cli(
     output: str,
     fair: bool,
     params_file: str,
+    ignore_undocumented: bool,
     undocumented: bool,
     skip_verify: bool,
     mapping: str,
@@ -422,15 +395,21 @@ def convert_cli(
             "The --input-file option is deprecated. Please use the positional arguments instead."
         )
 
-    convert(
-        tuple(file_list) + input_file,
-        reader,
-        nxdl,
-        output,
-        fair,
-        undocumented,
-        skip_verify,
-    )
+    try:
+        convert(
+            tuple(file_list) + input_file,
+            reader,
+            nxdl,
+            output,
+            fair,
+            undocumented,
+            skip_verify,
+            ignore_undocumented=ignore_undocumented,
+        )
+    except FileNotFoundError as exc:
+        raise click.BadParameter(
+            f"{nxdl} is not a valid application definition", param_hint="--nxdl"
+        ) from exc
 
 
 @main_cli.command()
@@ -463,21 +442,24 @@ def generate_template(nxdl: str, required: bool, pythonic: bool, output: str):
         f.write(text)
         f.close()
 
-    print_or_write = lambda txt: write_to_file(txt) if output else print(txt)
+    tree = generate_tree_from(nxdl)
 
-    nxdl_root, nxdl_f_path = get_nxdl_root_and_path(nxdl)
-    template = Template()
-    helpers.generate_template_from_nxdl(nxdl_root, template)
+    print_or_write = lambda txt: write_to_file(txt) if output else print(txt)
 
+    level: Literal["required", "recommended", "optional"] = "optional"
     if required:
-        template = Template(template.get_optionality("required"))
+        level = "required"
+    reqs = tree.required_fields_and_attrs_names(level=level)
+    template = {
+        helpers.convert_nxdl_path_dict_to_data_converter_dict(req): None for req in reqs
+    }
 
     if pythonic:
         print_or_write(str(template))
         return
     print_or_write(
         json.dumps(
-            template.get_accumulated_dict(),
+            template,
             indent=4,
             sort_keys=True,
             ensure_ascii=False,

{pynxtools-0.2.0 → pynxtools-0.3.0}/pynxtools/dataconverter/hdfdict.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 """Taken from: https://github.com/SiggiGue/hdfdict/blob/master/hdfdict/hdfdict.py"""
+
 from collections import UserDict
 from contextlib import contextmanager
 from datetime import datetime