pynxtools 0.3.0__tar.gz → 0.3.2__tar.gz

{pynxtools-0.3.0 → pynxtools-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pynxtools
-Version: 0.3.0
+Version: 0.3.2
 Summary: Extend NeXus for experiments and characterization in Materials Science and Materials Engineering and serve as a NOMAD parser implementation for NeXus.
 Author: The NOMAD Authors
 License: 
@@ -230,7 +230,6 @@ Requires-Dist: mergedeep
 Requires-Dist: importlib-metadata
 Requires-Dist: lxml>=4.9.1
 Requires-Dist: anytree
-Requires-Dist: pydantic
 Provides-Extra: docs
 Requires-Dist: mkdocs; extra == "docs"
 Requires-Dist: mkdocs-material; extra == "docs"
@@ -250,16 +249,16 @@ Requires-Dist: pip-tools; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Provides-Extra: convert
 Requires-Dist: pynxtools[apm,ellips,em,mpes,stm,xps,xrd]; extra == "convert"
+Provides-Extra: mpes
+Requires-Dist: pynxtools-mpes; extra == "mpes"
 Provides-Extra: apm
 Requires-Dist: pynxtools-apm; extra == "apm"
 Provides-Extra: em
 Requires-Dist: pynxtools-em; extra == "em"
-Provides-Extra: mpes
-Requires-Dist: pynxtools-mpes; extra == "mpes"
-Provides-Extra: xps
-Requires-Dist: pynxtools-xps; extra == "xps"
 Provides-Extra: stm
 Requires-Dist: pynxtools-stm; extra == "stm"
+Provides-Extra: xps
+Requires-Dist: pynxtools-xps; extra == "xps"
 Provides-Extra: xrd
 Requires-Dist: pynxtools-xrd; extra == "xrd"
 Provides-Extra: ellips

{pynxtools-0.3.0 → pynxtools-0.3.2}/dev-requirements.txt

@@ -1,9 +1,11 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml
 #
+anytree==2.12.1
+    # via pynxtools (pyproject.toml)
 ase==3.22.1
     # via pynxtools (pyproject.toml)
 babel==2.14.0
@@ -34,6 +36,8 @@ cycler==0.12.1
     # via matplotlib
 distlib==0.3.8
     # via virtualenv
+exceptiongroup==1.2.1
+    # via pytest
 filelock==3.13.3
     # via virtualenv
 fonttools==4.50.0
@@ -176,11 +180,21 @@ ruff==0.3.4
 scipy==1.12.0
     # via ase
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   anytree
+    #   python-dateutil
 structlog==24.1.0
     # via pynxtools (pyproject.toml)
 termcolor==2.4.0
     # via mkdocs-macros-plugin
+tomli==2.0.1
+    # via
+    #   build
+    #   coverage
+    #   mypy
+    #   pip-tools
+    #   pyproject-hooks
+    #   pytest
 types-pytz==2024.1.0.20240203
     # via pynxtools (pyproject.toml)
 types-pyyaml==6.0.12.20240311

{pynxtools-0.3.0 → pynxtools-0.3.2}/pynxtools/dataconverter/nexus_tree.py

@@ -28,11 +28,10 @@ on the fly when the tree is generated.
 It also allows for adding further nodes from the inheritance chain on the fly.
 """
 
-from typing import List, Literal, Optional, Set, Tuple, Union
+from typing import Any, List, Literal, Optional, Set, Tuple, Union
 
 import lxml.etree as ET
 from anytree.node.nodemixin import NodeMixin
-from pydantic import BaseModel, InstanceOf
 
 from pynxtools.dataconverter.helpers import (
     contains_uppercase,
@@ -97,7 +96,7 @@ NexusUnitCategory = Literal[
 namespaces = {"nx": "http://definition.nexusformat.org/nxdl/3.1"}
 
 
-class NexusNode(BaseModel, NodeMixin):
+class NexusNode(NodeMixin):
     """
     A NexusNode represents one node in the NeXus tree.
     It can be either a `group`, `field`, `attribute` or `choice` for which it has
@@ -140,8 +139,7 @@ class NexusNode(BaseModel, NodeMixin):
     type: Literal["group", "field", "attribute", "choice"]
     optionality: Literal["required", "recommended", "optional"] = "required"
     variadic: bool = False
-    variadic_siblings: List[InstanceOf["NexusNode"]] = []
-    inheritance: List[InstanceOf[ET._Element]] = []
+    inheritance: List[ET._Element]
 
     def _set_optionality(self):
         if not self.inheritance:
@@ -151,9 +149,26 @@ class NexusNode(BaseModel, NodeMixin):
         elif self.inheritance[0].attrib.get("optional"):
             self.optionality = "optional"
 
-    def __init__(self, parent: Optional["NexusNode"], **data) -> None:
-        super().__init__(**data)
+    def __init__(
+        self,
+        name: str,
+        type: Literal["group", "field", "attribute", "choice"],
+        optionality: Literal["required", "recommended", "optional"] = "required",
+        variadic: Optional[bool] = None,
+        parent: Optional["NexusNode"] = None,
+        inheritance: Optional[List[Any]] = None,
+    ) -> None:
+        super().__init__()
+        self.name = name
+        self.type = type
+        self.optionality = optionality
         self.variadic = contains_uppercase(self.name)
+        if variadic is not None:
+            self.variadic = variadic
+        if inheritance is not None:
+            self.inheritance = inheritance
+        else:
+            self.inheritance = []
         self.parent = parent
 
     def _construct_inheritance_chain_from_parent(self):
@@ -477,8 +492,9 @@ class NexusGroup(NexusNode):
             max_occurs,
         )
 
-    def __init__(self, **data) -> None:
+    def __init__(self, nx_class: str, **data) -> None:
         super().__init__(**data)
+        self.nx_class = nx_class
         self._set_occurence_limits()
         self._set_optionality()
 

{pynxtools-0.3.0 → pynxtools-0.3.2}/pynxtools/dataconverter/validation.py

@@ -606,7 +606,6 @@ def populate_full_tree(node: NexusNode, max_depth: Optional[int] = 5, depth: int
         # be fixed.
         return
     for child in node.get_all_children_names():
-        print(child)
         child_node = node.search_child_with_name(child)
         populate_full_tree(child_node, max_depth=max_depth, depth=depth + 1)
 

pynxtools-0.3.2/pynxtools/testing/nexus_conversion.py

@@ -0,0 +1,134 @@
+"""Generic test for reader plugins."""
+
+from typing import Literal
+
+import logging
+import os
+from glob import glob
+
+from pynxtools.dataconverter.helpers import get_nxdl_root_and_path
+from pynxtools.dataconverter.convert import get_reader, transfer_data_into_template
+from pynxtools.dataconverter.validation import validate_dict_against
+from pynxtools.dataconverter.writer import Writer
+from pynxtools.nexus import nexus
+
+
+def get_log_file(nxs_file, log_file, tmp_path):
+    """Get log file for the nexus file with read_nexus tools."""
+    logger = logging.getLogger(__name__)
+    logger.handlers.clear()
+    logger.setLevel(logging.DEBUG)
+    log_file = os.path.join(tmp_path, log_file)
+    handler = logging.FileHandler(log_file, "w")
+    formatter = logging.Formatter("%(levelname)s - %(message)s")
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    nexus_helper = nexus.HandleNexus(logger, nxs_file, None, None)
+    nexus_helper.process_nexus_master_file(None)
+    return log_file
+
+
+class ReaderTest:
+    """Generic test for reader plugins."""
+
+    def __init__(self, nxdl, reader_name, files_or_dir, tmp_path, caplog) -> None:
+        """Initialize the test object.
+
+        Parameters
+        ----------
+        nxdl : str
+            Name of the NXDL application definition that is to be tested by this reader plugin (e.g. NXsts, NXmpes, etc).
+        reader_name : str
+            The name of the reader class (e.g. stm, mpes, xps, ...) to be tested.
+        files_or_dir : str
+            List of input files or full path string to the example data directory that contains all the files
+            required for running the data conversion through the reader.
+        ref_nexus_file : str
+            Full path string to the reference NeXus file generated from the same
+            set of input files.
+        tmp_path : pathlib.PosixPath
+            Pytest fixture variable, used to clean up the files generated during the test.
+        caplog : _pytest.logging.LogCaptureFixture
+            Pytest fixture variable, used to capture the log messages during the test.
+        """
+
+        self.nxdl = nxdl
+        self.reader_name = reader_name
+        self.reader = get_reader(self.reader_name)
+        self.files_or_dir = files_or_dir
+        self.ref_nexus_file = ""
+        self.tmp_path = tmp_path
+        self.caplog = caplog
+        self.created_nexus = f"{tmp_path}/{os.sep}/output.nxs"
+
+    def convert_to_nexus(
+        self,
+        caplog_level: Literal["ERROR", "WARNING"] = "ERROR",
+        ignore_undocumented: bool = False,
+    ):
+        """
+        Test the example data for the reader plugin.
+        """
+        assert hasattr(
+            self.reader, "supported_nxdls"
+        ), f"Reader{self.reader} must have supported_nxdls attribute"
+        assert callable(self.reader.read), f"Reader{self.reader} must have read method"
+
+        if isinstance(self.files_or_dir, (list, tuple)):
+            example_files = self.files_or_dir
+        else:
+            example_files = sorted(glob(os.path.join(self.files_or_dir, "*")))
+        self.ref_nexus_file = [file for file in example_files if file.endswith(".nxs")][
+            0
+        ]
+        input_files = [file for file in example_files if not file.endswith(".nxs")]
+        assert self.ref_nexus_file, "Reference nexus (.nxs) file not found"
+        assert (
+            self.nxdl in self.reader.supported_nxdls
+        ), f"Reader does not support {self.nxdl} NXDL."
+
+        nxdl_root, nxdl_file = get_nxdl_root_and_path(self.nxdl)
+        assert os.path.exists(nxdl_file), f"NXDL file {nxdl_file} not found"
+
+        read_data = transfer_data_into_template(
+            input_file=input_files,
+            reader=self.reader_name,
+            nxdl_name=self.nxdl,
+            nxdl_root=nxdl_root,
+            skip_verify=True,
+        )
+
+        # Clear the log of `transfer_data_into_template`
+        self.caplog.clear()
+
+        with self.caplog.at_level(caplog_level):
+            _ = validate_dict_against(
+                self.nxdl, read_data, ignore_undocumented=ignore_undocumented
+            )
+        assert self.caplog.text == ""
+
+        Writer(read_data, nxdl_file, self.created_nexus).write()
+
+    def check_reproducibility_of_nexus(self):
+        """Reproducibility test for the generated nexus file."""
+        ref_log = get_log_file(self.ref_nexus_file, "ref_nexus.log", self.tmp_path)
+        gen_log = get_log_file(self.created_nexus, "gen_nexus.log", self.tmp_path)
+        with open(gen_log, "r", encoding="utf-8") as gen, open(
+            ref_log, "r", encoding="utf-8"
+        ) as ref:
+            gen_lines = gen.readlines()
+            ref_lines = ref.readlines()
+        if len(gen_lines) != len(ref_lines):
+            assert False, "Log files are different"
+        for ind, (gen_l, ref_l) in enumerate(zip(gen_lines, ref_lines)):
+            if gen_l != ref_l:
+                # skip version conflicts
+                if gen_l.startswith("DEBUG - value: v") and ref_l.startswith(
+                    "DEBUG - value: v"
+                ):
+                    continue
+                assert False, (
+                    f"Log files are different at line {ind}"
+                    f" generated: {gen_l} \n referenced : {ref_l}"
+                )

{pynxtools-0.3.0 → pynxtools-0.3.2}/pynxtools.egg-info/SOURCES.txt

@@ -402,4 +402,6 @@ pynxtools/eln_mapper/eln.py
 pynxtools/eln_mapper/eln_mapper.py
 pynxtools/eln_mapper/scheme_eln.py
 pynxtools/nexus/__init__.py
-pynxtools/nexus/nexus.py
+pynxtools/nexus/nexus.py
+pynxtools/testing/__init__.py
+pynxtools/testing/nexus_conversion.py

{pynxtools-0.3.0 → pynxtools-0.3.2}/pyproject.toml

@@ -35,7 +35,6 @@ dependencies = [
     "importlib-metadata",
     "lxml>=4.9.1",
     "anytree",
-    "pydantic",
 ]
 
 [project.urls]
@@ -65,21 +64,21 @@ dev = [
 convert = [
     "pynxtools[apm,em,mpes,xps,stm,xrd,ellips]",
 ]
+mpes = [
+    "pynxtools-mpes",
+]
 apm = [
     "pynxtools-apm",
 ]
 em = [
     "pynxtools-em",
 ]
-mpes = [
-    "pynxtools-mpes",
+stm = [
+    "pynxtools-stm",
 ]
 xps = [
     "pynxtools-xps",
 ]
-stm = [
-    "pynxtools-stm",
-]
 xrd = [
     "pynxtools-xrd",
 ]