kim-tools 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

kim_tools/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.2"
+__version__ = "0.4.3"
 
 from .aflow_util import *
 from .aflow_util import __all__ as aflow_all

kim_tools/test_driver/core.py

@@ -1,4 +1,4 @@
-################################################################################
+###############################################################################
 #
 #  CDDL HEADER START
 #
@@ -30,7 +30,6 @@
 Helper classes for KIM Test Drivers
 
 """
-import glob
 import json
 import logging
 import os
@@ -38,6 +37,8 @@ import shutil
 import tarfile
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from fnmatch import fnmatch
+from glob import glob
 from pathlib import Path
 from secrets import token_bytes
 from tempfile import NamedTemporaryFile, TemporaryDirectory
@@ -113,6 +114,21 @@ PROP_SEARCH_PATHS_INFO = (
 )
 TOKEN_NAME = "kim-tools.token"
 TOKENPATH = os.path.join("output", TOKEN_NAME)
+PIPELINE_EXCEPTIONS = ["output/pipeline.*"]
+
+
+def _glob_with_exceptions(
+    pattern: str, exceptions: List[str], recursive: bool = False
+) -> List[os.PathLike]:
+    """
+    Return a list of paths that match the glob "pattern" but not
+    the glob "exceptions"
+    """
+    match = glob(pattern, recursive=recursive)
+    # If we are willing to make Python 3.14 minimum, can use filterfalse
+    for exception in exceptions:
+        match = [n for n in match if not fnmatch(n, exception)]
+    return match
 
 
 def get_supported_lammps_atom_style(model: str) -> str:
@@ -565,10 +581,18 @@ class KIMTestDriver(ABC):
             List of files that were written by this class explicitly, that we won't
             touch when cleaning and backing up the output directory. Specified
             relative to 'output' directory.
+        __files_to_ignore_in_output (List[PathLike]):
+            List of globs of files to ignore when handling the output directory.
+            By default, this is set to the constant PIPELINE_EXCEPTIONS,
+            which contains files that need to be left untouched for the
+            OpenKIM pipeline. Top-level dotfiles are always ignored.
         __token (Optional[bytes]):
             Token that is written to TOKENPATH upon first evaluation. This
             is used to check that multiple Test Drivers are not being called
             concurrently, causing potential conflicts in the output directory
+        __times_called (Optional[int]):
+            Count of number of times the instance has been __call__'ed,
+            for numbering aux file archives
     """
 
     class NonKIMModelError(Exception):
@@ -579,7 +603,10 @@ class KIMTestDriver(ABC):
         """
 
     def __init__(
-        self, model: Union[str, Calculator], suppr_sm_lmp_log: bool = False
+        self,
+        model: Union[str, Calculator],
+        suppr_sm_lmp_log: bool = False,
+        files_to_ignore_in_output: List[str] = PIPELINE_EXCEPTIONS,
     ) -> None:
         """
         Args:
@@ -587,6 +614,9 @@ class KIMTestDriver(ABC):
                 ASE calculator or KIM model name to use
             suppr_sm_lmp_log:
                 Suppress writing a lammps.log
+            files_to_ignore_in_output (List[PathLike]):
+                List of globs of files to ignore when handling the output directory.
+                Top-level dotfiles are always ignored.
         """
         if isinstance(model, Calculator):
             self.__calc = model
@@ -602,7 +632,9 @@ class KIMTestDriver(ABC):
 
         self.__output_property_instances = "[]"
         self.__files_to_keep_in_output = []
+        self.__files_to_ignore_in_output = files_to_ignore_in_output
         self.__token = None
+        self.__times_called = None
 
     def _setup(self, material, **kwargs) -> None:
         """
@@ -617,19 +649,25 @@ class KIMTestDriver(ABC):
         if self.__token is None:
             # First time we've called this instance of the class
             assert len(self.property_instances) == 0
+            assert self.__times_called is None
+
+            self.__times_called = 0
 
             os.makedirs("output", exist_ok=True)
 
             # Move all top-level non-hidden files and directories
             # to backup
-            output_glob = glob.glob("output/*")
+            output_glob = _glob_with_exceptions(
+                "output/*", self.__files_to_ignore_in_output
+            )
             if len(output_glob) > 0:
                 i = 0
                 while os.path.exists(f"output.{i}"):
                     i += 1
                 output_bak_name = f"output.{i}"
                 msg = (
-                    "'output' directory is non-empty, backing up all "
+                    "'output' directory has files besides dotfiles and allowed "
+                    "exceptions, backing up all "
                     f"non-hidden files and directories to {output_bak_name}"
                 )
                 print(msg)
@@ -661,10 +699,13 @@ class KIMTestDriver(ABC):
                             " prevent conflicts in the output directory, this is not "
                             "allowed."
                         )
+            self.__times_called += 1
 
         # We should have a record of all non-hidden files in output. If any
         # untracked files are present, raise an error
-        output_glob = glob.glob("output/**", recursive=True)
+        output_glob = _glob_with_exceptions(
+            "output/**", self.__files_to_ignore_in_output, True
+        )
         for filepath in output_glob:
             if os.path.isfile(filepath):  # not tracking directories
                 if (
@@ -682,30 +723,38 @@ class KIMTestDriver(ABC):
         Archive aux files after a run
         """
         # Archive untracked files as aux files
-        i = 0
-        while f"aux_files.{i}.txz" in self.__files_to_keep_in_output:
-            assert os.path.isfile(f"output/aux_files.{i}.txz")
-            i += 1
-        tar_prefix = f"aux_files.{i}"
-        output_glob = glob.glob("output/**", recursive=True)
-        archived_files = []  # For deleting them later
-        with tarfile.open(f"output/{tar_prefix}.txz", "w:xz") as tar:
-            for filepath in output_glob:
-                if os.path.isfile(filepath):  # not tracking directories
+        tar_prefix = f"aux_files.{self.__times_called}"
+        archive_name = f"output/{tar_prefix}.txz"
+        assert not os.path.isfile(tar_prefix)
+        output_glob = _glob_with_exceptions(
+            "output/**", self.__files_to_ignore_in_output, True
+        )
+        archived_files = []  # For deleting them later, and checking that any exist
+        for filepath in output_glob:
+            if os.path.isfile(filepath):  # not tracking directories
+                output_relpath = os.path.relpath(filepath, "output")
+                if output_relpath not in self.__files_to_keep_in_output:
+                    archived_files.append(filepath)
+
+        if len(archived_files) > 0:
+            msg = f"Auxiliary files found after call, archiving them to {archive_name}"
+            print(msg)
+            logger.info(msg)
+
+            with tarfile.open(archive_name, "w:xz") as tar:
+                for filepath in archived_files:
                     output_relpath = os.path.relpath(filepath, "output")
-                    if output_relpath not in self.__files_to_keep_in_output:
-                        tar.add(
-                            os.path.join(filepath),
-                            os.path.join(tar_prefix, output_relpath),
-                        )
-                        archived_files.append(filepath)
-        self.__files_to_keep_in_output.append(f"{tar_prefix}.txz")
-        for filepath in archived_files:
-            os.remove(filepath)
-            try:
-                os.removedirs(os.path.dirname(filepath))
-            except OSError:
-                pass  # might not be empty yet
+                    tar.add(
+                        os.path.join(filepath),
+                        os.path.join(tar_prefix, output_relpath),
+                    )
+            self.__files_to_keep_in_output.append(f"{tar_prefix}.txz")
+            for filepath in archived_files:
+                os.remove(filepath)
+                try:
+                    os.removedirs(os.path.dirname(filepath))
+                except OSError:
+                    pass  # might not be empty yet
 
         # should not have removed output dir in any situation
         assert os.path.isdir("output")
@@ -722,9 +771,9 @@ class KIMTestDriver(ABC):
 
         Main operation of a Test Driver:
 
+            * Call :func:`~KIMTestDriver._init_output_dir`
             * Run :func:`~KIMTestDriver._setup` (the base class provides a barebones
               version, derived classes may override)
-            * Call :func:`~KIMTestDriver._init_output_dir`
             * Call :func:`~KIMTestDriver._calculate` (implemented by each individual
               Test Driver)
             * Call :func:`~KIMTestDriver._archive_aux_files`
@@ -744,14 +793,15 @@ class KIMTestDriver(ABC):
         # Set up the output directory
         self._init_output_dir()
 
-        # _setup is likely overridden by an derived class
-        self._setup(material, **kwargs)
-
-        # implemented by each individual Test Driver
-        self._calculate(**kwargs)
-
-        # Postprocess output directory for this invocation
-        self._archive_aux_files()
+        try:
+            # _setup is likely overridden by an derived class
+            self._setup(material, **kwargs)
+
+            # implemented by each individual Test Driver
+            self._calculate(**kwargs)
+        finally:
+            # Postprocess output directory for this invocation
+            self._archive_aux_files()
 
         # The current invocation returns a Python list of dictionaries containing all
         # properties computed during this run

{kim_tools-0.4.2.dist-info → kim_tools-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kim-tools
-Version: 0.4.2
+Version: 0.4.3
 Summary: Base classes and helper routines for writing KIM Tests
 Author-email: ilia Nikiforov <nikif002@umn.edu>, Ellad Tadmor <tadmor@umn.edu>, Claire Waters <bwaters@umn.edu>, "Daniel S. Karls" <karl0100umn@gmail.com>, Matt Bierbaum <matt.bierbaum@gmail.com>, Eric Fuemmeler <efuemmel@umn.edu>, Philipp Hoellmer <ph2484@nyu.edu>, Guanming Zhang <gz2241@nyu.edu>, Tom Egg <tje3676@nyu.edu>, Navaneeth Mohan <mohan227@umn.edu>
 Maintainer-email: ilia Nikiforov <nikif002@umn.edu>

{kim_tools-0.4.2.dist-info → kim_tools-0.4.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-kim_tools/__init__.py,sha256=ATvBQMEr5NBWxGzIDxLFJxMZIXTwxIx5RVS5-OO4H98,433
+kim_tools/__init__.py,sha256=Tw-dFo4IjQt_u8V_veHj6byvwl1KfmazTL1GdoWwcgM,433
 kim_tools/kimunits.py,sha256=aCouh7z6fQDR1rcbRTO72l-_RuccuznrKrNF18F-cEQ,6076
 kim_tools/aflow_util/__init__.py,sha256=lJnQ8fZCma80QVRQeKvY4MQ87oCWu-9KATV3dKJfpDc,80
 kim_tools/aflow_util/core.py,sha256=OmU4-m2jl81jfs0ni4bSECpC8GYt7-hEEH7cfKmrdoM,80886
@@ -2023,11 +2023,11 @@ kim_tools/symmetry_util/data/wyck_pos_xform_under_normalizer.json,sha256=6g1YuYh
 kim_tools/symmetry_util/data/wyckoff_multiplicities.json,sha256=qG2RPBd_-ejDIfz-E4ZhkHyRpIboxRy7oiXkdDf5Eg8,32270
 kim_tools/symmetry_util/data/wyckoff_sets.json,sha256=f5ZpHKDHo6_JWki1b7KUGoYLlhU-44Qikw_-PtbLssw,9248
 kim_tools/test_driver/__init__.py,sha256=KOiceeZNqkfrgZ66CiRiUdniceDrCmmDXQkOw0wXaCQ,92
-kim_tools/test_driver/core.py,sha256=ugPL_wDzKwT7ch7Xh6SFMCLTF1Qf8x405HqcGJ29fd4,106587
+kim_tools/test_driver/core.py,sha256=MtoJgGWPbqaMG1yOhNZd83Waw5qKwI4jJKkXfP1rMJc,108701
 kim_tools/vc/__init__.py,sha256=zXjhxXCKVMLBMXXWYG3if7VOpBnsFrn_RjVpnohDm5c,74
 kim_tools/vc/core.py,sha256=BIjzEExnQAL2S90a_npptRm3ACqAo4fZBtvTDBMWMdw,13963
-kim_tools-0.4.2.dist-info/licenses/LICENSE.CDDL,sha256=I2luEED_SHjuZ01B4rYG-AF_135amL24JpHvZ1Jhqe8,16373
-kim_tools-0.4.2.dist-info/METADATA,sha256=dCmTQkNmZhR6y-2GtlwbgMnFMh14kgKgTlkvGL4UHpw,2196
-kim_tools-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-kim_tools-0.4.2.dist-info/top_level.txt,sha256=w_YCpJ5ERigj9te74ln7k64tqj1VumOzM_s9dsalIWY,10
-kim_tools-0.4.2.dist-info/RECORD,,
+kim_tools-0.4.3.dist-info/licenses/LICENSE.CDDL,sha256=I2luEED_SHjuZ01B4rYG-AF_135amL24JpHvZ1Jhqe8,16373
+kim_tools-0.4.3.dist-info/METADATA,sha256=8bkatoFLO7BAGC-C6gGwIoCjI4s7HNqNLv-SmM10rZw,2196
+kim_tools-0.4.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+kim_tools-0.4.3.dist-info/top_level.txt,sha256=w_YCpJ5ERigj9te74ln7k64tqj1VumOzM_s9dsalIWY,10
+kim_tools-0.4.3.dist-info/RECORD,,