asyncmd 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

asyncmd/__init__.py

@@ -12,6 +12,13 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+The asyncmd toplevel module.
+
+It imports the central Trajectory objects and the config(uration) functions for
+user convenience.
+It also makes public some information on the asyncmd version/git_hash.
+"""
 from ._version import __version__, __git_hash__
 
 from . import config

asyncmd/_config.py

@@ -12,15 +12,22 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+Configuration dictionaries to influence asyncmd runtime behavior.
 
+NOTE: This file **only** contains the dictionaries with the values
+      and **no** functions to set them, the funcs all live in 'config.py'.
+      The idea here is that we can then without any issues import additional
+      stuff (like the config functions from 'slurm.py') in 'config.py'
+      without risking circular imports because all asyncmd files should only
+      need to import the _CONFIG and _SEMAPHORES dicts from '_config.py'.
+"""
+import asyncio
+import typing
 
-# NOTE: This file **only** contains the dictionaries with the values
-#       and **no** functions to set them, the funcs all live in 'config.py'.
-#       The idea here is that we can then without any issues import additional
-#       stuff (like the config functions from 'slurm.py') in 'config.py'
-#       without risking circular imports becasue all asyncmd files should only
-#       need to import the _CONFIG and _SEMAPHORES dicts from '_config.py'.
 
-
-_GLOBALS = {}
-_SEMAPHORES = {}
+_GLOBALS: dict[str, typing.Any] = {}
+_SEMAPHORES: dict[str, asyncio.BoundedSemaphore] = {}
+# These semaphores are optional (i.e. can be None, which means unlimited)
+# e.g. slurm_max_jobs
+_OPT_SEMAPHORES: dict[str, asyncio.BoundedSemaphore | None] = {}

asyncmd/_version.py

@@ -12,24 +12,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+Helpers to populate asyncmd.__version__.
+
+If we are in a git-repository (and detect commits since the last version-tagged
+commit) we will add a git-hash to the version.
+"""
 import os
 import subprocess
-
-
-def _get_version_from_pyproject():
-    """Get version string from pyproject.toml file."""
-    pyproject_toml = os.path.join(os.path.dirname(__file__),
-                                  "../../pyproject.toml")
-    with open(pyproject_toml) as f:
-        line = f.readline()
-        while line:
-            if line.startswith("version ="):
-                version_line = line
-                break
-            line = f.readline()
-    version = version_line.strip().split(" = ")[1]
-    version = version.replace('"', '').replace("'", "")
-    return version
+import importlib.metadata
 
 
 def _get_git_hash_and_tag():
@@ -37,39 +28,34 @@ def _get_git_hash_and_tag():
     git_hash = ""
     git_date = ""
     git_tag = ""
-    p = subprocess.Popen(
+    with subprocess.Popen(
             ["git", "log", "-1", "--format='%H || %as || %(describe:tags=true,match=v*)'"],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             cwd=os.path.dirname(__file__),
-                         )
-    stdout, stderr = p.communicate()
-    if p.returncode == 0:
+                          ) as p:
+        stdout, _ = p.communicate()
+        returncode = p.returncode
+    if not returncode:
         git_hash, git_date, git_describe = (stdout.decode("utf-8")
                                             .replace("'", "").replace('"', '')
                                             .strip().split("||"))
         git_date = git_date.strip().replace("-", "")
         git_describe = git_describe.strip()
-        if "-" not in git_describe and git_describe != "":
+        if git_describe and "-" not in git_describe:
             # git-describe returns either the git-tag or (if we are not exactly
             #  at a tag) something like
             #  $GITTAG-$NUM_COMMITS_DISTANCE-$CURRENT_COMMIT_HASH
             git_tag = git_describe[1:]  # strip of the 'v'
     return git_hash, git_date, git_tag
 
-try:
-    _version = _get_version_from_pyproject()
-except FileNotFoundError:
-    # pyproject.toml not found
-    import importlib.metadata
-    __version__ = importlib.metadata.version("asyncmd")
-    __git_hash__ = ""
+
+_version = importlib.metadata.version("asyncmd")
+_git_hash, _git_date, _git_tag = _get_git_hash_and_tag()
+__git_hash__ = _git_hash
+if _version == _git_tag or not _git_hash:
+    # dont append git_hash to version, if it is a version-tagged commit or if
+    # git_hash is empty (happens if git is installed but we are not in a repo)
+    __version__ = _version
 else:
-    _git_hash, _git_date, _git_tag = _get_git_hash_and_tag()
-    __git_hash__ = _git_hash
-    if _version == _git_tag or _git_hash == "":
-        # dont append git_hash to version, if it is a version-tagged commit or if
-        # git_hash is empty (happens if git is installed but we are not in a repo)
-        __version__ = _version
-    else:
-        __version__ = _version + f"+git{_git_date}.{_git_hash[:7]}"
+    __version__ = _version + f"+git{_git_date}.{_git_hash[:7]}"

asyncmd/config.py

@@ -12,23 +12,29 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+This module contains the implementation of functions configuring asyncmd behavior.
+
+It also import the configuration functions for submodules (like slurm) to make
+them accessible to users in one central place.
+"""
 import os
 import asyncio
 import logging
 import resource
-import typing
 
 
-from ._config import _GLOBALS, _SEMAPHORES
+from ._config import _GLOBALS, _SEMAPHORES, _OPT_SEMAPHORES
+from .trajectory.trajectory import _update_cache_type_for_all_trajectories
+# pylint: disable-next=unused-import
 from .slurm import set_slurm_settings, set_all_slurm_settings
-# TODO: Do we want to set the _GLOBALS defaults here? E.g. CACHE_TYPE="npz"?
 
 
 logger = logging.getLogger(__name__)
 
 
 # can be called by the user to (re) set maximum number of processes used
-def set_max_process(num=None, max_num=None):
+def set_max_process(num: int | None = None, max_num: int | None = None) -> None:
     """
     Set the maximum number of concurrent python processes.
 
@@ -45,16 +51,14 @@ def set_max_process(num=None, max_num=None):
         spawning hundreds of processes.
     """
     # NOTE: I think we should use a conservative default, e.g. 0.25*cpu_count()
-    # TODO: limit to 30-40?, i.e never higher even if we have 1111 cores?
+    # pylint: disable-next=global-variable-not-assigned
     global _SEMAPHORES
     if num is None:
-        logical_cpu_count = os.cpu_count()
-        if logical_cpu_count is not None:
-            num = int(logical_cpu_count / 4)
+        if (logical_cpu_count := os.cpu_count()) is not None:
+            num = max(1, int(logical_cpu_count / 4))
         else:
             # fallback if os.cpu_count() can not determine the number of cpus
             # play it save and not have more than 2?
-            # TODO: think about a good number!
             num = 2
     if max_num is not None:
         num = min((num, max_num))
@@ -64,7 +68,7 @@ def set_max_process(num=None, max_num=None):
 set_max_process()
 
 
-def set_max_files_open(num: typing.Optional[int] = None, margin: int = 30):
+def set_max_files_open(num: int | None = None, margin: int = 30) -> None:
     """
     Set the maximum number of concurrently opened files.
 
@@ -86,10 +90,11 @@ def set_max_files_open(num: typing.Optional[int] = None, margin: int = 30):
     """
     # ensure that we do not open too many files
     # resource.getrlimit returns a tuple (soft, hard); we take the soft-limit
-    # and to be sure 30 less (the reason beeing that we can not use the
+    # and to be sure 30 less (the reason being that we can not use the
     # semaphores from non-async code, but sometimes use the sync subprocess.run
     # and subprocess.check_call [which also need files/pipes to work])
     # also maybe we need other open files like a storage :)
+    # pylint: disable-next=global-variable-not-assigned
     global _SEMAPHORES
     rlim_soft = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
     if num is None:
@@ -125,10 +130,9 @@ set_max_files_open()
 # SLURM semaphore stuff:
 # TODO: move this to slurm.py? and initialize only if slurm is available?
 # slurm max job semaphore, if the user sets it it will be used,
-# otherwise we can use an unlimited number of syncronous slurm-jobs
+# otherwise we can use an unlimited number of synchronous slurm-jobs
 # (if the simulation requires that much)
-# TODO: document that somewhere, bc usually clusters have a job number limit?!
-def set_slurm_max_jobs(num: typing.Union[int, None]):
+def set_slurm_max_jobs(num: int | None) -> None:
     """
     Set the maximum number of simultaneously submitted SLURM jobs.
 
@@ -138,66 +142,95 @@ def set_slurm_max_jobs(num: typing.Union[int, None]):
         The maximum number of simultaneous SLURM jobs for this invocation of
         python/asyncmd. `None` means do not limit the maximum number of jobs.
     """
-    global _SEMAPHORES
+    # pylint: disable-next=global-variable-not-assigned
+    global _OPT_SEMAPHORES
     if num is None:
-        _SEMAPHORES["SLURM_MAX_JOB"] = None
+        _OPT_SEMAPHORES["SLURM_MAX_JOB"] = None
     else:
-        _SEMAPHORES["SLURM_MAX_JOB"] = asyncio.BoundedSemaphore(num)
+        _OPT_SEMAPHORES["SLURM_MAX_JOB"] = asyncio.BoundedSemaphore(num)
 
 
 set_slurm_max_jobs(num=None)
 
 
 # Trajectory function value config
-def set_default_trajectory_cache_type(cache_type: str):
+def set_trajectory_cache_type(cache_type: str,
+                              copy_content: bool = True,
+                              clear_old_cache: bool = False
+                              ) -> None:
     """
-    Set the default cache type for TrajectoryFunctionValues.
+    Set the cache type for TrajectoryFunctionWrapper values.
 
-    Note that this can be overwritten on a per trajectory basis by passing
-    ``cache_type`` to ``Trajectory.__init__``.
+    By default the content of the current caches is copied to the new caches.
+    To clear the old/previously set caches (after copying their values), pass
+    ``clear_old_cache=True``.
 
     Parameters
     ----------
     cache_type : str
         One of "h5py", "npz", "memory".
+    copy_content : bool, optional
+        Whether to copy the current cache content to the new cache,
+        by default True
+    clear_old_cache : bool, optional
+        Whether to clear the old/previously set cache, by default False.
 
     Raises
     ------
     ValueError
         Raised if ``cache_type`` is not one of the allowed values.
     """
+    # pylint: disable-next=global-variable-not-assigned
     global _GLOBALS
     allowed_values = ["h5py", "npz", "memory"]
-    cache_type = cache_type.lower()
-    if cache_type not in allowed_values:
+    if (cache_type := cache_type.lower()) not in allowed_values:
         raise ValueError(f"Given cache type must be one of {allowed_values}."
                          + f" Was: {cache_type}.")
-    _GLOBALS["TRAJECTORY_FUNCTION_CACHE_TYPE"] = cache_type
+    if _GLOBALS.get("TRAJECTORY_FUNCTION_CACHE_TYPE", "not_set") != cache_type:
+        # only do something if the new cache type differs from what we have
+        _GLOBALS["TRAJECTORY_FUNCTION_CACHE_TYPE"] = cache_type
+        _update_cache_type_for_all_trajectories(copy_content=copy_content,
+                                                clear_old_cache=clear_old_cache,
+                                                )
+
+
+set_trajectory_cache_type("npz")
 
 
-def register_h5py_cache(h5py_group, make_default: bool = False):
+def register_h5py_cache(h5py_group) -> None:
     """
     Register a h5py file or group for CV value caching.
 
     Note that this also sets the default cache type to "h5py", i.e. it calls
-    :func:`set_default_trajectory_cache_type` with ``cache_type="h5py"``.
+    :func:`set_trajectory_cache_type` with ``cache_type="h5py"``.
 
     Note that a ``h5py.File`` is just a slightly special ``h5py.Group``, so you
-    can pass either. :mod:`asyncmd` will use euther the file or the group as
+    can pass either. :mod:`asyncmd` will use either the file or the group as
     the root of its own stored values.
     E.g. you will have ``h5py_group["asyncmd/TrajectoryFunctionValueCache"]``
     always pointing to the cached trajectory values and if ``h5py_group`` is
-    the top-level group (i.e. the file) you also have ``(file["/asyncmd/TrajectoryFunctionValueCache"] == h5py_group["asyncmd/TrajectoryFunctionValueCache"])``.
+    the top-level group (i.e. the file) you also have
+    ``(file["/asyncmd/TrajectoryFunctionValueCache"] ==\
+ h5py_group["asyncmd/TrajectoryFunctionValueCache"])``.
 
     Parameters
     ----------
     h5py_group : h5py.Group or h5py.File
         The file or group to use for caching.
-    make_default: bool,
-        Whether we should also make "h5py" the default trajectory function
-        cache type. By default False.
     """
+    # pylint: disable-next=global-variable-not-assigned
     global _GLOBALS
-    if make_default:
-        set_default_trajectory_cache_type(cache_type="h5py")
     _GLOBALS["H5PY_CACHE"] = h5py_group
+    set_trajectory_cache_type(cache_type="h5py")
+
+
+def show_config() -> None:
+    """
+    Print/show current configuration.
+    """
+    print(f"Values controlling caching: {_GLOBALS}")
+    # pylint: disable-next=protected-access
+    sem_print = {key: sem._value
+                 for key, sem in {**_SEMAPHORES, **_OPT_SEMAPHORES}.items()
+                 if sem is not None}
+    print(f"Semaphores controlling resource usage: {sem_print}")

asyncmd/gromacs/__init__.py

@@ -12,5 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+This module exports all user-facing classes and functions for running gromacs.
+"""
 from .mdconfig import MDP
 from .mdengine import GmxEngine, SlurmGmxEngine

asyncmd/gromacs/mdconfig.py

@@ -12,6 +12,9 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with asyncmd. If not, see <https://www.gnu.org/licenses/>.
+"""
+This file contains the MDP class to parse, modify, and write gromacs mdp files.
+"""
 import shlex
 import logging
 from ..mdconfig import LineBasedMDConfig
@@ -27,18 +30,6 @@ class MDP(LineBasedMDConfig):
     option, list of values pairs. Includes automatic types for known options
     and keeps track if any options have been changed compared to the original
     file.
-
-    Parameters
-    ----------
-    original_file : str
-        absolute or relative path to original config file to parse
-
-    Methods
-    -------
-    write(outfile)
-        write the current (modified) configuration state to a given file
-    parse()
-        read the current original_file and update own state with it
     """
 
     _KEY_VALUE_SEPARATOR = " = "
@@ -298,11 +289,11 @@ class MDP(LineBasedMDConfig):
         #                               before or after the equal sign
         # 4. no splits at '=' and no splits at ';' -> weired line, probably
         #                                             not a valid line(?)
-        if splits_at_comment[0] == "":
+        if not splits_at_comment[0]:
             # option 2 (and 3 if the comment is before the equal sign)
             # comment sign is the first letter, so the whole line is
             # (most probably) a comment line
-            logger.debug(f"mdp line parsed as comment: {line}")
+            logger.debug("mdp line parsed as comment: %s", line)
             return {}
         if ((len(splits_at_equal) == 2 and len(splits_at_comment) == 1)  # option 1
             # or option 3 with equal sign before comment sign
@@ -318,10 +309,9 @@ class MDP(LineBasedMDConfig):
                 parser.commenters = ";"
                 # puncutation_chars=True adds "~-./*?=" to wordchars
                 # such that we do not split floats and file paths and similar
-                tokens = list(parser)
                 # gromacs mdp can have 0-N tokens/values to the RHS of the '='
-                if len(tokens) == 0:
-                    # line with empty options, e.g. 'define = '
+                if not (tokens := list(parser)):
+                    # zero tokens -> line with empty options, e.g. 'define = '
                     return {self._key_char_replace(key): []}
                 # lines with content, we always return a list (and let our
                 #  type_dispatch sort out the singleton options and the typing)