digichem-core 6.0.3__py3-none-any.whl → 6.10.1__py3-none-any.whl

digichem/image/vmd.py

@@ -35,7 +35,7 @@ class VMD_image_maker(Render_maker):
     options_name = "orbital"
     
     def __init__(self, *args, cube_file = None, rotations = None, auto_crop = True, rendering_style = "pastel", resolution = 1024, also_make_png = True, isovalue = 0.2, 
-                 vmd_executable = "vmd", tachyon_executable = "tachyon", vmd_logging = False,
+                 vmd_executable = "vmd", tachyon_executable = "tachyon", vmd_logging = False, num_cpu = 1,
                  **kwargs):
         """
         Constructor for Image_maker objects.
@@ -48,6 +48,7 @@ class VMD_image_maker(Render_maker):
         :param resolution: The max width or height of the rendered images in pixels.
         :param also_make_png: If True, additional images will be rendered in PNG format. This option is useful to generate higher quality images alongside more portable formats. If 'output' is a .png file, then it is wise to set this option to False (otherwise two png files will be rendered, which is a waste).
         :param isovalue: The isovalue to use for rendering isosurfaces. Has no effect when rendering only atoms.
+        :param num_cpu: Number of CPUs to use for multithreading.
         :param vmd_executable: 'Path' to the vmd executable to use for image rendering. Defaults to relying on the command 'vmd'.
         :param tachyon_executable: 'Path' to the tachyon executable to use for image rendering. Defaults to relying on the command 'tachyon'.
         :param vmd_logging: Whether to print output from vmd.
@@ -60,6 +61,7 @@ class VMD_image_maker(Render_maker):
             resolution = resolution,
             also_make_png = also_make_png,
             isovalue = isovalue,
+            num_cpu = num_cpu,
             **kwargs
         )
         
@@ -95,7 +97,7 @@ class VMD_image_maker(Render_maker):
         return [(axis, math.degrees(-theta)) for axis, theta in self._rotations]
     
     @classmethod
-    def from_options(self, output, *, cube_file = None, rotations = None, options, **kwargs):
+    def from_options(self, output, *, cube_file = None, rotations = None, options, num_cpu = 1, **kwargs):
         """
         Constructor that takes a dictionary of config like options.
         """        
@@ -112,6 +114,7 @@ class VMD_image_maker(Render_maker):
             vmd_executable = options['render']['vmd']['executable'],
             tachyon_executable = options['render']['vmd']['tachyon'],
             vmd_logging = options['logging']['render_logging'],
+            num_cpu = num_cpu,
             **kwargs
         )
     
@@ -344,6 +347,8 @@ class VMD_image_maker(Render_maker):
                         "{}".format(self.tachyon_executable),
                         scene_file.relative_to(working_directory),
                         "-aasamples", "12",
+                        # Note: this can get capped in a SLURM context...
+                        "-numthreads", "{}".format(self.num_cpu),
                         "-res", "{}".format(resolution), "{}".format(resolution),
                         "-o", tmpfile_name
                     ],

digichem/input/digichem_input.py

@@ -65,7 +65,7 @@ class Digichem_coords_ABC(Input_file, Molecule_mixin):
             # Note it's not the history of the old calc we want here, this old calc IS our new history.
             kwargs['history'] = result._id
             
-        if not kwargs.get('file_name'):
+        if not kwargs.get('file_name') and len(result.metadata.log_files):
             kwargs['file_name'] = result.metadata.log_files[0]
         
         if not kwargs.get('name'):
@@ -462,7 +462,7 @@ def si_from_file(file_name, file_type = None, *, gen3D = None, **kwargs):
             # NOTE: Here we assume files without an extension are log files.
             # This works fine for directories, but might change in future.
             elif file_type in ["dat", "log", "out", "output", None] \
-                or (auto_file_type and "".join(file_name.suffixes) in open_for_parsing.archive_formats()):
+                or (auto_file_type and "".join(file_name.suffixes) in open_for_parsing.get_archive_formats()):
                 # Generic log-file (output) format.
                 # Most formats (.log, .dat etc) we can parse with either Obabel or Digichem.
                 # Some broken log files (incomplete) we can parse only with Obabel.

digichem/memory.py

@@ -155,6 +155,16 @@ class Memory():
     
     def __eq__(self, other):
         return int(self) == other
+    
+    @classmethod
+    def is_memory(self, value):
+        """Convenience method to determine whether a value is a valid memory amount."""
+        try:
+            self(value)
+            return True
+        
+        except Exception:
+            return False
 
 
 class Turbomole_memory(Memory):

digichem/misc/io.py

@@ -6,8 +6,43 @@ import os
 import shutil
 import sys
 import warnings
+from uuid import uuid4
+import hashlib
 
 from digichem.datas import get_resource
+import digichem.log
+
+def checksum(*paths, hash_func = "sha1", buf_size = 1024 * 1024, ret_size = False):
+    """
+    Calculate the checksum of a file.
+
+    This function avoids reading the entire file into memory at once.
+
+    :param paths: The file(s) to calculate.
+    :param hash_func: The name of a hashlib function to pass to haslib.new.
+    :param buf_size: How much to read in a single pass. The default is 1MB.
+    :param ret_size: If True, this function will return a tuple of (checksum, file_size). Otherwise, only the checksum is returned.
+    """
+    hasher = hashlib.new(hash_func)
+    tot_size = 0
+
+    for pth in paths:
+        with open(pth, "rb") as file:
+            while True:
+                data = file.read(buf_size)
+
+                if len(data) == 0:
+                    # End of file
+                    break
+
+                tot_size += len(data)
+                hasher.update(data)
+    
+    if ret_size:
+        return (hasher.hexdigest(), tot_size)
+
+    else:
+        return hasher.hexdigest()
 
 def expand_path(pth):
     """
@@ -236,4 +271,52 @@ class Multi_file_wrapper():
         """
         # Close our file.
         self.close()
-        
+
+class Safe_path():
+    """
+    Get a 'safe' path to a file.
+
+    A safe path is made up of only alphanumeric characters, and is short. This function is useful for dealing
+    with antiquated programs that struggle with whitespace/non-alpha characters or have a max file name requirement.
+
+    This class should be used as a context manager. The returned path is a temporary symbolic link link to the true file.
+    The symbolic link will be created in the specified 'dir', which defaults to the CWD.
+    """
+
+    def __init__(self, unsafe_path, dir = "./", suffix = ""):
+        self.unsafe_path = unsafe_path
+        self.dir = dir
+        self.link = None
+        self.suffix = suffix
+
+
+    def enter(self):
+        """
+        Create the symlink.
+        """
+        self.link = Path(self.dir, ".{}".format(uuid4().hex) + self.suffix)
+        os.symlink(self.unsafe_path, self.link)
+
+    def close(self):
+        """
+        Remove the symlink.
+        """
+        try:
+            os.unlink(self.link)
+        
+        except FileExistsError:
+            digichem.log.get_logger().warning("Failed to remove temporary symbolic link '{}' -> '{}'; the file has already disappeared?".format(self.link, self.unsafe_path))
+            
+    def __enter__(self):
+        """
+        Magic function for the 'with' keyword.
+        """
+        self.enter()
+        return self
+        
+    def __exit__(self, etype, value, traceback):
+        """
+        Magic function for the 'with' keyword, called at the end of the block.
+        """
+        # Close our file.
+        self.close()

digichem/parse/__init__.py

@@ -4,10 +4,15 @@ Classes for parsing calculation result data.
 Most of the heavy lifting is done by cclib, we just extract additional data not currently handed by cclib.
 """
 
+# We're not bothered about these warnings.
+import warnings
+warnings.filterwarnings("ignore", "Module .* is under testing", category = UserWarning)
+warnings.filterwarnings("ignore", "Module .* is not fully tested", category = UserWarning)
+
 # These alignment classes are needed to parse correctly.
 from digichem.result.alignment.AAA import Adjusted_average_angle
 from digichem.result.alignment.AA import Average_angle
 from digichem.result.alignment.FAP import Furthest_atom_pair
 from digichem.result.alignment import Minimal
 
-from digichem.parse.util import from_log_files, parse_calculation, parse_and_merge_calculations, parse_multiple_calculations, parse_and_merge_multiple_calculations
+from digichem.parse.util import from_log_files, parse_calculation, parse_and_merge_calculations, parse_multiple_calculations, parse_and_merge_multiple_calculations, open_for_parsing

digichem/parse/base.py

@@ -31,20 +31,11 @@ custom_parsing_formats = [
 
 class Parser_abc():
     """ABC for all parsers."""
-    
-    def __init__(self, *log_files, raw_data = None, **kwargs):
+
+    def __init__(self, *, raw_data = None, **kwargs):
         """
         Top level constructor for calculation parsers.
-        
-        :param log_files: A list of output file to analyse/parse. The first log_file given will be used for naming purposes.
         """
-        # Set our name.
-        self.log_file_paths = self.sort_log_files([Path(log_file) for log_file in log_files if log_file is not None])
-        
-        # Panic if we have no logs.
-        if len(self.log_file_paths) == 0:
-            raise Digichem_exception("Cannot parse calculation output; no available log files. Are you sure the given path is a log file or directory containing log files?")
-        
         # An object that we will populate with raw results.
         self.data = raw_data
         
@@ -57,49 +48,21 @@ class Parser_abc():
                 self.parse()
         except Exception:
             raise Digichem_exception("Error parsing calculation result '{}'".format(self.description))
-
-    @classmethod
-    def from_logs(self, *log_files, **kwargs):
-        """
-        Intelligent constructor that will attempt to guess the location of aux files from a given log file(s).
-        
-        :param log_files: Output file(s) to parse or a directory of output files to parse.
-        """
-        # This default implementation does nothing smart.
-        return self(*log_files, **kwargs)
-    
-    @classmethod
-    def sort_log_files(self, log_files):
-        """
-        Sort a list of log files into a particular order, if required for this parser.
-        """
-        return log_files
     
-    @property
-    def log_file_path(self):
-        """
-        The main log file.
-        """
-        for log_file in self.log_file_paths:
-            if log_file.suffix.lower() == ".log":
-                return log_file
-            
-        return self.log_file_paths[0]
-            
     @property
     def name(self):
         """
         Short name to describe this calculation result.
         """
-        return self.log_file_path.with_suffix("").name
+        return "Parser"
     
     @property
     def description(self):
         """
         A name/path that describes the file(s) being parsed, used for error messages etc.
         """
-        return self.log_file_path
-    
+        return "Parser"
+
     def parse(self):
         """
         Extract results from our output files.
@@ -128,22 +91,14 @@ class Parser_abc():
         
         except Exception as e:
             return None
-        
+    
     def post_parse(self):
         """
         Perform any required operations after line-by-line parsing.
-        """
-        # Add our name.
-        if self.name is not None:
-            self.data.metadata['name'] = self.name
-            
+        """ 
         # Add current username.
         # TODO: It would probably be better if we used the name of the user who owns the output file, rather than the current user...
         self.data.metadata['user'] = self.get_current_username()
-        
-        # Set our file paths.
-        self.data.metadata['log_files'] = self.log_file_paths
-        self.data.metadata['auxiliary_files'] = self.auxiliary_files
             
     def process_all(self, options):
         """
@@ -154,7 +109,7 @@ class Parser_abc():
         """
         self.process(options)
         return [self.results]
-
+    
     def process(self, options):
         """
         Get a Result set object from this parser.
@@ -166,7 +121,8 @@ class Parser_abc():
         # Get our result set.
         self.results = Result_set(
             _id = self.data._id,
-            metadata = Metadata.from_parser(self)
+            metadata = Metadata.from_parser(self),
+            aux = self.data._aux if hasattr(self.data, '_aux') else None
             )
         
         alignment_class = Alignment.from_class_handle(options['alignment']) if options['alignment'] is not None else Minimal
@@ -218,3 +174,78 @@ class Parser_abc():
         
         # Return the populated result set for convenience.
         return self.results
+
+
+class File_parser_abc(Parser_abc):
+    """ABC for all parsers."""
+    
+    def __init__(self, *log_files, raw_data = None, **kwargs):
+        """
+        Top level constructor for calculation parsers.
+        
+        :param log_files: A list of output file to analyse/parse. The first log_file given will be used for naming purposes.
+        """
+        # Set our name.
+        self.log_file_paths = self.sort_log_files([Path(log_file) for log_file in log_files if log_file is not None])
+        
+        # Panic if we have no logs.
+        if len(self.log_file_paths) == 0:
+            raise Digichem_exception("Cannot parse calculation output; no available log files. Are you sure the given path is a log file or directory containing log files?")
+        
+        super().__init__(raw_data=raw_data, **kwargs)
+
+    @classmethod
+    def from_logs(self, *log_files, **kwargs):
+        """
+        Intelligent constructor that will attempt to guess the location of aux files from a given log file(s).
+        
+        :param log_files: Output file(s) to parse or a directory of output files to parse.
+        """
+        # This default implementation does nothing smart.
+        return self(*log_files, **kwargs)
+    
+    @classmethod
+    def sort_log_files(self, log_files):
+        """
+        Sort a list of log files into a particular order, if required for this parser.
+        """
+        return log_files
+    
+    @property
+    def log_file_path(self):
+        """
+        The main log file.
+        """
+        for log_file in self.log_file_paths:
+            if log_file.suffix.lower() == ".log":
+                return log_file
+            
+        return self.log_file_paths[0]
+            
+    @property
+    def name(self):
+        """
+        Short name to describe this calculation result.
+        """
+        return self.log_file_path.with_suffix("").name
+    
+    @property
+    def description(self):
+        """
+        A name/path that describes the file(s) being parsed, used for error messages etc.
+        """
+        return self.log_file_path
+        
+    def post_parse(self):
+        """
+        Perform any required operations after line-by-line parsing.
+        """
+        super().post_parse()
+
+        # Add our name.
+        if self.name is not None:
+            self.data.metadata['name'] = self.name
+        
+        # Set our file paths.
+        self.data.metadata['log_files'] = self.log_file_paths
+        self.data.metadata['auxiliary_files'] = self.auxiliary_files

digichem/parse/cclib.py

@@ -1,14 +1,20 @@
 from pathlib import Path
-import hashlib
+import itertools
+import csv
+import numpy
+import math
+from scipy import signal
 
 import digichem.log
-from digichem.parse.base import Parser_abc
+from digichem.parse.base import File_parser_abc
+from digichem.exception import Digichem_exception
+from digichem.misc.io import checksum
 
 # Hidden imports.
 #import cclib.io
 
 
-class Cclib_parser(Parser_abc):
+class Cclib_parser(File_parser_abc):
     """
     ABC for parsers that use cclib to do most of their work for them.
     """
@@ -25,11 +31,15 @@ class Cclib_parser(Parser_abc):
         """
         # Also save our aux files, stripping None.
         self.auxiliary_files = {name: aux_file for name,aux_file in auxiliary_files.items() if aux_file is not None}
+
+        # TODO: Does this belong here?
+        # Also have a look for a profile.csv file that we can us for performance metrics.
+        self.profile_file = Path(log_files[0].parent, "../Logs/profile.csv")
         
         super().__init__(*log_files)
         
     @classmethod
-    def from_logs(self, *log_files, **kwargs):
+    def from_logs(self, *log_files, hints = None, **kwargs):
         """
         Intelligent constructor that will attempt to guess the location of files from a given log file(s).
         
@@ -37,9 +47,11 @@ class Cclib_parser(Parser_abc):
         """
         # Have a look for aux. files.
         auxiliary_files = {}
+
+        basename = log_files[0].name if len(log_files) > 0 else ""
         
-        for found_log_file in log_files:
-            auxiliary_files.update(self.find_auxiliary_files(found_log_file))
+        for hint in itertools.chain(log_files, hints if hints is not None else []):
+            auxiliary_files.update(self.find_auxiliary_files(hint, basename))
             
         # Finally, update our auxiliary_files with kwargs, so any user specified aux files take precedence.
         auxiliary_files.update(kwargs)
@@ -47,7 +59,7 @@ class Cclib_parser(Parser_abc):
         return self(*log_files, **auxiliary_files)
     
     @classmethod
-    def find_auxiliary_files(self, hint):
+    def find_auxiliary_files(self, hint, basename):
         """
         Find auxiliary files from a given hint.
         
@@ -60,6 +72,11 @@ class Cclib_parser(Parser_abc):
         auxiliary_files = {}
         for file_type in self.INPUT_FILE_TYPES:
             for extension in file_type.extensions:
+                if hint.is_dir():
+                    # Peak inside.
+                    if Path(hint, basename).with_suffix(extension).exists():
+                        auxiliary_files[self.INPUT_FILE_TYPES[file_type]] = Path(hint, basename).with_suffix(extension)
+
                 if hint.with_suffix(extension).exists():
                     auxiliary_files[self.INPUT_FILE_TYPES[file_type]] = hint.with_suffix(extension)
         
@@ -86,18 +103,15 @@ class Cclib_parser(Parser_abc):
         
         # Get data from cclib.
         self.data = cclib.io.ccread(file_paths if len(file_paths) > 1 else file_paths[0])
+        if self.data is None:
+            raise Digichem_exception("Could not parse any data at all!")
         
         # Get a unique ID (checksum) from the given log files.
         # First, order the list of filenames so we also process in the same order.
         # We do this because not all parsers define a custom sort.
         
         file_paths.sort()
-        hasher = hashlib.sha1()
-        
-        for file_path in file_paths:
-            hasher.update(Path(file_path).read_bytes())
-            
-        self.data._id = hasher.hexdigest()
+        self.data._id = checksum(*file_paths, hash_func = "sha1")
         
         # Do some setup.
         self.pre_parse()
@@ -107,6 +121,82 @@ class Cclib_parser(Parser_abc):
             with open(log_file_path, "rt") as log_file:
                 for line in log_file:
                     self.parse_output_line(log_file, line)
+
+        # Add profiling data.
+        try:
+            self.parse_profile_file()
+        
+        except Exception:
+            digichem.log.get_logger().warning("Could not parse profile.csv file; profiling data will be unavailable", exc_info=True)
+
+    def parse_profile_file(self):
+        """
+        """
+        # Real calculations can end up with millions of rows, which is far too much data to handle.
+        # We will need to downsample if we have too many data points.
+        # First work out how many rows there are.
+        try:
+            with open(self.profile_file, "rb") as profile_file:
+                lines = sum(1 for _ in profile_file) -1 # Remove 1 line for the header.
+        
+        except FileNotFoundError:
+            # This is ok
+            return
+
+        if lines < 2:
+            return
+
+        max_lines  = 1000
+        factor = math.ceil(lines / max_lines)
+        
+        with open(self.profile_file) as profile_file:
+            reader = csv.reader(profile_file)
+
+            # Get the header.
+            headers = next(reader)
+
+            # Check headers match.
+            if (headers[0] != "Duration / s" or 
+               headers[1] != "Memory Used (Real) / bytes" or
+               headers[2] != "Memory Used (Real) / %" or
+               headers[3] != "Memory Available  (Real) / bytes" or
+               headers[4] != "Memory Available (Real) / %" or
+               headers[9] != "CPU Usage / %" or
+               headers[15] != "Output Directory Available / bytes" or
+               headers[17] != "Scratch Directory Available / bytes"
+            ):
+                raise Digichem_exception("wrong headers found in profile.csv file")
+            
+            # Then the body.
+            # TODO: Reading the entire file is not ideal...
+            data = numpy.genfromtxt(
+                profile_file,
+                delimiter=',',
+                # TODO: use something better.
+                filling_values = "0"
+            )
+
+        # We'll keep:
+        # - duration
+        # - memory used
+        # - memory used %
+        # - memory available
+        # - memory available %
+        # - cpu used
+        # - output space
+        # - scratch space
+        new_data = numpy.zeros((math.ceil(lines / factor), 8))
+
+        # Now decimate.
+        for i, k in enumerate([0,1,2,3,4,9,15,17]):
+            if factor > 1:
+                new_data[:, i] = signal.decimate(data[:, k], factor)
+            else:
+                new_data[:, i] = data[:, k]
+        
+        
+        self.data.metadata['performance'] = new_data
+        
         
     def parse_output_line(self, log_file, line):
         """

digichem/parse/dump.py

@@ -2,7 +2,7 @@
 import yaml
 import json
 
-from digichem.parse.base import Parser_abc
+from digichem.parse.base import File_parser_abc
 from digichem.result.tdm import Transition_dipole_moment
 from digichem.result.result import Result_set
 from digichem.result.metadata import Metadata
@@ -20,7 +20,7 @@ from digichem.result.alignment.base import Alignment, Minimal
 from digichem.result.nmr import NMR_list
 
 
-class Dump_multi_parser_abc(Parser_abc):
+class Dump_multi_parser_abc(File_parser_abc):
     """
     ABC for classes that can read multiple result sets from dumped data.
     """
@@ -122,7 +122,7 @@ class Json_multi_parser(Dump_multi_parser_abc):
             self.data = json.load(json_file)
 
 
-class Dump_parser_abc(Parser_abc):
+class Dump_parser_abc(File_parser_abc):
     """
     ABC for parsers that read dumped data.
     """

digichem/parse/orca.py

@@ -12,6 +12,7 @@ class Orca_parser(Cclib_parser):
     INPUT_FILE_TYPES = {
             file_types.orca_gbw_file: "gbw_file",
             file_types.orca_density_file: "density_file",
+            file_types.orca_density_info_file: "density_info_file"
         }
     
     def parse_output_line(self, log_file, line):

digichem/parse/pyscf.py

@@ -0,0 +1,25 @@
+import hashlib
+import json
+
+from cclib.bridge.cclib2pyscf import cclibfrommethods
+
+from digichem.parse.base import Parser_abc
+
+class Pyscf_parser(Parser_abc):
+    """
+    Top level class for parsing output from pyscf data.
+    """
+    
+    def __init__(self, mol_name, methods, **kwargs):
+        self.methods = methods
+        self.mol_name = mol_name
+        super().__init__(**kwargs)
+
+    def _parse(self):
+        self.data = cclibfrommethods(**self.methods)
+        # TODO: We need some way of generating a checksum
+        self.data._id = hashlib.sha1(json.dumps(dir(self.data), sort_keys = True).encode('utf-8')).hexdigest()
+        self.data.metadata['name'] = self.mol_name
+        self.data._aux = {'methods': self.methods}
+            
+            

digichem/parse/turbomole.py

@@ -25,9 +25,9 @@ class Turbomole_parser(Cclib_parser):
         """
         Sort a list of log files into a particular order, if required for this parser.
         """
-        from cclib.io.ccio import sort_turbomole_outputs
+        from cclib.parser.turbomoleparser import Turbomole
         
-        return sort_turbomole_outputs(log_files)
+        return Turbomole.sort_input(log_files)
     
     def duration_to_timedelta(self, duration_str):
         """
@@ -161,16 +161,16 @@ class Turbomole_parser(Cclib_parser):
         return self.results
             
     @classmethod
-    def find_auxiliary_files(self, hint):
+    def find_auxiliary_files(self, hint, basename):
         """
         Find auxiliary files from a given hint.
         
         :param hint: A path to a file to use as a hint to find additional files.
         :returns: A dictionary of found aux files.
         """
-        auxiliary_files = super().find_auxiliary_files(hint)
+        auxiliary_files = super().find_auxiliary_files(hint, basename)
         
-        parent = pathlib.Path(hint).parent
+        parent = pathlib.Path(hint).parent if not hint.is_dir() else hint
             
         # Find .cao density files.
         # First look for ground state density files.