ChessAnalysisPipeline 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

CHAP/common/reader.py

@@ -8,11 +8,12 @@ Description: Module for Writers used in multiple experiment-specific
 
 # System modules
 from os.path import (
+    isabs,
     isfile,
+    join,
     splitext,
 )
 from sys import modules
-from time import time
 
 # Third party modules
 import numpy as np
@@ -37,10 +38,39 @@ class BinaryFileReader(Reader):
         return data
 
 
+class FabioImageReader(Reader):
+    """Reader for images using the python package
+    [`fabio`](https://fabio.readthedocs.io/en/main/).
+    """
+    def read(self, filename, frame=None, inputdir='.'):
+        """Return the data from the image file(s) provided.
+
+        :param filename: The image filename, or glob pattern for image
+            filenames, to read.
+        :type filename: str
+        :param frame: The index of a specific frame to read from the
+            file(s), defaults to `None`.
+        :type filename: int, optional
+        :returns: Image data as a numpy array (or list of numpy
+            arrays, if a glob pattern matching more than one file was
+            provided).
+        """
+        from glob import glob
+        import fabio
+
+        filenames = glob(filename)
+        data = []
+        for f in filenames:
+            image = fabio.open(f, frame=frame)
+            data.append(image.data)
+            image.close()
+        return data
+
+
 class H5Reader(Reader):
     """Reader for h5 files.
     """
-    def read(self, filename, h5path='/'):
+    def read(self, filename, h5path='/', idx=None):
         """Return the data object stored at `h5path` in an h5-file.
 
         :param filename: The name of the h5-file to read from.
@@ -55,6 +85,8 @@ class H5Reader(Reader):
         from h5py import File
 
         data = File(filename, 'r')[h5path]
+        if idx is not None:
+            data = data[tuple(idx)]
         return data
 
 
@@ -142,8 +174,8 @@ class MapReader(Reader):
                 attrs={'long_name': f'{dim.label} ({dim.units})',
                        'data_type': dim.data_type,
                        'local_name': dim.name})
-            if map_config.map_type == 'structured':
-                nxentry.data.attrs[f'{dim.label}_indices'] = i
+#            if map_config.map_type == 'structured':
+#                nxentry.data.attrs[f'{dim.label}_indices'] = i
 
         # Create empty NXfields for all scalar data present in the
         # provided map configuration
@@ -167,15 +199,13 @@ class MapReader(Reader):
         # Create empty NXfields of appropriate shape for raw
         # detector data
         for detector_name in detector_names:
+            if not isinstance(detector_name, str):
+                detector_name = str(detector_name)
             detector_data = map_config.get_detector_data(
                 detector_name, (0,) * len(map_config.shape))
             nxentry.data[detector_name] = NXfield(value=np.zeros(
                 (*map_config.shape, *detector_data.shape)),
                 dtype=detector_data.dtype)
-#            data_shape = list(map_config.shape)+list(detector_data.shape)
-#            nxentry.data[detector_name] = NXfield(
-#                value=np.zeros(data_shape), shape=data_shape,
-#                dtype=detector_data.dtype)
 
         # Read and fill in maps of raw data
         if len(map_config.all_scalar_data) > 0 or len(detector_names) > 0:
@@ -184,6 +214,8 @@ class MapReader(Reader):
                     nxentry.data[data.label][map_index] = map_config.get_value(
                         data, map_index)
                 for detector_name in detector_names:
+                    if not isinstance(detector_name, str):
+                        detector_name = str(detector_name)
                     nxentry.data[detector_name][map_index] = \
                         map_config.get_detector_data(detector_name, map_index)
 
@@ -213,6 +245,146 @@ class NexusReader(Reader):
         return nxobject
 
 
+class NXdataReader(Reader):
+    """Reader for constructing an NXdata object from components"""
+    def read(self, name, nxfield_params, signal_name, axes_names, attrs={},
+             inputdir='.'):
+        """Return a basic NXdata object constructed from components.
+
+        :param name: The name of the NXdata group.
+        :type name: str
+        :param nxfield_params: List of sets of parameters for
+            `NXfieldReader` specifying the NXfields belonging to the
+            NXdata.
+        :type nxfield_params: list[dict]
+        :param signal_name: Name of the signal for the NXdata (must be
+            one of the names of the NXfields indicated in `nxfields`)
+        :type signal: str
+        :param axes_names: Name or names of the coordinate axes
+            NXfields associated with the signal (must be names of
+            NXfields indicated in `nxfields`)
+        :type axes_names: Union[str, list[str]]
+        :param attrs: Optional dictionary of additional attributes for
+            the NXdata
+        :type attrs: dict
+        :param inputdir: Input directory to use for `NXfieldReader`s,
+            defaults to `"."`
+        :type inputdir: str
+        :returns: A new NXdata object
+        :rtype: nexusformat.nexus.NXdata
+        """
+        from nexusformat.nexus import NXdata
+
+        # Read in NXfields
+        nxfields = [NXfieldReader().read(**params, inputdir=inputdir)
+                    for params in nxfield_params]
+        nxfields = {nxfield.nxname: nxfield for nxfield in nxfields}
+
+        # Get signal NXfield
+        try:
+            nxsignal = nxfields[signal_name]
+        except:
+            raise ValueError(
+                '`signal_name` must be the name of one of the NXfields '
+                + 'indicated in `nxfields`: '
+                + ', '.join(nxfields.keys())
+            )
+
+        # Get axes NXfield(s)
+        if isinstance(axes_names, str):
+            axes_names = [axes_names]
+        try:
+            nxaxes = [nxfields[axis_name] for axis_name in axes_names]
+        except:
+            raise ValueError(
+                '`axes_names` must contain only names of NXfields indicated '
+                + 'in `nxfields`: ' + ', '.join(nxfields.keys())
+            )
+        for i, nxaxis in enumerate(nxaxes):
+            if len(nxaxis) != nxsignal.shape[i]:
+                raise ValueError(
+                    f'Shape mismatch on signal dimension {i}: signal '
+                    + f'"{nxsignal.nxname}" has {nxsignal.shape[i]} values, '
+                    + f'but axis "{nxaxis.nxname}" has {len(nxaxis)} values.')
+
+        result = NXdata(signal=nxsignal, axes=nxaxes, name=name, attrs=attrs,
+                        **nxfields)
+        self.logger.info(result.tree)
+        return result
+
+
+class NXfieldReader(Reader):
+    """Reader for an NXfield with options to modify certain attributes."""
+    def read(self, filename, nxpath, nxname=None, update_attrs=None,
+             slice_params=None, inputdir='.'):
+        """Return a copy of the indicated NXfield from the file. Name
+        and attributes of the returned copy may be modified with the
+        `nxname` and `update_attrs` keyword arguments.
+
+        :param filename: Name of the NeXus file containing the NXfield to read.
+        :type filename: str
+        :param nxpath: Path in `nxfile` pointing to the NXfield to read.
+        :type nxpath: str
+        :param nxname: Optional new name for the returned NXfield,
+            defaults to None
+        :type nxname: str, optional
+        :param update_attrs: Optional dictonary used to add to /
+            update the original NXfield's attributes, defaults to None
+        :type update_attrs: dict, optional
+        :param slice_params: Parameters for returning just a slice of
+            the full field data. Slice parameters are provided in a
+            list dictionaries with integer values for any / all of the
+            following keys: `"start"`, `"end"`, `"step"`. Default
+            values used are: `"start"` - `0`, `"end"` -- `None`,
+            `"step"` -- `1`. The order of the list must correspond to
+            the order of the field's axes. Defaults to `None`.
+        :type slice_params: list[dict[str, int]], optional
+        :param inputdir: Directory containing `nxfile`, defaults to `"."`
+        :type inputdir: str
+        :returns: A copy of the indicated NXfield (with name and
+            attributes optionally modified).
+        :rtype: nexusformat.nexus.NXfield
+        """
+        # Third party modules
+        from nexusformat.nexus import nxload, NXfield
+
+        # Local modules
+        from CHAP.utils.general import nxcopy
+
+        if not isabs(filename):
+            filename = join(inputdir, filename)
+        nxroot = nxload(filename)
+        nxfield = nxroot[nxpath]
+
+        if nxname is None:
+            nxname = nxfield.nxname
+
+        attrs = nxfield.attrs
+        if update_attrs is not None:
+            attrs.update(update_attrs)
+
+        if slice_params is None:
+            value = nxfield.nxdata
+        else:
+            if len(slice_params) < nxfield.ndim:
+                slice_params.extend([{}] * (nxfield.ndim - len(slice_params)))
+            if len(slice_params) > nxfield.ndim:
+                slice_params = slice_params[0:nxfield.ndim]
+            slices = ()
+            default_slice = {'start': 0, 'end': None, 'step': 1}
+            for s in slice_params:
+                for k, v in default_slice.items():
+                    if k not in s:
+                        s[k] = v
+                slices = (*slices, slice(s['start'], s['end'], s['step']))
+            value = nxfield.nxdata[slices]
+
+        nxfield = NXfield(value=value, name=nxname, attrs=attrs)
+        self.logger.debug(f'Result -- nxfield.tree =\n{nxfield.tree}')
+
+        return nxfield
+
+
 class SpecReader(Reader):
     """Reader for CHESS SPEC scans"""
     def read(self, filename=None, spec_config=None, detector_names=[],

CHAP/common/writer.py

@@ -89,13 +89,15 @@ def write_yaml(data, filename, force_overwrite=False):
 
 def write_filetree(data, outputdir, force_overwrite=False):
     # System modules
-    from os import mkdir
+    from os import makedirs
 
     # Third party modules
     from nexusformat.nexus import (
         NXentry,
+        NXsubentry,
         NXgroup,
         NXobject,
+        NXroot,
         NXsubentry,
     )
 
@@ -104,7 +106,7 @@ def write_filetree(data, outputdir, force_overwrite=False):
                         f'{type(data).__name__} as a file tree to disk.')
 
     if not os_path.isdir(outputdir):
-        mkdir(outputdir)
+        makedirs(outputdir)
 
     for k, v in data.items():
         if isinstance(v, NXsubentry) and 'schema' in v.attrs:
@@ -114,15 +116,28 @@ def write_filetree(data, outputdir, force_overwrite=False):
                 write_txt(list(v.data), filename, force_overwrite)
             elif schema == 'json':
                 write_txt(str(v.data), filename, force_overwrite)
+            elif schema == 'yml' or schema == 'yaml':
+                from json import loads
+                write_yaml(loads(v.data.nxdata), filename, force_overwrite)
             elif schema == 'tif' or schema == 'tiff':
                 write_tif(v.data, filename, force_overwrite)
             elif schema == 'h5':
-                nxentry = NXentry()
+                if any(isinstance(vv, NXsubentry) for vv in v.values()):
+                    nxbase = NXroot()
+                else:
+                    nxbase = NXentry()
                 for kk, vv in v.attrs.items():
-                    nxentry.attrs[kk] = vv
+                    if kk not in ('schema', 'filename'):
+                        nxbase.attrs[kk] = vv
                 for kk, vv in v.items():
-                    nxentry[kk] = vv
-                write_nexus(nxentry, filename, force_overwrite)
+                    if isinstance(vv, NXsubentry):
+                        nxentry = NXentry()
+                        nxbase[vv.nxname] = nxentry
+                        for kkk, vvv in vv.items():
+                            nxentry[kkk] = vvv
+                    else:
+                        nxbase[kk] = vv
+                write_nexus(nxbase, filename, force_overwrite)
             else:
                 raise TypeError(f'Files of type {schema} not yet implemented')
         elif isinstance(v, NXgroup):
@@ -136,7 +151,7 @@ class ExtractArchiveWriter(Writer):
         and write the extracted archive to files.
 
         :param data: The data to write to archive.
-        :type data: CHAP.pipeline.PipelineData
+        :type data: list[PipelineData]
         :param filename: The name of the directory to write the archive
             files to.
         :type filename: str
@@ -162,7 +177,7 @@ class FileTreeWriter(Writer):
         directory tree stuctured like the NeXus tree.
 
         :param data: The data to write to disk.
-        :type data: CHAP.pipeline.PipelineData
+        :type data: list[PipelineData]
         :param outputdir: The name of the directory to write to.
         :type outputdir: str
         :param force_overwrite: Flag to allow data to be overwritten
@@ -204,6 +219,44 @@ class FileTreeWriter(Writer):
         return data
 
 
+class H5Writer(Writer):
+    """Writer for H5 files from an `nexusformat.nexus.NXdata` object"""
+    def write(self, data, filename, force_overwrite=False):
+        """Write the NeXus object contained in `data` to hdf5 file.
+
+        :param data: The data to write to file.
+        :type data: CHAP.pipeline.PipelineData
+        :param filename: The name of the file to write to.
+        :param force_overwrite: Flag to allow data in `filename` to be
+            overwritten if it already exists, defaults to `False`.
+        :type force_overwrite: bool, optional
+        :raises RuntimeError: If `filename` already exists and
+            `force_overwrite` is `False`.
+        :return: The data written to file.
+        :rtype: nexusformat.nexus.NXobject
+        """
+        # Third party modules
+        from h5py import File
+        from nexusformat.nexus import NXdata
+
+        data = self.unwrap_pipelinedata(data)[-1]
+        if not isinstance(data, NXdata):
+            raise ValueError('Invalid data parameter {(data)}')
+
+        mode = 'w' if force_overwrite else 'w-'
+        with File(filename, mode) as f:
+            f[data.signal] = data.nxsignal
+            for i, axes in enumerate(data.attrs['axes']):
+                f[axes] = data[axes]
+                f[data.signal].dims[i].label = \
+                    f'{axes} ({data[axes].units})' \
+                    if 'units' in data[axes].attrs else axes
+                f[axes].make_scale(axes)
+                f[data.signal].dims[i].attach_scale(f[axes])
+
+        return data
+
+
 class MatplotlibAnimationWriter(Writer):
     """Writer for saving matplotlib animations."""
     def write(self, data, filename, fps=1):
@@ -211,7 +264,7 @@ class MatplotlibAnimationWriter(Writer):
         contained in `data` to file.
 
         :param data: The matplotlib animation.
-        :type data: CHAP.pipeline.PipelineData
+        :type data: list[PipelineData]
         :param filename: The name of the file to write to.
         :type filename: str
         :param fps: Movie frame rate (frames per second),
@@ -224,7 +277,7 @@ class MatplotlibAnimationWriter(Writer):
         extension = os_path.splitext(filename)[1]
         if not extension:
             data.save(f'{filename}.gif', fps=fps)
-        elif extension in '.gif':
+        elif extension == '.gif':
             data.save(filename, fps=fps)
         elif extension == '.mp4':
             data.save(filename, writer='ffmpeg', fps=fps)
@@ -239,7 +292,7 @@ class MatplotlibFigureWriter(Writer):
         file.
 
         :param data: The matplotlib figure
-        :type data: CHAP.pipeline.PipelineData
+        :type data: list[PipelineData]
         :param filename: The name of the file to write to.
         :type filename: str
         :param savefig_kw: Keyword args to pass to
@@ -261,11 +314,11 @@ class MatplotlibFigureWriter(Writer):
 
 class NexusWriter(Writer):
     """Writer for NeXus files from `NXobject`-s"""
-    def write(self, data, filename, force_overwrite=False):
+    def write(self, data, filename, nxpath=None, force_overwrite=False):
         """Write the NeXus object contained in `data` to file.
 
         :param data: The data to write to file.
-        :type data: CHAP.pipeline.PipelineData
+        :type data: list[PipelineData]
         :param filename: The name of the file to write to.
         :param force_overwrite: Flag to allow data in `filename` to be
             overwritten if it already exists, defaults to `False`.
@@ -275,12 +328,130 @@ class NexusWriter(Writer):
         :return: The data written to file.
         :rtype: nexusformat.nexus.NXobject
         """
+        # Third party modules
+        from nexusformat.nexus import (
+            NXentry,
+            NXFile,
+            NXroot,
+        )
+        import os
         data = self.unwrap_pipelinedata(data)[-1]
-        write_nexus(data, filename, force_overwrite)
-
+        nxname = data.nxname
+        if not os.path.isfile(filename) and nxpath is not None:
+            self.logger.warning(
+                f'{filename} does not yet exist. Argument for nxpath ({nxpath}) '
+                + 'will be ignored.')
+            nxpath = None
+        if nxpath is None:
+            nxclass = data.nxclass
+            if nxclass == 'NXentry':
+                data = NXroot(data)
+                data[nxname].set_default()
+            elif nxclass != 'NXroot':
+                data = NXroot(NXentry(data))
+                if nxclass == 'NXdata':
+                    data.entry[nxname].set_default()
+                data.entry.set_default()
+            write_nexus(data, filename, force_overwrite)
+        else:
+            nxfile = NXFile(filename, 'rw')
+            root = nxfile.readfile()
+            if nxfile.get(nxpath) is None:
+                nxpath = root.NXentry[0].nxpath
+                self.logger.warning(
+                    f'Path "{nxpath}" not present in {filename}. '
+                    + f'Using {nxpath} instead.')
+            full_nxpath = os.path.join(nxpath, nxname)
+            self.logger.debug(f'Full path for object to write: {full_nxpath}')
+            if nxfile.get(full_nxpath) is not None:
+                self.logger.debug(
+                    f'{os.path.join(nxpath, nxname)} already exists in {filename}')
+                if force_overwrite:
+                    self.logger.warning(
+                        'Deleting existing NXobject at '
+                        + f'{os.path.join(nxpath, nxname)} in {filename}')
+                    del root[full_nxpath]
+            try:
+                root[full_nxpath] = data
+            except Exception as e:
+                nxfile.close()
+                raise e
+            nxfile.close()
         return data
 
 
+class PyfaiResultsWriter(Writer):
+    """Writer for results of one or more pyFAI integrations. Able to
+    handle multiple output formats. Currently supported formats are:
+    .npz, .nxs.
+    """
+    def write(self, data, filename, force_overwrite=False):
+        """Save pyFAI integration results to a file. Format is
+        determined automatically form the extension of `filename`.
+
+        :param data: Integration results to save.
+        :type data: Union[PipelineData,
+            list[pyFAI.containers.IntegrateResult]]
+        :param filename: Name of the file to which results will be
+            saved. Format of output is determined ffrom the
+            extension. Currently supported formats are: `.npz`,
+            `.nxs`
+        :type filename: str
+        """
+        import os
+
+        from pyFAI.containers import Integrate1dResult, Integrate2dResult
+
+        try:
+            results = self.unwrap_pipelinedata(data)[0]
+        except:
+            results = data
+        if not isinstance(results, list):
+            results = [results]
+        if not all([isinstance(r, Integrate1dResult) for r in results]) \
+           and not all([isinstance(r, Integrate2dResult) for r in results]):
+            raise Exception(
+                'Bad input data: all items must have the same type -- either '
+                + 'all pyFAI.containers.Integrate1dResult, or all '
+                + 'pyFAI.containers.Integrate2dResult.')
+
+        if os.path.isfile(filename):
+            if force_overwrite:
+                self.logger.warning(f'Removing existing file {filename}')
+                os.remove(filename)
+            else:
+                raise Exception(f'{filename} already exists.')
+        _, ext = os.path.splitext(filename)
+        if ext.lower() == '.npz':
+            self.write_npz(results, filename)
+        elif ext.lower() == '.nxs':
+            self.write_nxs(results, filename)
+        else:
+            raise Exception(f'Unsupported file format: {ext}')
+        self.logger.info(f'Wrote to {filename}')
+        return results
+
+    def write_npz(self, results, filename):
+        """Save `results` to the .npz file, `filename`"""
+        import numpy as np
+
+        data = {'radial': results[0].radial,
+                'intensity': [r.intensity for r in results]}
+        if hasattr(results[0], 'azimuthal'):
+            # 2d results
+            data['azimuthal'] = results[0].azimuthal
+        if all([r.sigma for r in results]):
+            # errors were included
+            data['sigma'] = [r.sigma for r in results]
+
+        np.savez(filename, **data)
+
+    def write_nxs(results, filename):
+        """Save `results` to the .nxs file, `filename`"""
+        raise NotImplementedError
+
+
+
 class TXTWriter(Writer):
     """Writer for plain text files from string or tuples or lists of
     strings."""
@@ -331,6 +502,12 @@ class YAMLWriter(Writer):
         :rtype: dict
         """
         data = self.unwrap_pipelinedata(data)[-1]
+        try:
+            from pydantic import BaseModel
+            if isinstance(data, BaseModel):
+                data = data.dict()
+        except:
+            pass
         write_yaml(data, filename, force_overwrite)
         return data
 

CHAP/edd/__init__.py

@@ -1,11 +1,20 @@
 """This subpackage contains `PipelineItems` unique to EDD data
 processing workflows.
 """
-# from CHAP.edd.reader import
+from CHAP.edd.reader import (EddMapReader,
+                             ScanToMapReader,
+                             SetupNXdataReader,
+                             UpdateNXdataReader,
+                             NXdataSliceReader)
 from CHAP.edd.processor import (DiffractionVolumeLengthProcessor,
-                                MCACeriaCalibrationProcessor,
+                                LatticeParameterRefinementProcessor,
+                                MCAEnergyCalibrationProcessor,
+                                MCATthCalibrationProcessor,
                                 MCADataProcessor,
-                                StrainAnalysisProcessor)
+                                MCAEnergyCalibrationProcessor,
+                                MCACalibratedDataPlotter,
+                                StrainAnalysisProcessor,
+                                CreateStrainAnalysisConfigProcessor)
 # from CHAP.edd.writer import
 
 from CHAP.common import MapProcessor