ChessAnalysisPipeline 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

CHAP/common/writer.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+'''
+File       : writer.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Module for Writers used in multiple experiment-specific workflows.
+'''
+
+# system modules
+import argparse
+import json
+import logging
+import os
+import sys
+
+# local modules
+from CHAP import Writer
+
+class ExtractArchiveWriter(Writer):
+    def _write(self, data, filename):
+        '''Take a .tar archive represented as bytes in `data` and write the
+        extracted archive to files.
+
+        :param data: the archive data
+        :type data: bytes
+        :param filename: the name of a directory to which the archive files will
+            be written
+        :type filename: str
+        :return: the original `data`
+        :rtype: bytes
+        '''
+
+        from io import BytesIO
+        import tarfile
+
+        tar = tarfile.open(fileobj=BytesIO(data))
+        tar.extractall(path=filename)
+
+        return(data)
+
+class NexusWriter(Writer):
+    def _write(self, data, filename, force_overwrite=False):
+        '''Write `data` to a NeXus file
+
+        :param data: the data to write to `filename`.
+        :type data: nexusformat.nexus.NXobject
+        :param filename: name of the file to write to.
+        :param force_overwrite: flag to allow data in `filename` to be
+            overwritten, if it already exists. 
+        :return: the original input data
+        '''
+
+        from nexusformat.nexus import NXobject
+        
+        if not isinstance(data, NXobject):
+            raise(TypeError(f'Cannot write object of type {type(data).__name__} to a NeXus file.'))
+
+        mode = 'w' if force_overwrite else 'w-'
+        data.save(filename, mode=mode)
+
+        return(data)
+
+class YAMLWriter(Writer):
+    def _write(self, data, filename, force_overwrite=False):
+        '''If `data` is a `dict`, write it to `filename`.
+
+        :param data: the dictionary to write to `filename`.
+        :type data: dict
+        :param filename: name of the file to write to.
+        :type filename: str
+        :param force_overwrite: flag to allow data in `filename` to be
+            overwritten if it already exists.
+        :type force_overwrite: bool
+        :raises TypeError: if `data` is not a `dict`
+        :raises RuntimeError: if `filename` already exists and
+            `force_overwrite` is `False`.
+        :return: the original input data
+        :rtype: dict
+        '''
+
+        import yaml
+
+        if not isinstance(data, (dict, list)):
+            raise(TypeError(f'{self.__name__}.write: input data must be a dict or list.'))
+
+        if not force_overwrite:
+            if os.path.isfile(filename):
+                raise(RuntimeError(f'{self.__name__}: {filename} already exists.'))
+
+        with open(filename, 'w') as outf:
+            yaml.dump(data, outf, sort_keys=False)
+
+        return(data)
+
+if __name__ == '__main__':
+    from CHAP.writer import main
+    main()

CHAP/edd/__init__.py

@@ -0,0 +1,7 @@
+# from CHAP.edd.reader import
+from CHAP.edd.processor import (MCACeriaCalibrationProcessor,
+                                MCADataProcessor)
+# from CHAP.edd.writer import
+
+from CHAP.common import (MapProcessor,
+                         StrainAnalysisProcessor)

CHAP/edd/models.py

@@ -0,0 +1,215 @@
+import numpy as np
+from pathlib import PosixPath
+from pydantic import (BaseModel,
+                      confloat,
+                      conint,
+                      conlist,
+                      constr,
+                      FilePath,
+                      validator)
+from scipy.interpolate import interp1d
+from typing import Optional
+
+
+class MCACeriaCalibrationConfig(BaseModel):
+    '''Class representing metadata required to perform a Ceria calibration for an
+    MCA detector.
+
+    :ivar spec_file: Path to the SPEC file containing the CeO2 scan
+    :ivar scan_number: Number of the CeO2 scan in `spec_file`
+    :ivar scan_step_index: Index of the scan step to use for calibration,
+        optional. If not specified, the calibration routine will be performed on
+        the average of all MCA spectra for the scan.
+
+    :ivar flux_file: csv file containing station beam energy in eV (column 0)
+        and flux (column 1)
+
+    :ivar detector_name: name of the MCA to calibrate
+    :ivar num_bins: number of channels on the MCA to calibrate
+    :ivar max_energy_kev: maximum channel energy of the MCA in keV
+
+    :ivar hexrd_h5_material_file: path to a HEXRD materials.h5 file containing an
+        entry for the material properties.
+    :ivar hexrd_h5_material_name: Name of the material entry in
+        `hexrd_h5_material_file`, defaults to `'CeO2'`.
+    :ivar lattice_parameter_angstrom: lattice spacing in angstrom to use for
+        the cubic CeO2 crystal, defaults to `5.41153`.
+
+    :ivar tth_max: detector rotation about hutch x axis, defaults to `90`.
+    :ivar hkl_tth_tol: minimum resolvable difference in 2&theta between two
+        unique HKL peaks, defaults to `0.15`.
+
+    :ivar fit_include_bin_ranges: list of MCA channel index ranges whose data
+        will be included in the calibration routine
+    :ivar fit_hkls: list of unique HKL indices to fit peaks for in the
+        calibration routine
+
+    :ivar tth_initial_guess: initial guess for 2&theta
+    :ivar slope_initial_guess: initial guess for detector channel energy
+        correction linear slope, defaults to `1.0`.
+    :ivar intercept_initial_guess: initial guess for detector channel energy
+        correction y-intercept, defaults to `0.0`.
+
+    :ivar tth_calibrated: calibrated value for 2&theta, defaults to None
+    :ivar slope_calibrated: calibrated value for detector channel energy
+        correction linear slope, defaults to `None`
+    :ivar intercept_calibrated: calibrated value for detector channel energy
+        correction y-intercept, defaluts to None
+
+    :ivar max_iter: maximum number of iterations of the calibration routine,
+        defaults to `10`.
+    :ivar tune_tth_tol: stop iteratively tuning 2&theta when an iteration
+        produces a change in the tuned value of 2&theta that is smaller than this
+        value, defaults to `1e-8`.
+    '''
+
+    spec_file: FilePath
+    scan_number: conint(gt=0)
+    scan_step_index: Optional[conint(ge=0)]
+
+    flux_file: FilePath
+
+    detector_name: constr(strip_whitespace=True, min_length=1)
+    num_bins: conint(gt=0)
+    max_energy_kev: confloat(gt=0)
+
+    hexrd_h5_material_file: FilePath
+    hexrd_h5_material_name: constr(strip_whitespace=True, min_length=1) = 'CeO2'
+    lattice_parameter_angstrom: confloat(gt=0) = 5.41153
+
+    tth_max: confloat(gt=0, allow_inf_nan=False) = 90.0
+    hkl_tth_tol: confloat(gt=0, allow_inf_nan=False) = 0.15
+
+    fit_include_bin_ranges: conlist(min_items=1,
+                                    item_type=conlist(item_type=conint(ge=0),
+                                                      min_items=2,
+                                                      max_items=2))
+    fit_hkls: conlist(item_type=conint(ge=0), min_items=1)
+
+    tth_initial_guess: confloat(gt=0, le=tth_max, allow_inf_nan=False)
+    slope_initial_guess: float = 1.0
+    intercept_initial_guess: float = 0.0
+    tth_calibrated: Optional[confloat(gt=0, allow_inf_nan=False)]
+    slope_calibrated: Optional[confloat(allow_inf_nan=False)]
+    intercept_calibrated: Optional[confloat(allow_inf_nan=False)]
+
+    max_iter: conint(gt=0) = 10
+    tune_tth_tol: confloat(ge=0) = 1e-8
+
+    @validator('fit_include_bin_ranges', each_item=True)
+    def validate_include_bin_range(cls, value, values):
+        '''Ensure no bin ranges are outside the boundary of the detector'''
+
+        num_bins = values.get('num_bins')
+        value[1] = min(value[1], num_bins)
+        return(value)
+
+    def mca_data(self):
+        '''Get the 1D array of MCA data to use for calibration.
+        
+        :return: MCA data
+        :rtype: np.ndarray
+        '''
+
+        from CHAP.common.utils.scanparsers import SMBMCAScanParser as ScanParser
+        scanparser = ScanParser(self.spec_file, self.scan_number)
+        if self.scan_step_index is None:
+            data = scanparser.get_all_detector_data(self.detector_name)
+            if scanparser.spec_scan_npts > 1:
+                data = np.average(data, axis=1)
+            else:
+                data = data[0]
+        else:
+            data = scanparser.get_detector_data(self.detector_name, self.scan_step_index)
+
+        return(np.array(data))
+
+    def mca_mask(self):
+        '''Get a boolean mask array to use on MCA data before fitting.
+
+        :return: boolean mask array
+        :rtype: numpy.ndarray
+        '''
+
+        mask = np.asarray([False]*self.num_bins)
+        bin_indices = np.arange(self.num_bins)
+        for min_, max_ in self.fit_include_bin_ranges:
+            _mask =  np.logical_and(bin_indices > min_, bin_indices < max_)
+            mask = np.logical_or(mask, _mask)
+
+        return(mask)
+
+    def flux_correction_interpolation_function(self):
+        '''Get an interpolation function to correct MCA data for relative energy
+        flux of the incident beam.
+
+        :return: energy flux correction interpolation function
+        :rtype: scipy.interpolate._polyint._Interpolator1D
+        '''
+
+        flux = np.loadtxt(self.flux_file)
+        energies = flux[:,0]/1.e3
+        relative_intensities = flux[:,1]/np.max(flux[:,1])
+        interpolation_function = interp1d(energies, relative_intensities)
+        return(interpolation_function)
+
+    def material(self):
+        '''Get CeO2 as a `CHAP.common.utils.Material` object.
+
+        :return: CeO2 material
+        :rtype: CHAP.common.utils.Material
+        '''
+
+        from CHAP.common.utils import Material
+        material = Material(material_name=self.hexrd_h5_material_name,
+                            material_file=self.hexrd_h5_material_file,
+                            lattice_parameters_angstroms=self.lattice_parameter_angstrom)
+        # The following kwargs will be needed if we allow the material to be
+        # built using xrayutilities (for now, we only allow hexrd to make the
+        # material):
+                            # sgnum=225,
+                            # atoms=['Ce4p', 'O2mdot'],
+                            # pos=[(0.,0.,0.), (0.25,0.75,0.75)],
+                            # enrgy=50000.) # Why do we need to specify an energy to get HKLs when using xrayutilities?
+        return(material)
+
+    def unique_ds(self):
+        '''Get a list of unique HKLs and their lattice spacings
+        
+        :return: unique HKLs and their lattice spacings in angstroms
+        :rtype: np.ndarray, np.ndarray
+        '''
+        
+        unique_hkls, unique_ds = self.material().get_unique_ds(tth_tol=self.hkl_tth_tol, tth_max=self.tth_max)
+
+        return(unique_hkls, unique_ds)
+
+    def fit_ds(self):
+        '''Get a list of HKLs and their lattice spacings that will be fit in the
+        calibration routine
+        
+        :return: HKLs to fit and their lattice spacings in angstroms
+        :rtype: np.ndarray, np.ndarray
+        '''
+        
+        unique_hkls, unique_ds = self.unique_ds()
+
+        fit_hkls = np.array([unique_hkls[i] for i in self.fit_hkls])
+        fit_ds = np.array([unique_ds[i] for i in self.fit_hkls])
+
+        return(fit_hkls, fit_ds)
+
+    def dict(self):
+        '''Return a representation of this configuration in a dictionary that is
+        suitable for dumping to a YAML file (one that converts all instances of
+        fields with type `PosixPath` to `str`).
+
+        :return: dictionary representation of the configuration.
+        :rtype: dict
+        '''
+
+        d = super().dict()
+        for k,v in d.items():
+            if isinstance(v, PosixPath):
+                d[k] = str(v)
+        return(d)

CHAP/edd/processor.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+'''
+File       : processor.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Module for Processors used only by EDD experiments
+'''
+
+# system modules
+import json
+
+# local modules
+from CHAP.processor import Processor
+from CHAP.common import StrainAnalysisProcessor
+
+class MCACeriaCalibrationProcessor(Processor):
+    '''A Processor using a CeO2 scan to obtain tuned values for the bragg
+    diffraction angle and linear correction parameters for MCA channel energies
+    for an EDD experimental setup.
+    '''
+
+    def _process(self, data):
+        '''Return tuned values for 2&theta and linear correction parameters for
+        the MCA channel energies.
+
+        :param data: input configuration for the raw data & tuning procedure
+        :type data: list[dict[str,object]]
+        :return: original configuration dictionary with tuned values added
+        :rtype: dict[str,float]
+        '''
+
+        calibration_config = self.get_config(data)
+
+        tth, slope, intercept = self.calibrate(calibration_config)
+
+        calibration_config.tth_calibrated = tth
+        calibration_config.slope_calibrated = slope
+        calibration_config.intercept_calibrated = intercept
+
+        return(calibration_config.dict())
+
+    def get_config(self, data):
+        '''Get an instance of the configuration object needed by this
+        `Processor` from a returned value of `Reader.read`
+
+        :param data: Result of `Reader.read` where at least one item has the
+            value `'MCACeriaCalibrationConfig'` for the `'schema'` key.
+        :type data: list[dict[str,object]]
+        :raises Exception: If a valid config object cannot be constructed from
+            `data`.
+        :return: a valid instance of a configuration object with field values
+            taken from `data`.
+        :rtype: MCACeriaCalibrationConfig
+        '''
+
+        from CHAP.edd.models import MCACeriaCalibrationConfig
+
+        calibration_config = False
+        if isinstance(data, list):
+            for item in data:
+                if isinstance(item, dict):
+                    if item.get('schema') == 'MCACeriaCalibrationConfig':
+                        calibration_config = item.get('data')
+                        break
+
+        if not calibration_config:
+            raise(ValueError('No MCA ceria calibration configuration found in input data'))
+
+        return(MCACeriaCalibrationConfig(**calibration_config))
+
+    def calibrate(self, calibration_config):
+        '''Iteratively calibrate 2&theta by fitting selected peaks of an MCA
+        spectrum until the computed strain is sufficiently small. Use the fitted
+        peak locations to determine linear correction parameters for the MCA's
+        channel energies.
+
+        :param calibration_config: object configuring the CeO2 calibration
+            procedure
+        :type calibration_config: MCACeriaCalibrationConfig
+        :return: calibrated values of 2&theta and linear correction parameters
+            for MCA channel energies : tth, slope, intercept
+        :rtype: float, float, float
+        '''
+
+        from CHAP.common.utils import Fit, FitMultipeak
+        import numpy as np
+        from scipy.constants import physical_constants
+
+        hc = (physical_constants['Planck constant in eV/Hz'][0]
+              * physical_constants['speed of light in vacuum'][0]
+              * 1e7) # We'll work in keV and A, not eV and m.
+
+        # Collect raw MCA data of interest
+        mca_data = calibration_config.mca_data()
+        mca_bin_energies = (np.arange(0, calibration_config.num_bins)
+                            * (calibration_config.max_energy_kev
+                               / calibration_config.num_bins))
+
+        # Mask out the corrected MCA data for fitting
+        mca_mask = calibration_config.mca_mask()
+        fit_mca_energies = mca_bin_energies[mca_mask]
+        fit_mca_intensities = mca_data[mca_mask]
+
+        # Correct raw MCA data for variable flux at different energies
+        flux_correct = calibration_config.flux_correction_interpolation_function()
+        mca_intensity_weights = flux_correct(fit_mca_energies)
+        fit_mca_intensities = fit_mca_intensities / mca_intensity_weights
+
+        # Get the HKLs and lattice spacings that will be used for fitting
+        tth = calibration_config.tth_initial_guess
+        fit_hkls, fit_ds = calibration_config.fit_ds()
+        c_1 = fit_hkls[:,0]**2 + fit_hkls[:,1]**2 + fit_hkls[:,2]**2
+
+        for iter_i in range(calibration_config.max_iter):
+
+            ### Perform the uniform fit first ###
+
+            # Get expected peak energy locations for this iteration's starting
+            # value of tth
+            fit_lambda = 2.0 * fit_ds * np.sin(0.5*np.radians(tth))
+            fit_E0 = hc / fit_lambda
+
+            # Run the uniform fit
+            best_fit, residual, best_values, best_errors, redchi, success = \
+                FitMultipeak.fit_multipeak(
+                    fit_mca_intensities,
+                    fit_E0,
+                    x=fit_mca_energies,
+                    fit_type='uniform')
+
+            # Extract values of interest from the best values for the uniform fit
+            # parameters
+            uniform_fit_centers = [best_values[f'peak{i+1}_center'] for i in range(len(calibration_config.fit_hkls))]
+            # uniform_a = best_values['scale_factor']
+            # uniform_strain = np.log(
+            #     (uniform_a 
+            #      / calibration_config.lattice_parameter_angstrom))
+            # uniform_tth = tth * (1.0 + uniform_strain)
+            # uniform_rel_rms_error = (np.linalg.norm(residual)
+            #                          / np.linalg.norm(fit_mca_intensities))
+
+            ### Next, perform the unconstrained fit ###
+
+            # Use the peak locations found in the uniform fit as the initial
+            # guesses for peak locations in the unconstrained fit
+            best_fit, residual, best_values, best_errors, redchi, success = \
+                FitMultipeak.fit_multipeak(
+                    fit_mca_intensities,
+                    uniform_fit_centers,
+                    x=fit_mca_energies,
+                    fit_type='unconstrained')
+
+            # Extract values of interest from the best values for the
+            # unconstrained fit parameters
+            unconstrained_fit_centers = np.array(
+                [best_values[f'peak{i+1}_center'] for i in range(len(calibration_config.fit_hkls))])
+            unconstrained_a = (0.5 * hc * np.sqrt(c_1)
+                               / (unconstrained_fit_centers
+                                  * abs(np.sin(0.5*np.radians(tth)))))
+            unconstrained_strains = np.log(
+                (unconstrained_a
+                 / calibration_config.lattice_parameter_angstrom))
+            unconstrained_strain = np.mean(unconstrained_strains)
+            unconstrained_tth = tth * (1.0 + unconstrained_strain)
+            unconstrained_rel_rms_error = (np.linalg.norm(residual)
+                                           / np.linalg.norm(fit_mca_intensities))
+
+
+            # Update tth for the next iteration of tuning
+            prev_tth = tth
+            tth = unconstrained_tth
+
+            # Stop tuning tth at this iteration if differences are small enough
+            if abs(tth - prev_tth) < calibration_config.tune_tth_tol:
+                break
+
+        # Fit line to expected / computed peak locations from the last
+        # unconstrained fit.
+        fit = Fit.fit_data(
+            fit_E0,
+            'linear',
+            x=unconstrained_fit_centers,
+            nan_policy='omit')
+        slope = fit.best_values['slope']
+        intercept = fit.best_values['intercept']
+
+        return(float(tth), float(slope), float(intercept))
+
+class MCADataProcessor(Processor):
+    '''A Processor to return data from an MCA, restuctured to incorporate the
+    shape & metadata associated with a map configuration to which the MCA data
+    belongs, and linearly transformed according to the results of a ceria
+    calibration.
+    '''
+
+    def _process(self, data):
+        '''Process configurations for a map and MCA detector(s), and return the
+        calibrated MCA data collected over the map.
+
+        :param data: input map configuration and results of ceria calibration
+        :type data: list[dict[str,object]]
+        :return: calibrated and flux-corrected MCA data
+        :rtype: nexusformat.nexus.NXentry
+        '''
+
+        map_config, calibration_config = self.get_configs(data)
+        nxroot = self.get_nxroot(map_config, calibration_config)
+
+        return(nxroot)
+
+    def get_configs(self, data):
+        '''Get instances of the configuration objects needed by this
+        `Processor` from a returned value of `Reader.read`
+
+        :param data: Result of `Reader.read` where at least one item has the
+            value `'MapConfig'` for the `'schema'` key, and at least one item has
+            the value `'MCACeriaCalibrationConfig'` for the `'schema'` key.
+        :type data: list[dict[str,object]]
+        :raises Exception: If valid config objects cannot be constructed from
+            `data`.
+        :return: valid instances of the configuration objects with field values
+            taken from `data`.
+        :rtype: tuple[MapConfig, MCACeriaCalibrationConfig]
+        '''
+
+        from CHAP.common.models import MapConfig
+        from CHAP.edd.models import MCACeriaCalibrationConfig
+
+        map_config = False
+        calibration_config = False
+        if isinstance(data, list):
+            for item in data:
+                if isinstance(item, dict):
+                    schema = item.get('schema')
+                    if schema == 'MapConfig':
+                        map_config = item.get('data')
+                    elif schema == 'MCACeriaCalibrationConfig':
+                        calibration_config = item.get('data')
+
+        if not map_config:
+            raise(ValueError('No map configuration found in input data'))
+        if not calibration_config:
+            raise(ValueError('No MCA ceria calibration configuration found in input data'))
+
+        return(MapConfig(**map_config), MCACeriaCalibrationConfig(**calibration_config))
+
+    def get_nxroot(self, map_config, calibration_config):
+        '''Get a map of the MCA data collected by the scans in `map_config`. The
+        MCA data will be calibrated and flux-corrected according to the
+        parameters included in `calibration_config`. The data will be returned
+        along with relevant metadata in the form of a NeXus structure.
+
+        :param map_config: the map configuration
+        :type map_config: MapConfig
+        :param calibration_config: the calibration configuration
+        :type calibration_config: MCACeriaCalibrationConfig
+        :return: a map of the calibrated and flux-corrected MCA data
+        :rtype: nexusformat.nexus.NXroot
+        '''
+
+        from CHAP.common import MapProcessor
+
+        from nexusformat.nexus import (NXdata,
+                                       NXdetector,
+                                       NXentry,
+                                       NXinstrument,
+                                       NXroot)
+        import numpy as np
+
+        nxroot = NXroot()
+
+        nxroot[map_config.title] = MapProcessor.get_nxentry(map_config)
+        nxentry = nxroot[map_config.title]
+
+        nxentry.instrument = NXinstrument()
+        nxentry.instrument.detector = NXdetector()
+        nxentry.instrument.detector.calibration_configuration = json.dumps(calibration_config.dict())
+
+        nxentry.instrument.detector.data = NXdata()
+        nxdata = nxentry.instrument.detector.data
+        nxdata.raw = np.empty((*map_config.shape, calibration_config.num_bins))
+        nxdata.raw.attrs['units'] = 'counts'
+        nxdata.channel_energy = (calibration_config.slope_calibrated
+                                * np.arange(0, calibration_config.num_bins)
+                                * (calibration_config.max_energy_kev
+                                   / calibration_config.num_bins)
+                                + calibration_config.intercept_calibrated)
+        nxdata.channel_energy.attrs['units'] = 'keV'
+
+        for scans in map_config.spec_scans:
+            for scan_number in scans.scan_numbers:
+                scanparser = scans.get_scanparser(scan_number)
+                for scan_step_index in range(scanparser.spec_scan_npts):
+                    map_index = scans.get_index(
+                        scan_number,
+                        scan_step_index,
+                        map_config)
+                    nxdata.raw[map_index] = scanparser.get_detector_data(
+                        calibration_config.detector_name,
+                        scan_step_index)
+
+        nxentry.data.makelink(
+            nxdata.raw,
+            name=calibration_config.detector_name)
+        nxentry.data.makelink(
+            nxdata.channel_energy,
+            name=f'{calibration_config.detector_name}_channel_energy')
+        if isinstance(nxentry.data.attrs['axes'], str):
+            nxentry.data.attrs['axes'] = [
+                nxentry.data.attrs['axes'],
+                f'{calibration_config.detector_name}_channel_energy']
+        else:
+            nxentry.data.attrs['axes'] += [f'{calibration_config.detector_name}_channel_energy']
+        nxentry.data.attrs['signal'] = calibration_config.detector_name
+
+        return(nxroot)
+
+if __name__ == '__main__':
+    from CHAP.processor import main
+    main()

CHAP/edd/reader.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python  
+
+if __name__ == '__main__':
+    from CHAP.reader import main
+    main()

CHAP/edd/writer.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python  
+
+if __name__ == '__main__':
+    from CHAP.writer import main
+    main()

CHAP/inference/__init__.py

@@ -0,0 +1,3 @@
+# from CHAP.inference.reader import
+from CHAP.inference.processor import TFaaSImageProcessor
+# from CHAP.inference.writer import