ChessAnalysisPipeline 0.0.14__py3-none-any.whl → 0.0.15__py3-none-any.whl

CHAP/foxden/__init__.py

@@ -0,0 +1,6 @@
+"""This subpackage contains pieces for communication with FOXDEN services.
+"""
+
+from CHAP.foxden.processor import (
+    FoxdenProvenanceProcessor,
+)

CHAP/foxden/processor.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : processor.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Processor module for FOXDEN services
+"""
+
+# system modules
+from time import time
+
+# local modules
+from CHAP import Processor
+from CHAP.foxden.writer import FoxdenWriter
+
+
+class FoxdenProvenanceProcessor(Processor):
+    """A Processor to communicate with FOXDEN provenance server."""
+#     def __init__(self):
+#         self.writer = FoxdenWriter()
+
+    def process(self, data, url, dryRun=False, verbose=False):
+        """process data API"""
+
+        t0 = time()
+        self.logger.info(f'Executing "process" with url {url} data {data} dryrun {dryRun}')
+        writer = FoxdenWriter()
+
+#         data = self.writer.write(data, url, dryRun)
+        data = writer.write(data, url, dryRun=dryRun)
+
+        self.logger.info(f'Finished "process" in {time()-t0:.3f} seconds\n')
+
+        return data
+
+
+if __name__ == '__main__':
+    # local modules
+    from CHAP.processor import main
+
+    main()

CHAP/foxden/writer.py

@@ -0,0 +1,65 @@
+"""FOXDE command line writer."""
+
+# system modules
+import os
+
+# Local modules
+from CHAP.writer import main
+
+class FoxdenWriter():
+    """FOXDEN writer writes data to specific FOXDEN service
+    """
+
+    def write(self, data, url, method="POST", headers={}, timeout=10, dryRun=False):
+        """Write the input data as text to a file.
+
+        :param data: input data
+        :type data: list[PipelineData]
+        :param url: url of service
+        :type url: str
+        :param method: HTTP method to use, POST for creation and PUT for update
+        :type method: str
+        :param headers: HTTP headers to use
+        :type headers: dictionary
+        :param timeout: timeout of HTTP request
+        :type timeout: str
+        :param dryRun: dryRun option to verify HTTP workflow
+        :type dryRun: boolean
+        :return: contents of the input data
+        :rtype: object
+        """
+        import requests
+        if 'Content-Type' not in headers:
+            headers['Content-type'] = 'application/json'
+        if 'Accept' not in headers:
+            headers['Accept'] = 'application/json'
+        if dryRun:
+            print("### HTTP writer call", url, headers, data)
+            return []
+        token = ""
+        fname = os.getenv("CHESS_WRITE_TOKEN")
+        if not fname:
+            msg = f'CHESS_WRITE_TOKEN env variable is not set'
+            raise Exception(msg)
+        with open(fname, 'r') as istream:
+            token = istream.read()
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        else:
+            msg = f'No valid write token found in CHESS_WRITE_TOKEN env variable'
+            raise Exception(msg)
+
+        # make actual HTTP request to FOXDEN service
+        if method.lower() == 'post':
+            resp = requests.post(url, headers=headers, timeout=timeout, data=data)
+        elif method.lower() == 'put':
+            resp = requests.put(url, headers=headers, timeout=timeout, data=data)
+        else:
+            msg = f"unsupporteed method {method}"
+            raise Exception(msg)
+        data = resp.content
+        return data
+
+
+if __name__ == '__main__':
+    main()

CHAP/pipeline.py

@@ -227,7 +227,7 @@ class MultiplePipelineItem(PipelineItem):
                 outputdir = os.path.normpath(os.path.join(
                     args['outputdir'], item_args.pop('outputdir')))
                 if not os.path.isdir(outputdir):
-                    os.mkdir(outputdir)
+                    os.makedirs(outputdir)
                 try:
                     tmpfile = NamedTemporaryFile(dir=outputdir)
                 except:

CHAP/runner.py

@@ -37,7 +37,7 @@ class RunConfig():
 
         # Check if root exists (create it if not) and is readable
         if not os.path.isdir(self.root):
-            os.mkdir(self.root)
+            os.makedirs(self.root)
         if not os.access(self.root, os.R_OK):
             raise OSError('root directory is not accessible for reading '
                           f'({self.root})')
@@ -57,7 +57,7 @@ class RunConfig():
             self.outputdir = os.path.realpath(
                 os.path.join(self.root, self.outputdir))
         if not os.path.isdir(self.outputdir):
-            os.mkdir(self.outputdir)
+            os.makedirs(self.outputdir)
         try:
             tmpfile = NamedTemporaryFile(dir=self.outputdir)
         except:
@@ -127,7 +127,7 @@ def setLogger(log_level="INFO"):
     logger.setLevel(log_level)
     log_handler = logging.StreamHandler()
     log_handler.setFormatter(logging.Formatter(
-        '{name:20}: {message}', style='{'))
+        '{name:20}: {levelname}: {message}', style='{'))
     logger.addHandler(log_handler)
     return logger, log_handler
 
@@ -170,7 +170,7 @@ def run(
                 newoutputdir = os.path.normpath(os.path.join(
                     kwargs['outputdir'], item_args.pop('outputdir')))
                 if not os.path.isdir(newoutputdir):
-                    os.mkdir(newoutputdir)
+                    os.makedirs(newoutputdir)
                 try:
                     tmpfile = NamedTemporaryFile(dir=newoutputdir)
                 except:

CHAP/tomo/models.py

@@ -7,7 +7,6 @@ from typing import (
 )
 from pydantic import (
     BaseModel,
-    StrictBool,
     conint,
     conlist,
     confloat,
@@ -168,9 +167,11 @@ class TomoSimConfig(BaseModel):
     :type detector: Detector
     :ivar sample_type: Sample type for the tomography simulator.
     :type sample_type: Literal['square_rod', 'square_pipe',
-        'hollow_cube', 'hollow_brick']
+        'hollow_cube', 'hollow_brick', 'hollow_pyramid']
     :ivar sample_size: Size of each sample dimension in mm (internally
-        converted to an integer number of pixels).
+        converted to an integer number of pixels). Enter three values
+        for sample_type == `'hollow_pyramid'`, the height and the side
+        at the respective bottom and the top of the pyramid.
     :type sample_size: list[float]
     :ivar wall_thickness: Wall thickness for pipe, cube, and brick in
         mm (internally converted to an integer number of pixels).
@@ -192,10 +193,11 @@ class TomoSimConfig(BaseModel):
     station: Literal['id1a3', 'id3a', 'id3b']
     detector: Detector.construct()
     sample_type: Literal[
-        'square_rod', 'square_pipe', 'hollow_cube', 'hollow_brick']
+        'square_rod', 'square_pipe', 'hollow_cube', 'hollow_brick',
+        'hollow_pyramid']
     sample_size: conlist(
         item_type=confloat(gt=0, allow_inf_nan=False),
-        min_items=1, max_items=2)
+        min_items=1, max_items=3)
     wall_thickness: Optional[confloat(ge=0, allow_inf_nan=False)]
     mu: Optional[confloat(gt=0, allow_inf_nan=False)] = 0.05
     theta_step: confloat(gt=0, allow_inf_nan=False)

CHAP/tomo/processor.py

@@ -8,7 +8,6 @@ Description: Module for Processors used only by tomography experiments
 """
 
 # System modules
-from os import mkdir
 from os import path as os_path
 from sys import exit as sys_exit
 from time import time
@@ -31,7 +30,6 @@ from CHAP.utils.general import (
     quick_imshow,
     nxcopy,
 )
-from CHAP.utils.fit import Fit
 from CHAP.processor import Processor
 from CHAP.reader import main
 
@@ -1395,28 +1393,28 @@ class Tomo:
 
         # Resize the combined tomography data stacks
         # - combined axis data order: row/-z,y,x
-        if self._interactive:
+        if self._interactive or self._save_figs:
             x_bounds, y_bounds, z_bounds = self._resize_reconstructed_data(
                 tomo_recon_combined, combine_data=True)
         else:
             x_bounds = tool_config.x_bounds
             if x_bounds is None:
                 self._logger.warning(
-                    'x_bounds unspecified, reconstruct data for full x-range')
+                    'x_bounds unspecified, combine data for full x-range')
             elif not is_int_pair(
                     x_bounds, ge=0, le=tomo_shape[2]):
                 raise ValueError(f'Invalid parameter x_bounds ({x_bounds})')
             y_bounds = tool_config.y_bounds
             if y_bounds is None:
                 self._logger.warning(
-                    'y_bounds unspecified, reconstruct data for full y-range')
+                    'y_bounds unspecified, combine data for full y-range')
             elif not is_int_pair(
                     y_bounds, ge=0, le=tomo_shape[1]):
                 raise ValueError(f'Invalid parameter y_bounds ({y_bounds})')
             z_bounds = tool_config.z_bounds
             if z_bounds is None:
                 self._logger.warning(
-                    'z_bounds unspecified, reconstruct data for full z-range')
+                    'z_bounds unspecified, combine data for full z-range')
             elif not is_int_pair(
                     z_bounds, ge=0, le=tomo_shape[0]):
                 raise ValueError(f'Invalid parameter z_bounds ({z_bounds})')
@@ -1706,18 +1704,46 @@ class Tomo:
             img_row_bounds = calibrate_center_rows
         else:
             if nxentry.instrument.source.attrs['station'] in ('id1a3', 'id3a'):
+                # System modules
+                from sys import float_info
+
+                # Third party modules
+                from nexusformat.nexus import (
+                    NXdata,
+                    NXfield,
+                )
+
+                # Local modules
+                from CHAP.utils.fit import FitProcessor
+
                 pixel_size = float(nxentry.instrument.detector.row_pixel_size)
                 # Try to get a fit from the bright field
                 row_sum = np.sum(tbf, 1)
-                fit = Fit.fit_data(
-                    row_sum, 'rectangle', x=np.array(range(len(row_sum))),
-                    form='atan', guess=True)
-                parameters = fit.best_values
+                num = len(row_sum)
+                fit = FitProcessor()
+                model = {'model': 'rectangle',
+                         'parameters': [
+                             {'name': 'amplitude',
+                              'value': row_sum.max()-row_sum.min(),
+                              'min': 0.0},
+                             {'name': 'center1', 'value': 0.25*num,
+                                 'min': 0.0, 'max': num},
+                             {'name': 'sigma1', 'value': num/7.0,
+                              'min': float_info.min},
+                             {'name': 'center2', 'value': 0.75*num,
+                              'min': 0.0, 'max': num},
+                             {'name': 'sigma2', 'value': num/7.0,
+                              'min': float_info.min}]}
+                bounds_fit = fit.process(
+                    NXdata(NXfield(row_sum, 'y'),
+                    NXfield(np.array(range(num)), 'x')),
+                    {'models': [model], 'method': 'trf'})
+                parameters = bounds_fit.best_values
                 row_low_fit = parameters.get('center1', None)
                 row_upp_fit = parameters.get('center2', None)
                 sig_low = parameters.get('sigma1', None)
                 sig_upp = parameters.get('sigma2', None)
-                have_fit = (fit.success and row_low_fit is not None
+                have_fit = (bounds_fit.success and row_low_fit is not None
                     and row_upp_fit is not None and sig_low is not None
                     and sig_upp is not None
                     and 0 <= row_low_fit < row_upp_fit <= row_sum.size
@@ -1787,7 +1813,7 @@ class Tomo:
         if calibrate_center_rows:
             title='Select two detector image row indices to '\
                   'calibrate rotation axis (in range '\
-                  f'[0, {first_image.shape[0]}])'
+                  f'[0, {first_image.shape[0]-1}])'
         else:
             title='Select detector image row bounds for data '\
                   f'reduction (in range [0, {first_image.shape[0]}])'
@@ -2452,8 +2478,9 @@ class Tomo:
                             f'{selected_center_offset:.2f}.png'))
             plt.close()
 
+            del recon_planes
+
         del sinogram
-        del recon_planes
 
         # Return the center location
         if self._interactive:
@@ -2834,16 +2861,16 @@ class Tomo:
         # Selecting x an y bounds (in z-plane)
         if x_bounds is None:
             if not self._interactive:
-                self._logger.warning('x_bounds unspecified, reconstruct '
-                                     'data for full x-range')
+                self._logger.warning('x_bounds unspecified, use data for '
+                                     'full x-range')
                 x_bounds = (0, tomo_recon_stacks[0].shape[2])
         elif not is_int_pair(
                 x_bounds, ge=0, le=tomo_recon_stacks[0].shape[2]):
             raise ValueError(f'Invalid parameter x_bounds ({x_bounds})')
         if y_bounds is None:
             if not self._interactive:
-                self._logger.warning('y_bounds unspecified, reconstruct '
-                                     'data for full y-range')
+                self._logger.warning('y_bounds unspecified, use data for '
+                                     'full y-range')
                 y_bounds = (0, tomo_recon_stacks[0].shape[1])
         elif not is_int_pair(
                 y_bounds, ge=0, le=tomo_recon_stacks[0].shape[1]):
@@ -2865,11 +2892,20 @@ class Tomo:
         tomosum = 0
         for i in range(num_tomo_stacks):
             tomosum = tomosum + np.sum(tomo_recon_stacks[i], axis=0)
-        fig, roi = select_roi_2d(
+        if self._save_figs:
+            if combine_data:
+                filename = os_path.join(
+                    self._outputdir, 'combined_data_xy_roi.png')
+            else:
+                filename = os_path.join(
+                    self._outputdir, 'reconstructed_data_xy_roi.png')
+        else:
+            filename = None
+        roi = select_roi_2d(
             tomosum, preselected_roi=preselected_roi,
             title_a='Reconstructed data summed over z',
             row_label='y', column_label='x',
-            interactive=self._interactive)
+            interactive=self._interactive, filename=filename)
         if roi is None:
             x_bounds = (0, tomo_recon_stacks[0].shape[2])
             y_bounds = (0, tomo_recon_stacks[0].shape[1])
@@ -2878,12 +2914,6 @@ class Tomo:
             y_bounds = (int(roi[2]), int(roi[3]))
         self._logger.debug(f'x_bounds = {x_bounds}')
         self._logger.debug(f'y_bounds = {y_bounds}')
-        # Plot results
-        if self._save_figs:
-            fig.savefig(
-                os_path.join(
-                    self._outputdir, 'reconstructed_data_xy_roi.png'))
-        plt.close()
 
         # Selecting z bounds (in xy-plane)
         # (only valid for a single image stack or when combining a stack)
@@ -2905,17 +2935,20 @@ class Tomo:
             tomosum = 0
             for i in range(num_tomo_stacks):
                 tomosum = tomosum + np.sum(tomo_recon_stacks[i], axis=(1,2))
-            fig, z_bounds = select_roi_1d(
+            if self._save_figs:
+                if combine_data:
+                    filename = os_path.join(
+                        self._outputdir, 'combined_data_z_roi.png')
+                else:
+                    filename = os_path.join(
+                        self._outputdir, 'reconstructed_data_z_roi.png')
+            else:
+                filename = None
+            z_bounds = select_roi_1d(
                 tomosum, preselected_roi=z_bounds,
                 xlabel='z', ylabel='Reconstructed data summed over x and y',
-                interactive=self._interactive)
+                interactive=self._interactive, filename=filename)
             self._logger.debug(f'z_bounds = {z_bounds}')
-            # Plot results
-            if self._save_figs:
-                fig.savefig(
-                    os_path.join(
-                        self._outputdir, 'reconstructed_data_z_roi.png'))
-            plt.close()
 
         return x_bounds, y_bounds, z_bounds
 
@@ -2957,6 +2990,9 @@ class TomoSimFieldProcessor(Processor):
         sample_size = config.sample_size
         if len(sample_size) == 1:
             sample_size = (sample_size[0], sample_size[0])
+        if sample_type == 'hollow_pyramid' and len(sample_size) != 3:
+            raise ValueError('Invalid combindation of sample_type '
+                             f'({sample_type}) and sample_size ({sample_size}')
         wall_thickness = config.wall_thickness
         mu = config.mu
         theta_step = config.theta_step
@@ -2993,15 +3029,22 @@ class TomoSimFieldProcessor(Processor):
 
         # Get the number of horizontal stacks bases on the diagonal
         # of the square and for now don't allow more than one
-        num_tomo_stack = 1 + int((sample_size[1]*np.sqrt(2)-pixel_size[1])
-                                 / (detector_size[1]*pixel_size[1]))
+        if (sample_size) == 3:
+            num_tomo_stack = 1 + int(
+                (max(sample_size[1:2])*np.sqrt(2)-pixel_size[1])
+                / (detector_size[1]*pixel_size[1]))
+        else:
+            num_tomo_stack = 1 + int((sample_size[1]*np.sqrt(2)-pixel_size[1])
+                                     / (detector_size[1]*pixel_size[1]))
         if num_tomo_stack > 1:
             raise ValueError('Sample is too wide for the detector')
 
         # Create the x-ray path length through a solid square
         # crosssection for a set of rotation angles.
-        path_lengths_solid = self._create_pathlength_solid_square(
-                sample_size[1], thetas, pixel_size[1], detector_size[1])
+        path_lengths_solid = None
+        if sample_type != 'hollow_pyramid':
+            path_lengths_solid = self._create_pathlength_solid_square(
+                    sample_size[1], thetas, pixel_size[1], detector_size[1])
 
         # Create the x-ray path length through a hollow square
         # crosssection for a set of rotation angles.
@@ -3027,7 +3070,12 @@ class TomoSimFieldProcessor(Processor):
         num_theta = len(thetas)
         vertical_shifts = []
         tomo_fields_stack = []
-        img_dim = (len(img_row_coords), path_lengths_solid.shape[1])
+        len_img_y = (detector_size[1]+1)//2
+        if len_img_y%2:
+            len_img_y = 2*len_img_y - 1
+        else:
+            len_img_y = 2*len_img_y
+        img_dim = (len(img_row_coords), len_img_y)
         intensities_solid = None
         intensities_hollow = None
         for n in range(num_tomo_stack):
@@ -3044,6 +3092,37 @@ class TomoSimFieldProcessor(Processor):
                     beam_intensity * np.exp(-mu*path_lengths_hollow)
                 for n in range(num_theta):
                     tomo_field[n,:,:] = intensities_hollow[n]
+            elif sample_type == 'hollow_pyramid':
+                outer_indices = \
+                    np.where(abs(img_row_coords) <= sample_size[0]/2)[0]
+                inner_indices = np.where(
+                    abs(img_row_coords) < sample_size[0]/2 - wall_thickness)[0]
+                wall_indices = list(set(outer_indices)-set(inner_indices))
+                ratio = abs(sample_size[1]-sample_size[2])/sample_size[0]
+                baselength = max(sample_size[1:2])
+                for i in wall_indices:
+                    path_lengths_solid = self._create_pathlength_solid_square(
+                        baselength - ratio*(
+                            img_row_coords[i] + 0.5*sample_size[0]),
+                        thetas, pixel_size[1], detector_size[1])
+                    intensities_solid = \
+                        beam_intensity * np.exp(-mu*path_lengths_solid)
+                    for n in range(num_theta):
+                        tomo_field[n,i] = intensities_solid[n]
+                for i in inner_indices:
+                    path_lengths_hollow = (
+                        self._create_pathlength_solid_square(
+                            baselength - ratio*(
+                                img_row_coords[i] + 0.5*sample_size[0]),
+                            thetas, pixel_size[1], detector_size[1])
+                        - self._create_pathlength_solid_square(
+                            baselength - 2*wall_thickness - ratio*(
+                                img_row_coords[i] + 0.5*sample_size[0]),
+                            thetas, pixel_size[1], detector_size[1]))
+                    intensities_hollow = \
+                        beam_intensity * np.exp(-mu*path_lengths_hollow)
+                    for n in range(num_theta):
+                        tomo_field[n,i] = intensities_hollow[n]
             else:
                 intensities_solid = \
                     beam_intensity * np.exp(-mu*path_lengths_solid)
@@ -3136,7 +3215,7 @@ class TomoSimFieldProcessor(Processor):
         """
         # Get the column coordinates
         img_y_coords = pixel_size * (0.5 * (1 - detector_size%2)
-            + np.asarray(range(int(0.5 * (detector_size+1)))))
+            + np.asarray(range((detector_size+1)//2)))
 
         # Get the path lenghts for position column coordinates
         lengths = np.zeros((len(thetas), len(img_y_coords)), dtype=np.float64)

CHAP/utils/__init__.py

@@ -2,3 +2,4 @@
 CHESS scan data, collecting interactive user input, and finding
 lattice properties of materials (among others).
 """
+from CHAP.utils.fit import FitProcessor