ChessAnalysisPipeline 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

CHAP/foxden/__init__.py

@@ -0,0 +1,6 @@
+"""This subpackage contains pieces for communication with FOXDEN services.
+"""
+
+from CHAP.foxden.processor import (
+    FoxdenProvenanceProcessor,
+)

CHAP/foxden/processor.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : processor.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Processor module for FOXDEN services
+"""
+
+# system modules
+from time import time
+
+# local modules
+from CHAP import Processor
+from CHAP.foxden.writer import FoxdenWriter
+
+
+class FoxdenProvenanceProcessor(Processor):
+    """A Processor to communicate with FOXDEN provenance server."""
+#     def __init__(self):
+#         self.writer = FoxdenWriter()
+
+    def process(self, data, url, dryRun=False, verbose=False):
+        """process data API"""
+
+        t0 = time()
+        self.logger.info(f'Executing "process" with url {url} data {data} dryrun {dryRun}')
+        writer = FoxdenWriter()
+
+#         data = self.writer.write(data, url, dryRun)
+        data = writer.write(data, url, dryRun=dryRun)
+
+        self.logger.info(f'Finished "process" in {time()-t0:.3f} seconds\n')
+
+        return data
+
+
+if __name__ == '__main__':
+    # local modules
+    from CHAP.processor import main
+
+    main()

CHAP/foxden/writer.py

@@ -0,0 +1,65 @@
+"""FOXDE command line writer."""
+
+# system modules
+import os
+
+# Local modules
+from CHAP.writer import main
+
+class FoxdenWriter():
+    """FOXDEN writer writes data to specific FOXDEN service
+    """
+
+    def write(self, data, url, method="POST", headers={}, timeout=10, dryRun=False):
+        """Write the input data as text to a file.
+
+        :param data: input data
+        :type data: list[PipelineData]
+        :param url: url of service
+        :type url: str
+        :param method: HTTP method to use, POST for creation and PUT for update
+        :type method: str
+        :param headers: HTTP headers to use
+        :type headers: dictionary
+        :param timeout: timeout of HTTP request
+        :type timeout: str
+        :param dryRun: dryRun option to verify HTTP workflow
+        :type dryRun: boolean
+        :return: contents of the input data
+        :rtype: object
+        """
+        import requests
+        if 'Content-Type' not in headers:
+            headers['Content-type'] = 'application/json'
+        if 'Accept' not in headers:
+            headers['Accept'] = 'application/json'
+        if dryRun:
+            print("### HTTP writer call", url, headers, data)
+            return []
+        token = ""
+        fname = os.getenv("CHESS_WRITE_TOKEN")
+        if not fname:
+            msg = f'CHESS_WRITE_TOKEN env variable is not set'
+            raise Exception(msg)
+        with open(fname, 'r') as istream:
+            token = istream.read()
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        else:
+            msg = f'No valid write token found in CHESS_WRITE_TOKEN env variable'
+            raise Exception(msg)
+
+        # make actual HTTP request to FOXDEN service
+        if method.lower() == 'post':
+            resp = requests.post(url, headers=headers, timeout=timeout, data=data)
+        elif method.lower() == 'put':
+            resp = requests.put(url, headers=headers, timeout=timeout, data=data)
+        else:
+            msg = f"unsupporteed method {method}"
+            raise Exception(msg)
+        data = resp.content
+        return data
+
+
+if __name__ == '__main__':
+    main()

CHAP/pipeline.py

@@ -113,7 +113,10 @@ class PipelineItem():
 
         mod_name, cls_name = schema.rsplit('.', 1)
         module = __import__(f'CHAP.{mod_name}', fromlist=cls_name)
-        model_config = getattr(module, cls_name)(**matching_config, **kwargs)
+        model_kwargs = {k: v for k, v in kwargs.items() \
+                        if k not in matching_config}
+        model_config = getattr(module, cls_name)(**matching_config,
+                                                 **model_kwargs)
 
         self.logger.debug(
             f'Got {schema} configuration in {time()-t0:.3f} seconds')
@@ -184,6 +187,8 @@ class MultiplePipelineItem(PipelineItem):
         :type items: list
         :rtype: list[PipelineData]
         """
+        # System modules
+        from tempfile import NamedTemporaryFile
 
         t0 = time()
         self.logger.info(f'Executing {len(items)} PipelineItems')
@@ -203,10 +208,37 @@ class MultiplePipelineItem(PipelineItem):
             mod_name, cls_name = item_name.rsplit('.', 1)
             module = __import__(f'CHAP.{mod_name}', fromlist=cls_name)
             item = getattr(module, cls_name)()
+            # Combine the command line arguments "inputdir",
+            # "outputdir" and "interactive" with the item's arguments
+            # joining "inputdir" and "outputdir" and giving precedence
+            # for "interactive" in the latter
+            args = {**kwargs}
+            if 'inputdir' in item_args:
+                inputdir = os.path.normpath(os.path.join(
+                    args['inputdir'], item_args.pop('inputdir')))
+                if not os.path.isdir(inputdir):
+                    raise OSError(
+                        f'input directory does not exist ({inputdir})')
+                if not os.access(inputdir, os.R_OK):
+                    raise OSError('input directory is not accessible for '
+                                  f'reading ({inputdir})')
+                args['inputdir'] = inputdir
+            if 'outputdir' in item_args:
+                outputdir = os.path.normpath(os.path.join(
+                    args['outputdir'], item_args.pop('outputdir')))
+                if not os.path.isdir(outputdir):
+                    os.makedirs(outputdir)
+                try:
+                    tmpfile = NamedTemporaryFile(dir=outputdir)
+                except:
+                    raise OSError('output directory is not accessible for '
+                                  f'writing ({outputdir})')
+                args['outputdir'] = outputdir
+            args = {**args, **item_args}
             if hasattr(item, 'write'):
-                item.execute(**item_args, **kwargs)[0]
+                item.execute(**args)[0]
             else:
-                data.append(item.execute(**item_args, **kwargs)[0])
+                data.append(item.execute(**args)[0])
 
         self.logger.info(
             f'Finished executing {len(items)} PipelineItems in {time()-t0:.0f}'

CHAP/runner.py

@@ -17,11 +17,11 @@ from CHAP.pipeline import Pipeline
 class RunConfig():
     """Representation of Pipeline run configuration."""
     opts = {'root': os.getcwd(),
-            'profile': False,
+            'inputdir': '.',
+            'outputdir': '.',
             'interactive': False,
             'log_level': 'INFO',
-            'inputdir': '.',
-            'outputdir': '.'}
+            'profile': False}
 
     def __init__(self, config={}):
         """RunConfig constructor
@@ -37,12 +37,12 @@ class RunConfig():
 
         # Check if root exists (create it if not) and is readable
         if not os.path.isdir(self.root):
-            os.mkdir(self.root)
+            os.makedirs(self.root)
         if not os.access(self.root, os.R_OK):
             raise OSError('root directory is not accessible for reading '
                           f'({self.root})')
 
-        # Check if input exists and is readable
+        # Check if inputdir exists and is readable
         if not os.path.isabs(self.inputdir):
             self.inputdir = os.path.realpath(
                 os.path.join(self.root, self.inputdir))
@@ -52,12 +52,12 @@ class RunConfig():
             raise OSError('input directory is not accessible for reading '
                           f'({self.inputdir})')
 
-        # Check if output exists (create it if not) and is writable
+        # Check if outputdir exists (create it if not) and is writable
         if not os.path.isabs(self.outputdir):
             self.outputdir = os.path.realpath(
                 os.path.join(self.root, self.outputdir))
         if not os.path.isdir(self.outputdir):
-            os.mkdir(self.outputdir)
+            os.makedirs(self.outputdir)
         try:
             tmpfile = NamedTemporaryFile(dir=self.outputdir)
         except:
@@ -127,30 +127,57 @@ def setLogger(log_level="INFO"):
     logger.setLevel(log_level)
     log_handler = logging.StreamHandler()
     log_handler.setFormatter(logging.Formatter(
-        '{name:20}: {message}', style='{'))
+        '{name:20}: {levelname}: {message}', style='{'))
     logger.addHandler(log_handler)
     return logger, log_handler
 
-def run(pipeline_config,
-        inputdir=None, outputdir=None, interactive=False,
+def run(
+        pipeline_config, inputdir=None, outputdir=None, interactive=False,
         logger=None, log_level=None, log_handler=None):
     """
     Run given pipeline_config
 
     :param pipeline_config: CHAP pipeline config
     """
+    # System modules
+    from tempfile import NamedTemporaryFile
+
     objects = []
     kwds = []
     for item in pipeline_config:
         # load individual object with given name from its module
-        kwargs = {'interactive': interactive,
-                  'inputdir': inputdir,
-                  'outputdir': outputdir}
+        kwargs = {'inputdir': inputdir,
+                  'outputdir': outputdir,
+                  'interactive': interactive}
         if isinstance(item, dict):
             name = list(item.keys())[0]
-            # Combine the "interactive" command line argument with the object's keywords
-            # giving precedence of "interactive" in the latter
-            kwargs = {**kwargs, **item[name]}
+            item_args = item[name]
+            # Combine the function's input arguments "inputdir",
+            # "outputdir" and "interactive" with the item's arguments
+            # joining "inputdir" and "outputdir" and giving precedence
+            # for "interactive" in the latter
+            if 'inputdir' in item_args:
+                newinputdir = os.path.normpath(os.path.join(
+                    kwargs['inputdir'], item_args.pop('inputdir')))
+                if not os.path.isdir(newinputdir):
+                    raise OSError(
+                        f'input directory does not exist ({newinputdir})')
+                if not os.access(newinputdir, os.R_OK):
+                    raise OSError('input directory is not accessible for '
+                                  f'reading ({newinputdir})')
+                kwargs['inputdir'] = newinputdir
+            if 'outputdir' in item_args:
+                newoutputdir = os.path.normpath(os.path.join(
+                    kwargs['outputdir'], item_args.pop('outputdir')))
+                if not os.path.isdir(newoutputdir):
+                    os.makedirs(newoutputdir)
+                try:
+                    tmpfile = NamedTemporaryFile(dir=newoutputdir)
+                except:
+                    raise OSError('output directory is not accessible for '
+                                  f'writing ({newoutputdir})')
+                kwargs['outputdir'] = newoutputdir
+            kwargs = {**kwargs, **item_args}
         else:
             name = item
         if "users" in name:

CHAP/tomo/models.py

@@ -7,7 +7,6 @@ from typing import (
 )
 from pydantic import (
     BaseModel,
-    StrictBool,
     conint,
     conlist,
     confloat,
@@ -93,6 +92,9 @@ class TomoFindCenterConfig(BaseModel):
         min_items=2, max_items=2)]
     center_offset_min: Optional[confloat(allow_inf_nan=False)]
     center_offset_max: Optional[confloat(allow_inf_nan=False)]
+    center_search_range: Optional[conlist(
+        item_type=confloat(allow_inf_nan=False),
+        min_items=1, max_items=3)]
     gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
     ring_width: Optional[confloat(ge=0, allow_inf_nan=False)]
 
@@ -111,6 +113,10 @@ class TomoReconstructConfig(BaseModel):
     :ivar secondary_iters: Number of secondary iterations in the tomopy
         image reconstruction algorithm, defaults to 0.
     :type secondary_iters: int, optional
+    :ivar gaussian_sigma: Standard deviation for the Gaussian filter
+        applied to image reconstruction visualizations, defaults to no
+        filtering performed.
+    :type gaussian_sigma: float, optional
     :ivar remove_stripe_sigma: Damping parameter in Fourier space in
         tomopy's horizontal stripe removal tool, defaults to no
         correction performed.
@@ -126,6 +132,7 @@ class TomoReconstructConfig(BaseModel):
     z_bounds: Optional[
         conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
     secondary_iters: conint(ge=0) = 0
+    gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
     remove_stripe_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
     ring_width: Optional[confloat(ge=0, allow_inf_nan=False)]
 
@@ -160,9 +167,11 @@ class TomoSimConfig(BaseModel):
     :type detector: Detector
     :ivar sample_type: Sample type for the tomography simulator.
     :type sample_type: Literal['square_rod', 'square_pipe',
-        'hollow_cube', 'hollow_brick']
+        'hollow_cube', 'hollow_brick', 'hollow_pyramid']
     :ivar sample_size: Size of each sample dimension in mm (internally
-        converted to an integer number of pixels).
+        converted to an integer number of pixels). Enter three values
+        for sample_type == `'hollow_pyramid'`, the height and the side
+        at the respective bottom and the top of the pyramid.
     :type sample_size: list[float]
     :ivar wall_thickness: Wall thickness for pipe, cube, and brick in
         mm (internally converted to an integer number of pixels).
@@ -184,10 +193,11 @@ class TomoSimConfig(BaseModel):
     station: Literal['id1a3', 'id3a', 'id3b']
     detector: Detector.construct()
     sample_type: Literal[
-        'square_rod', 'square_pipe', 'hollow_cube', 'hollow_brick']
+        'square_rod', 'square_pipe', 'hollow_cube', 'hollow_brick',
+        'hollow_pyramid']
     sample_size: conlist(
         item_type=confloat(gt=0, allow_inf_nan=False),
-        min_items=1, max_items=2)
+        min_items=1, max_items=3)
     wall_thickness: Optional[confloat(ge=0, allow_inf_nan=False)]
     mu: Optional[confloat(gt=0, allow_inf_nan=False)] = 0.05
     theta_step: confloat(gt=0, allow_inf_nan=False)