ChessAnalysisPipeline 0.0.15__py3-none-any.whl → 0.0.16__py3-none-any.whl

CHAP/runner.py

@@ -21,25 +21,35 @@ class RunConfig():
             'outputdir': '.',
             'interactive': False,
             'log_level': 'INFO',
-            'profile': False}
+            'profile': False,
+            'spawn': 0}
 
-    def __init__(self, config={}):
-        """RunConfig constructor
+    def __init__(self, config={}, comm=None):
+        """RunConfig constructor.
 
-        :param config: Pipeline configuration options
-        :type config: dict
+        :param config: Pipeline configuration options,
+            defaults to `{}`.
+        :type config: dict, optional
+        :param comm: MPI communicator, defaults to `None`.
+        :type comm: mpi4py.MPI.Comm, optional
         """
         # System modules
         from tempfile import NamedTemporaryFile
 
+        # Make sure os.makedirs is only called from the root node
+        if comm is None:
+            rank = 0
+        else:
+            rank = comm.Get_rank()
         for opt in self.opts:
             setattr(self, opt, config.get(opt, self.opts[opt]))
 
         # Check if root exists (create it if not) and is readable
-        if not os.path.isdir(self.root):
-            os.makedirs(self.root)
-        if not os.access(self.root, os.R_OK):
-            raise OSError('root directory is not accessible for reading '
+        if not rank:
+            if not os.path.isdir(self.root):
+                os.makedirs(self.root)
+            if not os.access(self.root, os.R_OK):
+                raise OSError('root directory is not accessible for reading '
                           f'({self.root})')
 
         # Check if inputdir exists and is readable
@@ -56,16 +66,21 @@ class RunConfig():
         if not os.path.isabs(self.outputdir):
             self.outputdir = os.path.realpath(
                 os.path.join(self.root, self.outputdir))
-        if not os.path.isdir(self.outputdir):
-            os.makedirs(self.outputdir)
-        try:
-            tmpfile = NamedTemporaryFile(dir=self.outputdir)
-        except:
-            raise OSError('output directory is not accessible for writing '
-                          f'({self.outputdir})')
+        if not rank:
+            if not os.path.isdir(self.outputdir):
+                os.makedirs(self.outputdir)
+            try:
+                tmpfile = NamedTemporaryFile(dir=self.outputdir)
+            except:
+                raise OSError('output directory is not accessible for writing '
+                              f'({self.outputdir})')
 
         self.log_level = self.log_level.upper()
 
+        # Make sure os.makedirs completes before continuing all nodes
+        if comm is not None:
+            comm.barrier()
+
 def parser():
     """Return an argument parser for the `CHAP` CLI. This parser has
     one argument: the input CHAP configuration file.
@@ -77,50 +92,93 @@ def parser():
     return parser
 
 def main():
-    """Main function"""
+    """Main function."""
+    # Third party modules
+    try:
+        from mpi4py import MPI
+        have_mpi = True
+        comm = MPI.COMM_WORLD
+    except:
+        have_mpi = False
+        comm = None
+
     args = parser().parse_args()
 
-    # read input config file
+    # Read the input config file
     configfile = args.config
     with open(configfile) as file:
         config = safe_load(file)
-    run_config = RunConfig(config.get('config', {}))
+
+    # Check if run was a worker spawned by another Processor
+    run_config = RunConfig(config.get('config', {}), comm)
+    if have_mpi and run_config.spawn:
+        sub_comm = MPI.Comm.Get_parent()
+        common_comm = sub_comm.Merge(True)
+        # Read worker specific input config file
+        if run_config.spawn > 0:
+            with open(f'{configfile}_{common_comm.Get_rank()}') as file:
+                config = safe_load(file)
+                run_config = RunConfig(config.get('config', {}), common_comm)
+        else:
+            with open(f'{configfile}_{sub_comm.Get_rank()}') as file:
+                config = safe_load(file)
+                run_config = RunConfig(config.get('config', {}), comm)
+    else:
+        common_comm = comm
+
+    # Get the pipeline configurations
     pipeline_config = config.get('pipeline', [])
 
-    # profiling setup
+    # Profiling setup
     if run_config.profile:
         from cProfile import runctx  # python profiler
         from pstats import Stats     # profiler statistics
-        cmd = 'runner(run_config, pipeline_config)'
+        cmd = 'runner(run_config, pipeline_config, common_comm)'
         runctx(cmd, globals(), locals(), 'profile.dat')
         info = Stats('profile.dat')
         info.sort_stats('cumulative')
         info.print_stats()
     else:
-        runner(run_config, pipeline_config)
+        runner(run_config, pipeline_config, common_comm)
+
+    # Disconnect the spawned worker
+    if have_mpi and run_config.spawn:
+        common_comm.barrier()
+        sub_comm.Disconnect()
 
-def runner(run_config, pipeline_config):
-    """Main runner funtion
+def runner(run_config, pipeline_config, comm=None):
+    """Main runner funtion.
 
-    :param run_config: CHAP run configuration
-    :type run_config: RunConfig
-    :param pipeline_config: CHAP Pipeline configuration
+    :param run_config: CHAP run configuration.
+    :type run_config: CHAP.runner.RunConfig
+    :param pipeline_config: CHAP Pipeline configuration.
     :type pipeline_config: dict
+    :param comm: MPI communicator, defaults to `None`.
+    :type comm: mpi4py.MPI.Comm, optional
+    :return: The pipeline's returned data field.
     """
+    # System modules
+    from time import time
+
     # logging setup
     logger, log_handler = setLogger(run_config.log_level)
     logger.info(f'Input pipeline configuration: {pipeline_config}\n')
 
-    # run pipeline
-    run(pipeline_config,
+    # Run the pipeline
+    t0 = time()
+    data = run(pipeline_config,
         run_config.inputdir, run_config.outputdir, run_config.interactive,
-        logger, run_config.log_level, log_handler)
+        logger, run_config.log_level, log_handler, comm)
+    logger.info(f'Executed "run" in {time()-t0:.3f} seconds')
+    return data
 
-def setLogger(log_level="INFO"):
-    """
-    Helper function to set CHAP logger
+def setLogger(log_level='INFO'):
+    """Helper function to set CHAP logger.
 
-    :param log_level: logger level, default INFO
+    :param log_level: Logger level, defaults to `"INFO"`.
+    :type log_level: str
+    :return: The CHAP logger and logging handler.
+    :rtype: logging.Logger, logging.StreamHandler
     """
     logger = logging.getLogger(__name__)
     log_level = getattr(logging, log_level.upper())
@@ -133,22 +191,46 @@ def setLogger(log_level="INFO"):
 
 def run(
         pipeline_config, inputdir=None, outputdir=None, interactive=False,
-        logger=None, log_level=None, log_handler=None):
-    """
-    Run given pipeline_config
+        logger=None, log_level=None, log_handler=None, comm=None):
+    """Run a given pipeline_config.
 
-    :param pipeline_config: CHAP pipeline config
+    :param pipeline_config: CHAP Pipeline configuration.
+    :type pipeline_config: dict
+    :param inputdir: Input directory, defaults to `None'`.
+    :type inputdir: str, optional
+    :param outputdir: Output directory, defaults to `None'`.
+    :type outputdir: str, optional
+    :param interactive: Allows for user interactions,
+        defaults to `False`.
+    :type interactive: bool, optional
+    :param logger: CHAP logger, defaults to `None`.
+    :type logger: logging.Logger, optional
+    :param log_level: Logger level, defaults to `None`.
+    :type log_level: str, optional
+    :param log_handler: logging handler, defaults to `None`.
+    :type log_handler: logging.StreamHandler, optional
+    :param comm: MPI communicator, defaults to `None`.
+    :type comm: mpi4py.MPI.Comm, optional
+    :return: The `data` field of the first item in the returned
+        list of pipeline items.
     """
     # System modules
     from tempfile import NamedTemporaryFile
 
+    # Make sure os.makedirs is only called from the root node
+    if comm is None:
+        rank = 0
+    else:
+        rank = comm.Get_rank()
+
     objects = []
     kwds = []
     for item in pipeline_config:
-        # load individual object with given name from its module
+        # Load individual object with given name from its module
         kwargs = {'inputdir': inputdir,
                   'outputdir': outputdir,
-                  'interactive': interactive}
+                  'interactive': interactive,
+                  'comm': comm}
         if isinstance(item, dict):
             name = list(item.keys())[0]
             item_args = item[name]
@@ -156,39 +238,41 @@ def run(
             # "outputdir" and "interactive" with the item's arguments
             # joining "inputdir" and "outputdir" and giving precedence
             # for "interactive" in the latter
-            if 'inputdir' in item_args:
-                newinputdir = os.path.normpath(os.path.join(
-                    kwargs['inputdir'], item_args.pop('inputdir')))
-                if not os.path.isdir(newinputdir):
-                    raise OSError(
-                        f'input directory does not exist ({newinputdir})')
-                if not os.access(newinputdir, os.R_OK):
-                    raise OSError('input directory is not accessible for '
-                                  f'reading ({newinputdir})')
-                kwargs['inputdir'] = newinputdir
-            if 'outputdir' in item_args:
-                newoutputdir = os.path.normpath(os.path.join(
-                    kwargs['outputdir'], item_args.pop('outputdir')))
-                if not os.path.isdir(newoutputdir):
-                    os.makedirs(newoutputdir)
-                try:
-                    tmpfile = NamedTemporaryFile(dir=newoutputdir)
-                except:
-                    raise OSError('output directory is not accessible for '
-                                  f'writing ({newoutputdir})')
-                kwargs['outputdir'] = newoutputdir
-            kwargs = {**kwargs, **item_args}
+            if item_args is not None:
+                if 'inputdir' in item_args:
+                    newinputdir = os.path.normpath(os.path.join(
+                        kwargs['inputdir'], item_args.pop('inputdir')))
+                    if not os.path.isdir(newinputdir):
+                        raise OSError(
+                            f'input directory does not exist ({newinputdir})')
+                    if not os.access(newinputdir, os.R_OK):
+                        raise OSError('input directory is not accessible for '
+                                      f'reading ({newinputdir})')
+                    kwargs['inputdir'] = newinputdir
+                if 'outputdir' in item_args:
+                    newoutputdir = os.path.normpath(os.path.join(
+                        kwargs['outputdir'], item_args.pop('outputdir')))
+                    if not rank:
+                        if not os.path.isdir(newoutputdir):
+                            os.makedirs(newoutputdir)
+                        try:
+                            tmpfile = NamedTemporaryFile(dir=newoutputdir)
+                        except:
+                            raise OSError('output directory is not accessible '
+                                          f'for writing ({newoutputdir})')
+                    kwargs['outputdir'] = newoutputdir
+                kwargs = {**kwargs, **item_args}
         else:
             name = item
         if "users" in name:
-            # load users module. This is required in CHAPaaS which can
+            # Load users module. This is required in CHAPaaS which can
             # have common area for users module. Otherwise, we will be
             # required to have invidual user's PYTHONPATHs to load user
             # processors.
             try:
                 import users
             except ImportError:
-                if logger:
+                if logger is not None:
                     logger.error(f'Unable to load {name}')
                 continue
             clsName = name.split('.')[-1]
@@ -199,23 +283,28 @@ def run(
             modName, clsName = name.split('.')
             module = __import__(f'CHAP.{modName}', fromlist=[clsName])
             obj = getattr(module, clsName)()
-        if log_level:
+        if log_level is not None:
             obj.logger.setLevel(log_level)
-        if log_handler:
+        if log_handler is not None:
             obj.logger.addHandler(log_handler)
-        if logger:
+        if logger is not None:
             logger.info(f'Loaded {obj}')
         objects.append(obj)
         kwds.append(kwargs)
     pipeline = Pipeline(objects, kwds)
-    if log_level:
+    if log_level is not None:
         pipeline.logger.setLevel(log_level)
-    if log_handler:
+    if log_handler is not None:
         pipeline.logger.addHandler(log_handler)
-    if logger:
+    if logger is not None:
         logger.info(f'Loaded {pipeline} with {len(objects)} items\n')
         logger.info(f'Calling "execute" on {pipeline}')
-    pipeline.execute()
+
+    # Make sure os.makedirs completes before continuing all nodes
+    if comm is not None:
+        comm.barrier()
+
+    return pipeline.execute()[0]['data']
 
 
 if __name__ == '__main__':

CHAP/tomo/models.py

@@ -36,7 +36,7 @@ class Detector(BaseModel):
     columns: conint(gt=0)
     pixel_size: conlist(
         item_type=confloat(gt=0, allow_inf_nan=False),
-        min_items=1, max_items=2)
+        min_length=1, max_length=2)
     lens_magnification: confloat(gt=0, allow_inf_nan=False) = 1.0
 
 
@@ -53,8 +53,8 @@ class TomoReduceConfig(BaseModel):
     :type delta_theta: float, optional
     """
     img_row_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
-    delta_theta: Optional[confloat(gt=0, allow_inf_nan=False)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
+    delta_theta: Optional[confloat(gt=0, allow_inf_nan=False)] = None
 
 
 class TomoFindCenterConfig(BaseModel):
@@ -84,19 +84,19 @@ class TomoFindCenterConfig(BaseModel):
         reconstruction in pixels, defaults to no filtering performed.
     :type ring_width: float, optional
     """
-    center_stack_index: Optional[conint(ge=0)]
+    center_stack_index: Optional[conint(ge=0)] = None
     center_rows: Optional[conlist(
-        item_type=conint(ge=0), min_items=2, max_items=2)]
+        item_type=conint(ge=0), min_length=2, max_length=2)] = None
     center_offsets: Optional[conlist(
         item_type=confloat(allow_inf_nan=False),
-        min_items=2, max_items=2)]
-    center_offset_min: Optional[confloat(allow_inf_nan=False)]
-    center_offset_max: Optional[confloat(allow_inf_nan=False)]
+        min_length=2, max_length=2)] = None
+    center_offset_min: Optional[confloat(allow_inf_nan=False)] = None
+    center_offset_max: Optional[confloat(allow_inf_nan=False)] = None
     center_search_range: Optional[conlist(
         item_type=confloat(allow_inf_nan=False),
-        min_items=1, max_items=3)]
-    gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
-    ring_width: Optional[confloat(ge=0, allow_inf_nan=False)]
+        min_length=1, max_length=3)] = None
+    gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)] = None
+    ring_width: Optional[confloat(ge=0, allow_inf_nan=False)] = None
 
 
 class TomoReconstructConfig(BaseModel):
@@ -126,15 +126,15 @@ class TomoReconstructConfig(BaseModel):
     :type ring_width: float, optional
     """
     x_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
     y_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
     z_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
     secondary_iters: conint(ge=0) = 0
-    gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
-    remove_stripe_sigma: Optional[confloat(ge=0, allow_inf_nan=False)]
-    ring_width: Optional[confloat(ge=0, allow_inf_nan=False)]
+    gaussian_sigma: Optional[confloat(ge=0, allow_inf_nan=False)] = None
+    remove_stripe_sigma: Optional[confloat(ge=0, allow_inf_nan=False)] = None
+    ring_width: Optional[confloat(ge=0, allow_inf_nan=False)] = None
 
 
 class TomoCombineConfig(BaseModel):
@@ -150,11 +150,11 @@ class TomoCombineConfig(BaseModel):
     :type z_bounds: list[int], optional
     """
     x_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
     y_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
     z_bounds: Optional[
-        conlist(item_type=conint(ge=-1), min_items=2, max_items=2)]
+        conlist(item_type=conint(ge=-1), min_length=2, max_length=2)] = None
 
 
 class TomoSimConfig(BaseModel):
@@ -186,19 +186,19 @@ class TomoSimConfig(BaseModel):
     :type beam_intensity: float, optional
     :ivar background_intensity: Background intensity in counts,
         defaults to 20.
-    :type background_intensity:: float, optional
+    :type background_intensity: float, optional
     :ivar slit_size: Vertical beam height in mm, defaults to 1.0.
-    :type slit_size:: float, optional
+    :type slit_size: float, optional
     """
     station: Literal['id1a3', 'id3a', 'id3b']
-    detector: Detector.construct()
+    detector: Detector.model_construct()
     sample_type: Literal[
         'square_rod', 'square_pipe', 'hollow_cube', 'hollow_brick',
         'hollow_pyramid']
     sample_size: conlist(
         item_type=confloat(gt=0, allow_inf_nan=False),
-        min_items=1, max_items=3)
-    wall_thickness: Optional[confloat(ge=0, allow_inf_nan=False)]
+        min_length=1, max_length=3)
+    wall_thickness: Optional[confloat(ge=0, allow_inf_nan=False)] = None
     mu: Optional[confloat(gt=0, allow_inf_nan=False)] = 0.05
     theta_step: confloat(gt=0, allow_inf_nan=False)
     beam_intensity: Optional[confloat(gt=0, allow_inf_nan=False)] = 1.e9