ChessAnalysisPipeline 0.0.17.dev3__py3-none-any.whl

CHAP/reader.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+"""
+File       : reader.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: generic Reader module
+
+Define a generic `Reader` object.
+"""
+
+# System modules
+import argparse
+import logging
+import os
+from sys import modules
+
+# Third party modules
+from pydantic import (
+    PrivateAttr,
+    constr,
+    model_validator,
+)
+
+# Local modules
+from CHAP.pipeline import PipelineItem
+
+
+def validate_reader_model(reader):
+    reader._mapping_filename = reader.filename
+    filename = os.path.normpath(os.path.realpath(
+        os.path.join(reader.inputdir, reader.filename)))
+    if (not os.path.isfile(filename)
+            and not os.path.dirname(reader.filename)):
+        reader.logger.warning(
+            f'Unable to find {reader.filename} in '
+            f'{reader.inputdir}, looking in '
+            f'{reader.outputdir}')
+        filename = os.path.normpath(os.path.realpath(
+            os.path.join(reader.outputdir, reader.filename)))
+    # Note that reader.filename has str type instead of FilePath
+    # since its existence is not yet gueranteed (it can be writen
+    # over the course of the pipeline's execution). So postpone
+    # validation until the entire pipeline gets validated.
+    if not os.path.isfile(filename):
+        reader.logger.warning(
+            f'Unable to find {reader.filename} during validation')
+    reader.filename = filename
+    return reader
+
+
+class Reader(PipelineItem):
+    """Generic file reader.
+
+    The job of any `Reader` in a `Pipeline` is to provide data stored
+    in a file to the next `PipelineItem`. Note that a `Reader` used on
+    its own disrupts the flow of data in a `Pipeline` -- it does not
+    receive or pass along any data returned by the previous
+    `PipelineItem`.
+
+    :ivar filename: Name of file to read from.
+    :type filename: str
+    """
+    filename: constr(strip_whitespace=True, min_length=1)
+
+    _mapping_filename: PrivateAttr(default=None)
+
+    _validate_filename = model_validator(mode="after")(
+        validate_reader_model)
+
+    def read(self):
+        """Read and return the contents of `filename` as text.
+
+        :return: The file content.
+        :rtype: str
+        """
+        if not self.filename:
+            self.logger.warning(
+                'No file name is given, skipping read operation')
+            return None
+        try:
+            with open(self.filename) as f:
+                data = f.read()
+        except Exception:
+            return None
+        return data
+
+
+class OptionParser():
+    """User based option parser."""
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument(
+            '--filename', action='store',
+            dest='filename', default='', help='Input file')
+        self.parser.add_argument(
+            '--reader', action='store',
+            dest='reader', default='Reader', help='Reader class name')
+        self.parser.add_argument(
+            '--log-level', choices=logging._nameToLevel.keys(),
+            dest='log_level', default='INFO', help='logging level')
+
+
+def main(opt_parser=OptionParser):
+    """Main function."""
+    optmgr = opt_parser()
+    opts = optmgr.parser.parse_args()
+    cls_name = opts.reader
+    try:
+        reader_cls = getattr(modules[__name__], cls_name)
+    except AttributeError:
+        print(f'Unsupported reader {cls_name}')
+        raise
+
+    reader = reader_cls()
+    reader.logger.setLevel(getattr(logging, opts.log_level))
+    log_handler = logging.StreamHandler()
+    log_handler.setFormatter(logging.Formatter(
+        '{name:20}: {message}', style='{'))
+    reader.logger.addHandler(log_handler)
+    data = reader.read(filename=opts.filename)
+    print(f'Reader {reader} reads from {opts.filename}, data {data}')
+
+
+if __name__ == '__main__':
+    main()

CHAP/runner.py

@@ -0,0 +1,277 @@
+"""
+File       : runner.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description:
+"""
+
+def parser():
+    """Return an argument parser for the `CHAP` CLI. This parser has
+    one argument: the input CHAP configuration file.
+    """
+    # System modules
+    from argparse import ArgumentParser
+    pparser = ArgumentParser(prog='PROG')
+
+    pparser.add_argument(
+        'config', action='store', default='', help='Input configuration file')
+    pparser.add_argument(
+        '-p', '--pipeline', nargs='*', help='Pipeline name(s)')
+    return pparser
+
+def main():
+    """Main function."""
+    # System modules
+    from yaml import safe_load
+
+    # Local modules
+    from CHAP.models import RunConfig
+
+    try:
+        # Third party modules
+        # pylint: disable=c-extension-no-member
+        from mpi4py import MPI
+
+        have_mpi = True
+        comm = MPI.COMM_WORLD
+    except ImportError:
+        have_mpi = False
+        comm = None
+
+    args = parser().parse_args()
+
+    # Read the input config file
+    configfile = args.config
+    with open(configfile) as file:
+        config = safe_load(file)
+    #RV Add to input_files in provenance data writer
+
+    # Check if executed as a worker spawned by another Processor
+    run_config = RunConfig(**config.pop('config'), comm=comm)
+    if have_mpi and run_config.spawn:
+        # pylint: disable=c-extension-no-member
+        sub_comm = MPI.Comm.Get_parent()
+        common_comm = sub_comm.Merge(True)
+        # Read worker specific input config file
+        if run_config.spawn > 0:
+            with open(f'{configfile}_{common_comm.Get_rank()}') as file:
+                config = safe_load(file)
+                run_config = RunConfig(
+                    **config.pop('config'), comm=common_comm)
+        else:
+            with open(f'{configfile}_{sub_comm.Get_rank()}') as file:
+                config = safe_load(file)
+                run_config = RunConfig(**config.pop('config'), comm=comm)
+    else:
+        common_comm = comm
+
+    # Get the pipeline configurations
+    sub_pipelines = args.pipeline
+    pipeline_config = []
+    if sub_pipelines is None:
+        for sub_pipeline in config.values():
+            pipeline_config += sub_pipeline
+    else:
+        for sub_pipeline in sub_pipelines:
+            if sub_pipeline in config:
+                pipeline_config += config.get(sub_pipeline)
+            else:
+                raise ValueError(
+                    f'Invalid pipeline option: \'{sub_pipeline}\' missing in '
+                    f'the pipeline configuration ({list(config.keys())})')
+
+    # Run the pipeline with or without profiling
+    if run_config.profile:
+        # System modules
+        from cProfile import runctx  # python profiler
+        from pstats import Stats     # profiler statistics
+
+        cmd = 'runner(run_config, pipeline_config, common_comm)'
+        runctx(cmd, globals(), locals(), 'profile.dat')
+        info = Stats('profile.dat')
+        info.sort_stats('cumulative')
+        info.print_stats()
+    else:
+        runner(run_config, pipeline_config, common_comm)
+
+    # Disconnect the spawned worker
+    if have_mpi and run_config.spawn:
+        common_comm.barrier()
+        sub_comm.Disconnect()
+
+def runner(run_config, pipeline_config, comm=None):
+    """Main runner funtion.
+
+    :param run_config: CHAP run configuration.
+    :type run_config: CHAP.runner.RunConfig
+    :param pipeline_config: CHAP Pipeline configuration.
+    :type pipeline_config: dict
+    :param comm: MPI communicator.
+    :type comm: mpi4py.MPI.Comm, optional
+    :return: The pipeline's returned data field.
+    """
+    # System modules
+    from time import time
+
+    # Logging setup
+    logger, log_handler = set_logger(run_config.log_level)
+    logger.info(f'Input pipeline configuration: {pipeline_config}\n')
+
+    # Run the pipeline
+    t0 = time()
+    data = run(run_config, pipeline_config, logger, log_handler, comm)
+    logger.info(f'Executed "run" in {time()-t0:.3f} seconds')
+
+    return data
+
+def set_logger(log_level='INFO'):
+    """Helper function to set CHAP logger.
+
+    :param log_level: Logger level, defaults to `"INFO"`.
+    :type log_level: str
+    :return: The CHAP logger and logging handler.
+    :rtype: logging.Logger, logging.StreamHandler
+    """
+    # System modules
+    import logging
+
+    logger = logging.getLogger(__name__)
+    log_level = getattr(logging, log_level.upper())
+    logger.setLevel(log_level)
+    log_handler = logging.StreamHandler()
+    log_handler.setFormatter(logging.Formatter(
+        '{asctime}: {name:20}: {levelname}: {message}',
+        datefmt='%Y-%m-%d %H:%M:%S', style='{'))
+    logger.addHandler(log_handler)
+    return logger, log_handler
+
+def run(
+        run_config, pipeline_config, logger=None, log_handler=None, comm=None):
+    """Run a given pipeline_config.
+
+    :param run_config: CHAP run configuration.
+    :type run_config: CHAP.runner.RunConfig
+    :param pipeline_config: CHAP Pipeline configuration.
+    :type pipeline_config: dict
+    :param logger: CHAP logger.
+    :type logger: logging.Logger, optional
+    :param log_handler: Logging handler.
+    :type log_handler: logging.StreamHandler, optional
+    :param comm: MPI communicator.
+    :type comm: mpi4py.MPI.Comm, optional
+    :return: The `data` field of the first item in the returned
+        list of pipeline items.
+    """
+    # System modules
+    from logging import getLogger
+    import os
+    from tempfile import NamedTemporaryFile
+
+    # Local modules
+    from CHAP.pipeline import Pipeline
+
+    # Make sure os.makedirs is only called from the root node
+    if comm is None:
+        rank = 0
+    else:
+        rank = comm.Get_rank()
+
+    pipeline_args = []
+    pipeline_mmcs = []
+    for item in pipeline_config:
+
+        # Load individual object with given name from its module
+        config = run_config.model_dump()
+        if isinstance(item, dict):
+            name = list(item.keys())[0]
+            item_args = item.get(name)
+            # Picking "inputdir" and "outputdir" from the item or from
+            # the default run configuration, giving precedence to the
+            # former
+            if 'inputdir' in item_args:
+                inputdir = item_args.pop('inputdir')
+                if not os.path.isabs(inputdir):
+                    inputdir = os.path.normpath(os.path.realpath(
+                        os.path.join(run_config.inputdir, inputdir)))
+                if not os.path.isdir(inputdir):
+                    raise OSError(
+                        f'input directory does not exist ({inputdir})')
+                if not os.access(inputdir, os.R_OK):
+                    raise OSError('input directory is not accessible for '
+                                  f'reading ({inputdir})')
+                config['inputdir'] = inputdir
+            if 'outputdir' in item_args:
+                outputdir = item_args.pop('outputdir')
+                if not os.path.isabs(outputdir):
+                    outputdir = os.path.normpath(os.path.realpath(
+                        os.path.join(run_config.outputdir, outputdir)))
+                if not rank:
+                    if not os.path.isdir(outputdir):
+                        os.makedirs(outputdir)
+                    try:
+                        NamedTemporaryFile(dir=outputdir)
+                    except Exception as exc:
+                        raise OSError(
+                            'output directory is not accessible for '
+                            f'writing ({outputdir})') from exc
+                config['outputdir'] = outputdir
+        else:
+            name = item
+
+        # Initialize the object's identifiers
+        if 'users' in name:
+            # Load users module. This is required in CHAPaaS which can
+            # have common area for users module. Otherwise, we will be
+            # required to have invidual user's PYTHONPATHs to load user
+            # processors.
+            try:
+                # Third party modules
+                # pylint: disable=unused-import
+                import users
+            except ImportError:
+                if logger is not None:
+                    logger.error(f'Unable to load {name}')
+                continue
+            cls_name = name.split('.')[-1]
+            mod_name = '.'.join(name.split('.')[:-1])
+            module = __import__(mod_name, fromlist=[cls_name])
+        else:
+            mod_name, cls_name = name.split('.')
+            module = __import__(f'CHAP.{mod_name}', fromlist=[cls_name])
+
+        pipeline_mmcs.append(getattr(module, cls_name))
+
+        # Initialize the object's runtime arguments
+        item_args['comm'] = comm  #FIX make comm a field in RunConfig?
+        if 'name' not in item_args:
+            item_args['name'] = cls_name
+        item_args.update(config)
+        item_logger = getLogger(name)
+        if log_handler is not None:
+            item_logger.addHandler(log_handler)
+        item_args['logger'] = item_logger
+        if logger is not None:
+            logger.info(
+                f'Initialized input fields for an instance of {cls_name}')
+        pipeline_args.append(item_args)
+    pipeline = Pipeline(mmcs=pipeline_mmcs, args=pipeline_args)
+    pipeline.logger.setLevel(run_config.log_level)
+    if log_handler is not None:
+        pipeline.logger.addHandler(log_handler)
+    if logger is not None:
+        logger.info(f'Loaded {pipeline} with {len(pipeline_mmcs)} items\n')
+
+    # Make sure os.makedirs completes before continuing all nodes
+    if comm is not None:
+        comm.barrier()
+
+    # Execute the pipeline
+    if logger is not None:
+        logger.info(f'Calling "execute" on {pipeline}')
+    result = pipeline.execute()
+    if result:
+        return result[0]['data']
+    return result
+
+
+if __name__ == '__main__':
+    main()

CHAP/saxswaxs/__init__.py

@@ -0,0 +1,7 @@
+"""This subpackage contains `PipelineItems` unique to SAXSWAXS data
+processing workflows.
+"""
+
+# from CHAP.saxswaxs.processor import
+# from CHAP.saxswaxs.reader import
+# from CHAP.saxswaxs.writer import

CHAP/saxswaxs/processor.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""Processors used only by SAXSWAXS experiments."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.processor import main
+
+    main()

CHAP/saxswaxs/reader.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""SAXSWAXS command line reader."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.reader import main
+
+    main()

CHAP/saxswaxs/writer.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""SAXSWAXS command line writer."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.writer import main
+
+    main()

CHAP/server.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+"""
+File       : server.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Python server with thread pool and CHAP pipeline
+
+### Client side:
+cat /tmp/chap.json
+{"pipeline": [{"common.PrintProcessor": {}}], "input": 1}
+
+### curl call to the server with our CHAP pipeline
+curl -X POST -H "Content-type: application/json" -d@/tmp/chap.json http://localhost:5000/pipeline
+{"pipeline": [{"common.PrintProcessor":{}}], "status":"ok"}
+
+### Server side:
+flask --app server run
+ * Serving Flask app 'server'
+ * Debug mode: off
+WARNING: This is a development server. Do not use it in a production
+deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+Press CTRL+C to quit
+...
+
+CHAP.server         : Call pipeline args=()
+    kwds={'pipeline': [{'common.PrintProcessor': {}}]}
+CHAP.server         : pipeline [{'common.PrintProcessor': {}}]
+CHAP.server         : Loaded
+    <CHAP.common.processor.PrintProcessor object at 0x10e0f1ed0>
+CHAP.server         : Loaded
+    <CHAP.pipeline.Pipeline object at 0x10e0f1f10> with 1 items
+
+CHAP.server         : Calling "execute" on <CHAP.pipeline.Pipeline
+    object at 0x10e0f1f10>
+Pipeline            : Executing "execute"
+
+Pipeline            : Calling "process" on
+    <CHAP.common.processor.PrintProcessor object at 0x10e0f1ed0>
+PrintProcessor      : Executing "process" with
+    type(data)=<class 'NoneType'>
+PrintProcessor data :
+None
+PrintProcessor      : Finished "process" in 0.000 seconds
+
+Pipeline            : Executed "execute" in 0.000 seconds
+127.0.0.1 - - [07/Apr/2023 09:11:22] "POST /pipeline HTTP/1.1" 200 -
+"""
+
+# System modules
+import time
+from queue import Queue
+
+# Third-party modules
+
+# Flask modules
+from flask import Flask, request
+
+# Local modules
+from CHAP.TaskManager import TaskManager, start_new_thread
+from CHAP.runner import run, set_logger
+
+
+# Task manager to execute our tasks
+taskManager = TaskManager()
+
+# Flask Server
+app = Flask(__name__)
+
+# daemon task queue
+task_queue = Queue()
+
+@app.route("/")
+def index_route():
+    """Server main end-point."""
+
+    return "CHAP daemon"
+
+@app.route("/run")
+def run_route():
+    """Server main end-point."""
+
+    ttask = request.args.get('task')
+    task_queue.put(ttask)
+    return f'Execute {ttask}'
+
+@app.route("/pipeline", methods=["POST"])
+def pipeline_route():
+    """Server /pipeline end-point."""
+
+    content = request.json
+    if 'pipeline' in content:
+        # spawn new pipeline task
+        jobs = []
+        jobs.append(taskManager.spawn(task, pipeline=content['pipeline']))
+        taskManager.joinall(jobs)
+        return {'status': 'ok', 'pipeline': content['pipeline']}
+    return {'status': 'fail', 'reason': 'no pipeline in incoming request'}
+
+def task(*args, **kwds):
+    """Helper function to execute CHAP pipeline."""
+
+    log_level = 'INFO'
+    logger, log_handler = set_logger(log_level)
+    logger.info(f'call pipeline args={args} kwds={kwds}')
+    pipeline = kwds['pipeline']
+    logger.info(f'pipeline\n{pipeline}')
+    run(pipeline, logger, log_level, log_handler)
+
+def daemon(name, queue, interval):
+    """Daemon example based on Queue."""
+
+    print(f'Daemon {name}')
+    while True:
+        if queue.qsize() == 0:
+            print('Default action')
+            time.sleep(interval)
+        else:
+            ttask = queue.get()
+            if ttask == 'exit':
+                return
+            print(f'daemon run {ttask}')
+
+# start daemon thread in addition to Flask server
+start_new_thread('daemon', daemon, ('daemon', task_queue, 3))

CHAP/sin2psi/__init__.py

@@ -0,0 +1,7 @@
+"""This subpackage contains `PipelineItems` unique to sin2psi data
+processing workflows.
+"""
+
+# from CHAP.sin2psi.processor import
+# from CHAP.sin2psi.reader import
+# from CHAP.sin2psi.writer import

CHAP/sin2psi/processor.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""Processors used only by sin2psi experiments."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.processor import main
+
+    main()

CHAP/sin2psi/reader.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""sin2psi command line reader."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.reader import main
+
+    main()

CHAP/sin2psi/writer.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""sin2psi command line writer."""
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.writer import main
+
+    main()

CHAP/tomo/__init__.py

@@ -0,0 +1,15 @@
+"""This subpackage contains `PipelineItems` unique to tomography data
+processing workflows.
+"""
+
+from CHAP.tomo.processor import (
+    TomoMetadataProcessor,
+    TomoCHESSMapConverter,
+    TomoDataProcessor,
+    TomoSimFieldProcessor,
+    TomoDarkFieldProcessor,
+    TomoBrightFieldProcessor,
+    TomoSpecProcessor,
+)
+# from CHAP.tomo.reader import
+from CHAP.tomo.writer import TomoWriter