ChessAnalysisPipeline 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

CHAP/processor.py

@@ -9,101 +9,67 @@ Description: Processor module
 
 # system modules
 import argparse
-import json
+from inspect import getfullargspec
 import logging
-import sys
+from sys import modules
 from time import time
 
-# local modules
-# from pipeline import PipelineObject
 
 class Processor():
-    """
-    Processor represent generic processor
-    """
+    """Processor represent generic processor"""
     def __init__(self):
-        """
-        Processor constructor
-        """
+        """Processor constructor"""
         self.__name__ = self.__class__.__name__
         self.logger = logging.getLogger(self.__name__)
         self.logger.propagate = False
 
-    def process(self, data):
-        """
-        process data API
+    def process(self, data, **_process_kwargs):
+        """process data API
+
+        :param _process_kwargs: keyword arguments to pass to
+            `self._process`, defaults to `{}`
+        :type _process_kwargs: dict, optional
         """
 
         t0 = time()
         self.logger.info(f'Executing "process" with type(data)={type(data)}')
 
-        data = self._process(data)
+        _valid_process_args = {}
+        allowed_args = getfullargspec(self._process).args \
+            + getfullargspec(self._process).kwonlyargs
+        for k, v in _process_kwargs.items():
+            if k in allowed_args:
+                _valid_process_args[k] = v
+            else:
+                self.logger.warning(f'Ignoring invalid arg to _process: {k}')
+
+        data = self._process(data, **_valid_process_args)
 
         self.logger.info(f'Finished "process" in {time()-t0:.3f} seconds\n')
 
-        return(data)
+        return data
 
     def _process(self, data):
+        """Private method to carry out the mechanics of the specific
+        Processor.
+
+        :param data: input data
+        :return: processed data
+        """
         # If needed, extract data from a returned value of Reader.read
         if isinstance(data, list):
-            if all([isinstance(d,dict) for d in data]):
+            if all(isinstance(d, dict) for d in data):
                 data = data[0]['data']
+        if data is None:
+            return []
         # process operation is a simple print function
         data += "process part\n"
         # and we return data back to pipeline
         return data
 
 
-class TFaaSImageProcessor(Processor):
-    '''
-    A Processor to get predictions from TFaaS inference server.
-    '''
-    def process(self, data, url, model, verbose=False):
-        """
-        process data API
-        """
-
-        t0 = time()
-        self.logger.info(f'Executing "process" with url {url} model {model}')
-
-        data = self._process(data, url, model, verbose)
-
-        self.logger.info(f'Finished "process" in {time()-t0:.3f} seconds\n')
-
-        return(data)
-
-    def _process(self, data, url, model, verbose):
-        '''Print and return the input data.
-
-        :param data: Input image data, either file name or actual image data
-        :type data: object
-        :return: `data`
-        :rtype: object
-        '''
-        from MLaaS.tfaas_client import predictImage
-        from pathlib import Path
-        self.logger.info(f"input data {type(data)}")
-        if isinstance(data, str) and Path(data).is_file():
-            imgFile = data
-            data = predictImage(url, imgFile, model, verbose)
-        else:
-            rdict = data[0]
-            import requests
-            img = rdict['data']
-            session = requests.Session()
-            rurl = url + '/predict/image'
-            payload = dict(model=model)
-            files = dict(image=img)
-            self.logger.info(f"HTTP request {rurl} with image file and {payload} payload")
-            req = session.post(rurl, files=files, data=payload )
-            data = req.content
-            data = data.decode("utf-8").replace('\n', '')
-            self.logger.info(f"HTTP response {data}")
-
-        return(data)
-
 class OptionParser():
-    '''User based option parser'''
+    """User based option parser"""
     def __init__(self):
         self.parser = argparse.ArgumentParser(prog='PROG')
         self.parser.add_argument(
@@ -116,26 +82,29 @@ class OptionParser():
             '--log-level', choices=logging._nameToLevel.keys(),
             dest='log_level', default='INFO', help='logging level')
 
+
 def main(opt_parser=OptionParser):
-    '''Main function'''
+    """Main function"""
 
-    optmgr  = opt_parser()
+    optmgr = opt_parser()
     opts = optmgr.parser.parse_args()
-    clsName = opts.processor
+    cls_name = opts.processor
     try:
-        processorCls = getattr(sys.modules[__name__],clsName)
-    except:
-        print(f'Unsupported processor {clsName}')
-        sys.exit(1)
+        processor_cls = getattr(modules[__name__], cls_name)
+    except AttributeError:
+        print(f'Unsupported processor {cls_name}')
+        raise
 
-    processor = processorCls()
+    processor = processor_cls()
     processor.logger.setLevel(getattr(logging, opts.log_level))
     log_handler = logging.StreamHandler()
-    log_handler.setFormatter(logging.Formatter('{name:20}: {message}', style='{'))
+    log_handler.setFormatter(logging.Formatter(
+        '{name:20}: {message}', style='{'))
     processor.logger.addHandler(log_handler)
     data = processor.process(opts.data)
 
-    print(f"Processor {processor} operates on data {data}")
+    print(f'Processor {processor} operates on data {data}')
+
 
 if __name__ == '__main__':
     main()

CHAP/reader.py

@@ -1,82 +1,88 @@
 #!/usr/bin/env python
-'''
+"""
 File       : reader.py
 Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
 Description: generic Reader module
-'''
+"""
 
 # system modules
 import argparse
-import json
+from inspect import getfullargspec
 import logging
-import sys
+from sys import modules
 from time import time
 
-# local modules
-# from pipeline import PipelineObject
 
 class Reader():
-    '''
-    Reader represent generic file writer
-    '''
+    """Reader represent generic file writer"""
 
     def __init__(self):
-        '''
-        Constructor of Reader class
-        '''
+        """Constructor of Reader class"""
         self.__name__ = self.__class__.__name__
         self.logger = logging.getLogger(self.__name__)
         self.logger.propagate = False
 
     def read(self, type_=None, schema=None, encoding=None, **_read_kwargs):
-        '''Read API
+        """Read API
 
         Wrapper to read, format, and return the data requested.
 
-        :param type_: the expected type of data read from `filename`, defualts
-            to `None`
+        :param type_: the expected type of data read from `filename`,
+            defualts to `None`
         :type type_: type, optional
-        :param schema: the expected schema of the data read from `filename`,
-            defaults to `None`
+        :param schema: the expected schema of the data read from
+            `filename`, defaults to `None`
         :type schema: str, otional
-        :param _read_kwargs: keyword arguments to pass to `self._read`, defaults
-            to `{}`
+        :param _read_kwargs: keyword arguments to pass to
+            `self._read`, defaults to `{}`
         :type _read_kwargs: dict, optional
-        :return: list with one item: a dictionary containing the data read from
-            `filename`, the name of this `Reader`, and the values of `type_` and
-            `schema`.
+        :return: list with one item: a dictionary containing the data
+            read from `filename`, the name of this `Reader`, and the
+            values of `type_` and `schema`.
         :rtype: list[dict[str,object]]
-        '''
+        """
 
         t0 = time()
-        self.logger.info(f'Executing "read" with type={type_}, schema={schema}, kwargs={_read_kwargs}')
+        self.logger.info(f'Executing "read" with type={type_}, '
+                         f'schema={schema}, kwargs={_read_kwargs}')
+
+        _valid_read_args = {}
+        allowed_args = getfullargspec(self._read).args \
+            + getfullargspec(self._read).kwonlyargs
+        for k, v in _read_kwargs.items():
+            if k in allowed_args:
+                _valid_read_args[k] = v
+            else:
+                self.logger.warning(f'Ignoring invalid arg to _read: {k}')
 
         data = [{'name': self.__name__,
-                 'data': self._read(**_read_kwargs),
+                 'data': self._read(**_valid_read_args),
                  'type': type_,
                  'schema': schema,
                  'encoding': encoding}]
 
         self.logger.info(f'Finished "read" in {time()-t0:.3f} seconds\n')
-        return(data)
+        return data
 
     def _read(self, filename):
-        '''Read and return the data from requested from `filename`
+        """Read and return the data from requested from `filename`
 
         :param filename: Name of file to read from
         :return: specific number of bytes from a file
-        '''
+        """
 
         if not filename:
-            self.logger.warning('No file name is given, will skip read operation')
+            self.logger.warning(
+                'No file name is given, will skip read operation')
             return None
 
         with open(filename) as file:
             data = file.read()
-        return(data)
+        return data
+
 
 class OptionParser():
-    '''User based option parser'''
+    """User based option parser"""
     def __init__(self):
         self.parser = argparse.ArgumentParser(prog='PROG')
         self.parser.add_argument(
@@ -89,26 +95,29 @@ class OptionParser():
             '--log-level', choices=logging._nameToLevel.keys(),
             dest='log_level', default='INFO', help='logging level')
 
+
 def main(opt_parser=OptionParser):
-    '''Main function'''
+    """Main function"""
 
-    optmgr  = opt_parser()
+    optmgr = opt_parser()
     opts = optmgr.parser.parse_args()
-    clsName = opts.reader
+    cls_name = opts.reader
     try:
-        readerCls = getattr(sys.modules[__name__],clsName)
-    except:
-        print(f'Unsupported reader {clsName}')
-        sys.exit(1)
+        reader_cls = getattr(modules[__name__], cls_name)
+    except AttributeError:
+        print(f'Unsupported reader {cls_name}')
+        raise
 
-    reader = readerCls()
+    reader = reader_cls()
     reader.logger.setLevel(getattr(logging, opts.log_level))
     log_handler = logging.StreamHandler()
-    log_handler.setFormatter(logging.Formatter('{name:20}: {message}', style='{'))
+    log_handler.setFormatter(logging.Formatter(
+        '{name:20}: {message}', style='{'))
     reader.logger.addHandler(log_handler)
     data = reader.read(filename=opts.filename)
 
     print(f'Reader {reader} reads from {opts.filename}, data {data}')
 
+
 if __name__ == '__main__':
     main()

CHAP/runner.py

@@ -1,80 +1,121 @@
-#!/usr/bin/env python
-#-*- coding: utf-8 -*-
-#pylint: disable=
 """
 File       : runner.py
 Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
-Description: 
+Description:
 """
 
 # system modules
 import argparse
 import logging
-import os
-import sys
-import yaml
+from yaml import safe_load
 
 # local modules
 from CHAP.pipeline import Pipeline
 
 
 class OptionParser():
+    """User based option parser"""
     def __init__(self):
-        "User based option parser"
+        """OptionParser class constructor"""
         self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--config", action="store",
-            dest="config", default="", help="Input configuration file")
-        self.parser.add_argument('--log-level', choices=logging._nameToLevel.keys(),
+        self.parser.add_argument(
+            '--config', action='store', dest='config', default='',
+            help='Input configuration file')
+        self.parser.add_argument(
+            '--interactive', action='store_true', dest='interactive',
+            help='Allow interactive processes')
+        self.parser.add_argument(
+            '--log-level', choices=logging._nameToLevel.keys(),
             dest='log_level', default='INFO', help='logging level')
+        self.parser.add_argument(
+            '--profile', action='store_true', dest='profile',
+            help='profile output')
+
 
 def main():
-    "Main function"
-    optmgr  = OptionParser()
+    """Main function"""
+    optmgr = OptionParser()
     opts = optmgr.parser.parse_args()
-    runner(opts)
+    if opts.profile:
+        from cProfile import runctx  # python profiler
+        from pstats import Stats     # profiler statistics
+        cmd = 'runner(opts)'
+        runctx(cmd, globals(), locals(), 'profile.dat')
+        info = Stats('profile.dat')
+        info.sort_stats('cumulative')
+        info.print_stats()
+    else:
+        runner(opts)
+
 
 def runner(opts):
+    """Main runner function
+
+    :param opts: object containing input parameters
+    :type opts: OptionParser
     """
-    Main runner function
 
-    :param opts: opts is an instance of argparse.Namespace which contains all input parameters
+    log_level = opts.log_level.upper()
+    logger, log_handler = setLogger(log_level)
+    config = {}
+    with open(opts.config) as file:
+        config = safe_load(file)
+    logger.info(f'Input configuration: {config}\n')
+    pipeline_config = config.get('pipeline', [])
+    run(pipeline_config, opts.interactive, logger, log_level, log_handler)
+
+def setLogger(log_level="INFO"):
     """
+    Helper function to set CHAP logger
 
+    :param log_level: logger level, default INFO
+    """
     logger = logging.getLogger(__name__)
-    log_level = getattr(logging, opts.log_level.upper())
+    log_level = getattr(logging, log_level.upper())
     logger.setLevel(log_level)
     log_handler = logging.StreamHandler()
-    log_handler.setFormatter(logging.Formatter('{name:20}: {message}', style='{'))
+    log_handler.setFormatter(logging.Formatter(
+        '{name:20}: {message}', style='{'))
     logger.addHandler(log_handler)
+    return logger, log_handler
 
-    config = {}
-    with open(opts.config) as file:
-        config = yaml.safe_load(file)
-    logger.info(f'Input configuration: {config}\n')
-    pipeline_config = config.get('pipeline', [])
+def run(pipeline_config, interactive=False, logger=None, log_level=None, log_handler=None):
+    """
+    Run given pipeline_config
+
+    :param pipeline_config: CHAP pipeline config
+    """
     objects = []
     kwds = []
     for item in pipeline_config:
         # load individual object with given name from its module
+        kwargs = {'interactive': interactive}
         if isinstance(item, dict):
             name = list(item.keys())[0]
-            kwargs = item[name]
+            # Combine the "interactive" command line argument with the object's keywords
+            # giving precedence of "interactive" in the latter
+            kwargs = {**kwargs, **item[name]}
         else:
             name = item
-            kwargs = {}
         modName, clsName = name.split('.')
         module = __import__(f'CHAP.{modName}', fromlist=[clsName])
         obj = getattr(module, clsName)()
-        obj.logger.setLevel(log_level)
-        obj.logger.addHandler(log_handler)
-        logger.info(f'Loaded {obj}')
+        if log_level:
+            obj.logger.setLevel(log_level)
+        if log_handler:
+            obj.logger.addHandler(log_handler)
+        if logger:
+            logger.info(f'Loaded {obj}')
         objects.append(obj)
         kwds.append(kwargs)
     pipeline = Pipeline(objects, kwds)
-    pipeline.logger.setLevel(log_level)
-    pipeline.logger.addHandler(log_handler)
-    logger.info(f'Loaded {pipeline} with {len(objects)} items\n')
-    logger.info(f'Calling "execute" on {pipeline}')
+    if log_level:
+        pipeline.logger.setLevel(log_level)
+    if log_handler:
+        pipeline.logger.addHandler(log_handler)
+    if logger:
+        logger.info(f'Loaded {pipeline} with {len(objects)} items\n')
+        logger.info(f'Calling "execute" on {pipeline}')
     pipeline.execute()
 
 

CHAP/saxswaxs/processor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.processor import main

CHAP/saxswaxs/reader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.reader import main

CHAP/saxswaxs/writer.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.writer import main

CHAP/server.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : server.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Python server with thread pool and CHAP pipeline
+
+### Client side:
+cat /tmp/chap.json
+{
+  "pipeline": [{"common.PrintProcessor": {}}],
+  "input": 1
+}
+
+### curl call to the server with our CHAP pipeline
+curl -X POST -H "Content-type: application/json" -d@/tmp/chap.json http://localhost:5000/pipeline
+{"pipeline":[{"common.PrintProcessor":{}}],"status":"ok"}
+
+### Server side:
+flask --app server run
+ * Serving Flask app 'server'
+ * Debug mode: off
+WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
+ * Running on http://127.0.0.1:5000
+Press CTRL+C to quit
+...
+
+CHAP.server         : call pipeline args=() kwds={'pipeline': [{'common.PrintProcessor': {}}]}
+CHAP.server         : pipeline
+[{'common.PrintProcessor': {}}]
+CHAP.server         : Loaded <CHAP.common.processor.PrintProcessor object at 0x10e0f1ed0>
+CHAP.server         : Loaded <CHAP.pipeline.Pipeline object at 0x10e0f1f10> with 1 items
+
+CHAP.server         : Calling "execute" on <CHAP.pipeline.Pipeline object at 0x10e0f1f10>
+Pipeline            : Executing "execute"
+
+Pipeline            : Calling "process" on <CHAP.common.processor.PrintProcessor object at 0x10e0f1ed0>
+PrintProcessor      : Executing "process" with type(data)=<class 'NoneType'>
+PrintProcessor data :
+None
+PrintProcessor      : Finished "process" in 0.000 seconds
+
+Pipeline            : Executed "execute" in 0.000 seconds
+127.0.0.1 - - [07/Apr/2023 09:11:22] "POST /pipeline HTTP/1.1" 200 -
+"""
+
+# system modules
+import time
+import logging
+from queue import Queue
+
+# thrid-party modules
+
+# Flask modules
+from flask import Flask, request, jsonify
+
+# CHAP modules
+from CHAP.TaskManager import TaskManager, start_new_thread 
+from CHAP.runner import run, setLogger
+
+
+# Task manager to execute our tasks
+taskManager = TaskManager()
+
+# Flask Server
+app = Flask(__name__)
+
+# daemon task queue
+task_queue = Queue()
+
+@app.route("/")
+def index_route():
+    """
+    Server main end-point
+    """
+    return "CHAP daemon"
+
+@app.route("/run")
+def run_route():
+    """
+    Server main end-point
+    """
+    task = request.args.get('task')
+    task_queue.put(task)
+    return f"Execute {task}"
+
+@app.route("/pipeline", methods=["POST"])
+def pipeline_route():
+    """
+    Server /pipeline end-point
+    """
+    content = request.json
+    if 'pipeline' in content:
+        # spawn new pipeline task
+        jobs = []
+        jobs.append(taskManager.spawn(task, pipeline=content['pipeline']))
+        taskManager.joinall(jobs)
+        return {"status": "ok", "pipeline": content['pipeline']}
+    else:
+        return {"status": "fail", "reason": "no pipeline in incoming request"}
+
+def task(*args, **kwds):
+    """
+    Helper function to execute CHAP pipeline
+    """
+    log_level = "INFO"
+    logger, log_handler = setLogger(log_level)
+    logger.info(f"call pipeline args={args} kwds={kwds}")
+    pipeline = kwds['pipeline']
+    logger.info(f"pipeline\n{pipeline}")
+    run(pipeline, logger, log_level, log_handler)
+
+def daemon(name, queue, interval):
+    """
+    Daemon example based on Queue
+    """
+    print(f"Daemon {name}")
+    while True:
+        if queue.qsize() == 0:
+            print("Default action")
+            time.sleep(interval)
+        else:
+            task = queue.get()
+            if task == "exit":
+                return
+            print(f"daemon run {task}")
+
+# start daemon thread in addition to Flask server
+start_new_thread("daemon", daemon, ("daemon", task_queue, 3))

CHAP/sin2psi/processor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.processor import main

CHAP/sin2psi/reader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.reader import main

CHAP/sin2psi/writer.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python  
+#!/usr/bin/env python
 
 if __name__ == '__main__':
     from CHAP.writer import main

CHAP/tomo/__init__.py

@@ -1,5 +1,2 @@
-# from CHAP.tomo.reader import
-from CHAP.tomo.processor import TomoDataProcessor
-# from CHAP.tomo.writer import
-
 from CHAP.common import MapProcessor
+from CHAP.tomo.processor import TomoDataProcessor