ChessAnalysisPipeline 0.0.17.dev3__py3-none-any.whl

CHAP/TaskManager.py

@@ -0,0 +1,216 @@
+"""
+Python thread pool, see
+http://code.activestate.com/recipes/577187-python-thread-pool/
+Author: Valentin Kuznetsov <vkuznet [AT] gmail [DOT] com>
+"""
+from builtins import range
+
+# System modules
+import time
+import json
+import hashlib
+import logging
+import threading
+from queue import Queue
+
+
+def genkey(query):
+    """Generate a new key-hash for a given query. We use md5 hash for
+    the query and key is just hex representation of this hash.
+    """
+    if  isinstance(query, dict):
+        record = dict(query)
+        query = json.JSONEncoder(sort_keys=True).encode(record)
+    keyhash = hashlib.md5()
+    keyhash.update(query.encode('utf-8', 'strict'))
+    return keyhash.hexdigest()
+
+def set_thread_name(ident, name):
+    """Set thread name for given identified."""
+    for thr in threading.enumerate():
+        if  thr.ident == ident:
+            thr.name = name
+            break
+
+class StoppableThread(threading.Thread):
+    """Thread class with a stop() method. The thread itself has to
+    check regularly for the stopped() condition.
+    """
+    def __init__(self, target, name, args):
+        super(StoppableThread, self).__init__(
+            target=target, name=name, args=args)
+        self._stop_event = threading.Event()
+
+    def stop(self):
+        """Set event to stop the thread."""
+        self._stop_event.set()
+
+    def stopped(self):
+        """Return stopped status of the thread."""
+        return self._stop_event.is_set()
+
+    def running(self):
+        """Return running status of the thread."""
+        return not self._stop_event.is_set()
+
+def start_new_thread(name, func, args, unique=False):
+    """Wrapper wroung standard thread.strart_new_thread call."""
+    if unique:
+        threads = sorted(threading.enumerate())
+        for thr in threads:
+            if name == thr.name:
+                return thr
+#     thr = threading.Thread(target=func, name=name, args=args)
+    thr = StoppableThread(target=func, name=name, args=args)
+    thr.daemon = True
+    thr.start()
+    return thr
+
+class UidSet():
+    """UID holder keeps track of uid frequency."""
+    def __init__(self):
+        self.set = {}
+
+    def add(self, uid):
+        """Add given uid or increment uid occurence in a set."""
+        if not uid:
+            return
+        if uid in self.set:
+            self.set[uid] += 1
+        else:
+            self.set[uid] = 1
+
+    def discard(self, uid):
+        """Either discard or downgrade uid occurence in a set."""
+        if uid in self.set:
+            self.set[uid] -= 1
+        if uid in self.set and not self.set[uid]:
+            del self.set[uid]
+
+    def __contains__(self, uid):
+        """Check if uid present in a set."""
+        if uid in self.set:
+            return True
+        return False
+
+    def get(self, uid):
+        """Get value for given uid."""
+        return self.set.get(uid, 0)
+
+class Worker(threading.Thread):
+    """Thread executing worker from a given tasks queue."""
+    def __init__(self, name, taskq, pidq, uidq, logger=None):
+        self.logger = logging.getLogger() if logger is None else logger
+        threading.Thread.__init__(self, name=name)
+        self.exit = 0
+        self.tasks = taskq
+        self.pids = pidq
+        self.uids = uidq
+        self.daemon = True
+        self.start()
+
+    def force_exit(self):
+        """Force run loop to exit in a hard way."""
+        self.exit = 1
+
+    def run(self):
+        """Run thread loop."""
+        while True:
+            if self.exit:
+                return
+            task = self.tasks.get()
+            if task is None:
+                return
+            if self.exit:
+                return
+            if isinstance(task, str):
+                print(f'Worker daemon run {task}')
+            elif isinstance(task, tuple) and len(task) == 5:
+                evt, pid, func, args, kwargs = task
+                try:
+                    func(*args, **kwargs)
+                    self.pids.discard(pid)
+                except Exception as exc:
+                    self.pids.discard(pid)
+                    msg = f'func={func} args={args} kwargs={kwargs}'
+                    self.logger.error(f'error {str(exc)}, call {msg}')
+                evt.set()
+            else:
+                print(f'Unsupported task {task}')
+
+class TaskManager():
+    """Task manager class based on thread module which executes
+    assigned tasks concurently. It uses a pool of thread workers,
+    queue of tasks and pid set to monitor jobs execution.
+
+    .. doctest::
+
+        Use case:
+        mgr  = TaskManager()
+        jobs = []
+        jobs.append(mgr.spawn(func, args))
+        mgr.joinall(jobs)
+    """
+    def __init__(self, nworkers=10, name='TaskManager'):
+        self.logger = logging.getLogger()
+        self.name = name
+        self.pids = set()
+        self.uids = UidSet()
+        self.tasks = Queue()
+        self.workers = [Worker(name, self.tasks, self.pids, self.uids,
+                               self.logger)
+                        for _ in range(0, nworkers)]
+
+    def status(self):
+        """Return status of task manager queue."""
+        info = {'qsize':self.tasks.qsize(), 'full':self.tasks.full(),
+                'unfinished':self.tasks.unfinished_tasks,
+                'nworkers':len(self.workers)}
+        return {self.name: info}
+
+    def nworkers(self):
+        """Return number of workers associated with this manager."""
+        return len(self.workers)
+
+    def spawn(self, func, *args, **kwargs):
+        """Spawn new process for given function."""
+        pid = kwargs.get('pid', genkey(str(args) + str(kwargs)))
+        evt = threading.Event()
+        if  not pid in self.pids:
+            self.pids.add(pid)
+            task = (evt, pid, func, args, kwargs)
+            self.tasks.put(task)
+        else:
+            # the event was not added to task list, invoke set()
+            # to pass it in wait() call, see joinall
+            evt.set()
+        return evt, pid
+
+    def remove(self, pid):
+        """Remove pid and associative process from the queue."""
+        self.pids.discard(pid)
+
+    def is_alive(self, pid):
+        """Check worker queue if given pid of the process is still
+        running.
+        """
+        return pid in self.pids
+
+    def clear(self, tasks):
+        """Clear all tasks in a queue. It allows current jobs to run,
+        but will block all new requests till workers event flag is set
+        again.
+        """
+        # Each task is return from spawn, i.e. a pair (evt, pid)
+        _ = [t[0].clear() for t in tasks]
+
+    def joinall(self, tasks):
+        """Join all tasks in a queue and quit."""
+        # Each task is return from spawn, i.e. a pair (evt, pid)
+        _ = [t[0].wait() for t in tasks]
+
+    def quit(self):
+        """Put None task to all workers and let them quit."""
+        _ = [self.tasks.put(None) for _ in self.workers]
+        # Let workers threads cool-off and quit
+        time.sleep(1)

CHAP/__init__.py

@@ -0,0 +1,27 @@
+"""The ChessAnalysisPipeline (CHAP) provides infrastructure to
+construct and run X-ray data processing / analysis workflows using a
+set of modular components. We call these components `PipelineItem`s
+(subclassed into `Reader`s, `Processor`s, and `Writer`s). A `Pipeline`
+uses a sequence of `PipelineItem`s to execute a data processing
+workflow where the data returned by one `PipelineItem` becomes input
+for the next one.
+
+Many `PipelineItem`s can be shared by data processing workflows for
+multiple different X-ray techniques, while others may be unique to
+just a single technique. The `PipelineItem`s that are shared by many
+techniques are organized in the `CHAP.common` subpackage.
+`PipelineItem`s unique to a tomography workflow, for instance, are
+organized in the `CHAP.tomo` subpackage.
+
+[`CHAP.utils`](CHAP.utils.md) contains a
+broad selection of utilities to assist in some common tasks that
+appear in specific `Processor` implementations.
+"""
+
+from CHAP.models import CHAPBaseModel
+from CHAP.reader import Reader
+from CHAP.processor import Processor
+from CHAP.writer import Writer
+
+version = 'v0.0.17.dev3'
+#version = 'v0.0.17'

CHAP/common/__init__.py

@@ -0,0 +1,57 @@
+"""This subpackage of `CHAP` contains `PipelineItem`\ s that are or can
+be used in workflows for processing data from multiple different X-ray
+techniques.
+"""
+
+from CHAP.common.processor import (
+#    AnimationProcessor,
+    AsyncProcessor,
+    BinarizeProcessor,
+    ConvertStructuredProcessor,
+    ImageProcessor,
+    MapProcessor,
+    MPICollectProcessor,
+    MPIMapProcessor,
+    MPISpawnMapProcessor,
+    NexusToNumpyProcessor,
+#    NexusToTiffsprocessor,
+    NexusToXarrayProcessor,
+    NormalizeNexusProcessor,
+    NormalizeMapProcessor,
+    PrintProcessor,
+    PyfaiAzimuthalIntegrationProcessor,
+    RawDetectorDataMapProcessor,
+    SetupNXdataProcessor,
+    UpdateNXvalueProcessor,
+    UpdateNXdataProcessor,
+    UnstructuredToStructuredProcessor,
+    NXdataToDataPointsProcessor,
+    XarrayToNexusProcessor,
+    XarrayToNumpyProcessor,
+#    SumProcessor,
+)
+from CHAP.common.reader import (
+    BinaryFileReader,
+    ConfigReader,
+    FabioImageReader,
+    H5Reader,
+    LinkamReader,
+    NexusReader,
+    NXdataReader,
+    NXfieldReader,
+    SpecReader,
+    URLReader,
+    YAMLReader,
+)
+from CHAP.common.writer import (
+    ExtractArchiveWriter,
+    FileTreeWriter,
+    H5Writer,
+    ImageWriter,
+    MatplotlibAnimationWriter,
+    MatplotlibFigureWriter,
+    NexusWriter,
+    PyfaiResultsWriter,
+    YAMLWriter,
+    TXTWriter,
+)

CHAP/common/models/__init__.py

@@ -0,0 +1,8 @@
+"""This subpackage contains `PipelineItems` that are common to various
+processing workflows.
+"""
+
+from CHAP.common.models.common import (
+    BinarizeProcessorConfig,
+    ImageProcessorConfig,
+)

CHAP/common/models/common.py

@@ -0,0 +1,124 @@
+"""Common Pydantic model classes."""
+
+# System modules
+from typing import (
+    Literal,
+    Optional,
+    Union,
+)
+
+# Third party modules
+from pydantic import (
+    confloat,
+    conint,
+    conlist,
+    constr,
+    field_validator,
+)
+#from typing_extensions import Annotated
+
+# Local modules
+from CHAP.models import CHAPBaseModel
+
+
+class BinarizeProcessorConfig(CHAPBaseModel):
+    """Configuration class to binarize a dataset in a 2D or 3D
+    array-like object or a NeXus NXdata or NXfield object.
+
+    :param method: Binarization method, defaults to `'CHAP'`
+        (CHAP's internal implementation of Otzu's method).
+    :type method: Literal['CHAP', 'isodata', 'minimum', 'otsu', 'yen']
+    :param num_bin: The number of bins used to calculate the
+        histogram in the binarization algorithms, defaults to `256`.
+    :type num_bin: int, optional
+    :param nxpath: The path to a specific NeXus NXdata or NXfield
+        object in the NeXus file tree to read the input data from
+        (ignored for non-NeXus input objects).
+    :type nxpath: str, optional
+    :param remove_original_data: Removes the original data field
+        (ignored for non-NeXus input objects), defaults to `False`.
+    :type remove_original_data: bool, optional
+    """
+    method: Optional[Literal[
+        'CHAP', 'isodata', 'minimum', 'otsu', 'yen']] = 'CHAP'
+    num_bin: Optional[conint(ge=0)] = 256
+    nxpath: Optional[str] = None
+    remove_original_data: Optional[bool] = False
+
+
+class ImageProcessorConfig(CHAPBaseModel):
+    """Class representing the configuration of various image selection
+    and visualization types of processors.
+
+    :param animation: Create an animation for an image stack
+        (ignored for a single image), defaults to `False`.
+    :type animation: bool, optional
+    :param axis: Axis direction or name for the image slice(s),
+        defaults to `0`.
+    :type axis: Union[int, str], optional
+    :param coord_range: Coordinate value range of the selected image
+        slice(s), up to three floating point numbers (start, end,
+        step), defaults to `None`, which enables index_range to select
+        the image slice(s). Include only `coord_range` or
+        `index_range`, not both.
+    :type coord_range: Union[float, list[float]], optional
+    :param index_range: Array index range of the selected image
+        slice(s), up to three integers (start, end, step).
+        Set index_range to -1 to select the center image slice
+        of an image stack in the `axis` direction. Only used when
+        coord_range = `None`. Defaults to `None`, which will include
+        all slices.
+    :type index_range: Union[int, list[int]], optional
+    :ivar fileformat: Image (stack) return file type, defaults to
+        'png' for a single image, 'tif' for an image stack, or
+        'gif' for an animation.
+    :type fileformat: Literal['gif', 'jpeg', 'png', 'tif'], optional
+    :param vrange: Data value range in image slice(s), defaults to
+        `None`, which uses the full data value range in the slice(s).
+    :type vrange: list[float, float]
+    """
+    animation: Optional[bool] = False
+    axis: Optional[Union[conint(ge=0), constr(min_length=1)]] = 0
+    coord_range: Optional[Union[
+        confloat(allow_inf_nan=False),
+        conlist(min_length=2, max_length=3,
+                item_type=confloat(allow_inf_nan=False))]] = None
+    index_range: Optional[Union[
+        int,
+        conlist(
+            min_length=2, max_length=3, item_type=Union[None, int])]] = None
+    fileformat: Optional[Literal['gif', 'jpeg', 'png', 'tif']] = None
+    vrange: Optional[
+        conlist(min_length=2, max_length=2,
+                item_type=confloat(allow_inf_nan=False))] = None
+
+    @field_validator('index_range', mode='before')
+    @classmethod
+    def validate_index_range(cls, index_range):
+        """Validate the index_range.
+
+        :ivar index_range: Array index range of the selected image
+            slice(s), defaults to `None`..
+        :type index_range: Union[float, list[float]], optional
+        :return: Validated index_range.
+        :rtype: list[int]
+        """
+        if isinstance(index_range, int):
+            return index_range
+        return [None if isinstance(i, str) and i.lower() == 'none' else i
+                for i in index_range]
+
+    @field_validator('vrange', mode='before')
+    @classmethod
+    def validate_vrange(cls, vrange):
+        """Validate the vrange.
+
+        :ivar vrange: Data value range in image slice(s),
+            defaults to `None`..
+        :type vrange: list[float, float], optional
+        :return: Validated vrange.
+        :rtype: list[float, float]
+        """
+        if isinstance(vrange, (list, tuple)) and len(vrange) == 2:
+            return [min(vrange), max(vrange)]
+        return vrange