cdxcore 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

cdxcore/__init__.py

@@ -4,4 +4,4 @@ Created on June 2022
 @author: hansb
 """
 
-__version__ = "0.1.10"  # auto-updated by setup.py
+__version__ = "0.1.11"  # auto-updated by setup.py

cdxcore/crman.py

@@ -147,7 +147,10 @@ class CRMan(object):
         """
         return self._current
         
-    def write(self, text : str, end : str = '', flush : bool = True, channel : Callable = None ):
+    def write(self, text : str, 
+                    end : str = '', 
+                    flush : bool = True, 
+                    channel : Callable = None ):
         r"""
         Write to a ``channel``,
         

cdxcore/jcpool.py

@@ -1,34 +1,57 @@
-# -*- coding: utf-8 -*-
 """
-Simple multi-processing wrapper around the already great joblib.paralllel.
+Overview
+--------
+
+Simple multi-processing conv wrapper around (already great)
+`joblib.Parallel() <https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html>`__.
+
 The minor additions are that parallel processing will be a tad more convenient for dictionaries,
-and that it supports routing cdxbasics.cdxbasics.Context messaging via a Queue to a single thread.
+and that it supports routing :class:`cdxcore.verbose.Context` messaging via a
+:class:`multiprocessing.Queue` to a single thread.
+
+Import
+------
+.. code-block:: python
+
+    from cdxcore.jcpool import JCPool
+    
+Documentation
+-------------
 """
 
-from joblib import Parallel as joblib_Parallel, delayed as jl_delayed
+from joblib import Parallel as joblib_Parallel, delayed as _jl_delayed, cpu_count
 from multiprocessing import Manager, Queue
 from threading import Thread, get_ident as get_thread_id
 import gc as gc
 from collections import OrderedDict
 from collections.abc import Mapping, Callable, Sequence, Iterable
 import functools as functools
+import uuid as uuid
+import os as os
+import datetime as datetime
 
 from .verbose import Context, Timer
 from .subdir import SubDir
+from .uniquehash import unique_hash8
 
 class ParallelContextChannel( Context ):
     """
-    Lightweight channel for cdxbasics.verbose.Context which is pickle'able
-    Implements trivial Context channel() protocol.
+    Lightweight :class:`cdxcore.verbose.Context` ``channel`` which is pickle'able.
+
+    This channel sends messages it receives to a :class:`multiprocessing.Queue`.
     """
-    def __init__(self, *, cid, maintid, queue):
+    def __init__(self, *, cid, maintid, queue, f_verbose):
         self._queue        = queue
         self._cid          = cid
         self._maintid      = maintid
+        self._f_verbose    = f_verbose
     def __call__(self, msg : str, flush : bool ):
-        """ Context channel call (outside process) to send messages to 'report' """
+        """
+        Sends ``msg`` via a :class:`multiprocessing.Queue` to the main thread for
+        printing.
+        """
         if get_thread_id() == self._maintid:
-            print(msg, end='', flush=True)
+            self._f_verbose.write(msg, end='', flush=flush)
         else:
             return self._queue.put( (msg, flush) )
 
@@ -51,7 +74,12 @@ class _ParallelContextOperator( object ):
             self._queue        = self._mgr.Queue()
             self._thread       = Thread(target=self.report, kwargs=dict(cid=cid, queue=self._queue, f_verbose=f_verbose, verbose_interval=verbose_interval), daemon=True)
             self._mp_context   = Context( f_verbose, 
-                                          channel=ParallelContextChannel( cid=self._cid, queue=self._queue, maintid=self._tid ) )
+                                          channel=ParallelContextChannel(
+                                                    cid=self._cid, 
+                                                    queue=self._queue, 
+                                                    maintid=self._tid,
+                                                    f_verbose=f_verbose
+                                                    ) )
             self._thread.start()
             pool_verbose.write(f"done; this took {tme}.", head=False)
 
@@ -97,7 +125,7 @@ class _ParallelContextOperator( object ):
                 raise r
             msg, flush = r
             if tme.interval_test(verbose_interval):
-                print(msg, end='', flush=flush)
+                f_verbose.write(msg, end='', flush=flush)
 
     def __enter__(self):
         return self.mp_context
@@ -196,104 +224,251 @@ def _parallel_to_list(pool, jobs : Sequence ) -> Sequence:
         An list with the results in order of the input.
     """
     assert not isinstance( jobs, Mapping ), ("'jobs' is a Mapping. Use parallel_to_dict() instead.", type(jobs))
-    r = _parallel_to_dict( pool, { i: j for i, j in enumerate(jobs) } )
-    return list( r[i] for i in range(len(jobs)) ) 
+    lst = { i: j for i, j in enumerate(jobs) }
+    r   = _parallel_to_dict( pool, lst )
+    return list( r[i] for i in lst ) 
 
 class JCPool( object ):
-    """
-    Parallel Job Context Pool
+    r"""
+    Parallel Job Context Pool.
     
-    Simple wrapper around joblib.Parallel which allows using cdxbasics.verbose.Context objects seemlessly:
-    use of any contexts from a different process will send messages via a Queue to the main process
+    Simple wrapper around `joblib.Parallel() <https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html>`__ 
+    which allows worker processes to use :class:`cdxcore.verbose.Context` to report
+    progress updates. For this purpose, :class:`cdxcore.verbose.Context` 
+    will send output messages via a :class:`multiprocessing.Queue`
+    to the main process
     where a sepeate thread prints these messages out.
-    Using a fixed pool object also avoids relaunching processes.
+    
+    Using a fixed central pool object in  your code base
+    avoids relaunching processes.
 
-    Finally, the parallel pool allows working directly with dictionaries which is useful for asynchronous
-    processing (which is the default).
+    Functions passed to :meth:`cdxcore.jcpool.JCPool.parallel` and related functions must
+    be decorated with :dec:`cdxcore.jcpool.JCPool.delayed`.
 
-    Usage
-    -----
-    Assume we have a function such as:
+    **List/Generator Usage**
 
+    The following code is a standard prototype for using :func:`cdxcore.jcpool.JCPool.parallel`
+    following closely the `joblib paradigm <https://joblib.readthedocs.io/en/latest/parallel.html>`__:
+
+    .. code-block:: python
+
+        from cdxcore.verbose import Context
+        from cdxcore.jcpool import JCPool
+        import time as time 
+        import numpy as np
+
+        pool    = JCPool( num_workers=4 )   # global pool. Reuse where possible
+        
         def f( ticker, tdata, verbose : Context ):
-            #...
-            tx = 0.
-            ty = 1.
-            verbose.write(f"Result for {ticker}: {tx}, {ty}")
-            return tx, ty # tuple result for illustration
-
-    List/Generator
-    --------------
-    Use the pool.context() context handler to convert a Context 'verbose' object into a multi-processing channel.
-    Then pass a generator to pool.parallel
-    
-        pool    = JPool( num_workers=4 )
+            # some made up function
+            q  = np.quantile( tdata, 0.35, axis=0 )
+            tx = q[0]
+            ty = q[1]
+            time.sleep(0.5)
+            verbose.write(f"Result for {ticker}: {tx:.2f}, {ty:.2f}")
+            return tx, ty
+        
+        tickerdata =\
+         { 'SPY': np.random.normal(size=(1000,2)),
+           'GLD': np.random.normal(size=(1000,2)), 
+           'BTC': np.random.normal(size=(1000,2))
+         } 
+        
         verbose = Context("all")
-        with pool.context( verbose ) as verbose:
-            for tx, ty in pool.parallel( pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose ) for ticker, tdata in self.data.items() ):
-                print(f"Returned {tx}, {ty}")
-            print("Done")
+        with verbose.write_t("Launching analysis") as tme:
+            with pool.context( verbose ) as verbose:
+                for tx, ty in pool.parallel(
+                            pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                            for ticker, tdata in tickerdata.items() ):
+                    verbose.report(1,f"Returned {tx:.2f}, {ty:.2f}")
+        verbose.write(f"Analysis done; this took {tme}.")
     
-    Dict
-    ----
-    Similar construct, but with a dictionary. Considering the asynchronous nature of the returned data it is often desirable
-    to keep track of a result identifier. This is automated with the dictionary usage pattern:
+    The output from this code is asynchronous:
+
+    .. code-block:: python
+
+        00: Launching analysis
+        02:     Result for SPY: -0.43, -0.39
+        01:   Returned -0.43, -0.39
+        02:     Result for BTC: -0.39, -0.45
+        01:   Returned -0.39, -0.45
+        02:     Result for GLD: -0.41, -0.43
+        01:   Returned -0.41, -0.43
+        00: Analysis done; this took 0.73s.        
+
+    **Dict**
+
+    Considering the asynchronous nature of the returned data it is often desirable
+    to keep track of results by some identifier. In above example ``ticker``
+    was not available in the main loop.
+    This pattern is automated with the dictionary usage pattern:
     
-        pool = JPool( num_workers=4 )
+    .. code-block:: python
+       :emphasize-lines: 26,27,28,29
+
+        from cdxcore.verbose import Context
+        from cdxcore.jcpool import JCPool
+        import time as time 
+        import numpy as np
+
+        pool    = JCPool( num_workers=4 )   # global pool. Reuse where possible
+        
+        def f( ticker, tdata, verbose : Context ):
+            # some made up function
+            q  = np.quantile( tdata, 0.35, axis=0 )
+            tx = q[0]
+            ty = q[1]
+            time.sleep(0.5)
+            verbose.write(f"Result for {ticker}: {tx:.2f}, {ty:.2f}")
+            return tx, ty
+        
+        tickerdata =\
+         { 'SPY': np.random.normal(size=(1000,2)),
+           'GLD': np.random.normal(size=(1000,2)), 
+           'BTC': np.random.normal(size=(1000,2))
+         } 
+        
         verbose = Context("all")
-        with pool.context( verbose ) as verbose:
-            for ticker, tx, ty in pool.parallel( { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose ) for ticker, tdata in self.data.items() } ):
-                print(f"Returned {tx}, {ty} for {ticker}")
-            print("Done")
-                    
-    Note that pool.parallel when applied to a dictionary does not return a dictionary, but a sequence of tuples.
+        with verbose.write_t("Launching analysis") as tme:
+            with pool.context( verbose ) as verbose:
+                for ticker, tx, ty in pool.parallel(
+                        { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                          for ticker, tdata in tickerdata.items() } ):
+            verbose.report(1,f"Returned {ticker} {tx:.2f}, {ty:.2f}")
+        verbose.write(f"Analysis done; this took {tme}.")
+    
+    This generates the following output::
+
+        00: Launching analysis
+        02:     Result for SPY: -0.34, -0.41
+        01:   Returned SPY -0.34, -0.41
+        02:     Result for GLD: -0.38, -0.41
+        01:   Returned GLD -0.38, -0.41
+        02:     Result for BTC: -0.34, -0.32
+        01:   Returned BTC -0.34, -0.32
+        00: Analysis done; this took 5s.
+    
+    Note that :func:`cdxcore.jcpool.JCPool.parallel` when applied to a dictionary does not return a dictionary,
+    but a sequence of tuples.
     As in the example this also works if the function being called returns tuples itself; in this case the returned data
     is extended by the key of the dictionary provided.
     
-    In order to retrieve a dictionary use
+    In order to retrieve a dictionary use :func:`cdxcore.jcpool.JCPool.parallel_to_dict`::
 
-        pool = JPool( num_workers=4 )
         verbose = Context("all")
         with pool.context( verbose ) as verbose:
-            r = pool.parallel_to_dict( { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose ) for ticker, tdata in self.data.items() } )
-            print("Done")
+            r = pool.parallel_to_dict( { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose )
+                                         for ticker, tdata in self.data.items() } )
 
-    Note that in this case the function returns after all items have been processed.
+    Note that in this case the function returns only after all jobs have been processed.
+    
+    Parameters
+    ----------
+        num_workers : int, optional
+            
+            The number of workers. If ``num_workers`` is ``1`` then no parallel process or thread is started.
+            Just as for `joblib <https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html>`__ you can
+            use a negative ``num_workers`` to set the number of workers to the ``number of CPUs + num_workers + 1``.
+            For example, a ``num_workers`` of ``-2`` will use as many jobs as CPUs are present less one.
+            If ``num_workers`` is negative, the effective number of workers will be at least ``1``.
+            
+            Default is ``1``.
+        
+        threading : bool, optional
+        
+            If ``False``, the default, then the pool will act as a ``"loky"`` multi-process pool with the associated overhead
+            of managing data accross processes.
+            
+            If ``True``, then the pool is a ``"threading"`` pool. This helps for functions whose code releases
+            Python's `global interpreter lock <https://wiki.python.org/moin/GlobalInterpreterLock>`__, for example
+            when engaged in heavy I/O or compiled code such as :mod:`numpy`., :mod:`pandas`,
+            or generated with `numba <https://numba.pydata.org/>`__.
+            
+        tmp_root_dir : str | SubDir, optional
+        
+            Temporary directory for memory mapping large arrays. This is a root directory; the function
+            will create a temporary sub-directory with a name generated from the current state of the system.
+            This sub-directory will be deleted upon destruction of ``JCPool`` or when :meth:`cdxcore.jcpool.JCPool.terminate`
+            is called.
+            
+            This parameter can also be ``None`` in which case the `default behaviour <https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html>`__
+            of :class:`joblib.Parallel` is used.
+            
+            Default is ``"!/.cdxmp"``.
+            
+        verbose : Context, optional
+            
+            A :class:`cdxcore.verbose.Context` object used to print out multi-processing/threading information.
+            This is *not* the ``Context`` provided to child processes/threads.
+            
+            Default is ``quiet``.
+
+        parallel_kwargs : dict, optional
+        
+            Additional keywords for :class:`joblib.Parallel`.
+    
     """
     def __init__(self, num_workers      : int = 1,
                        threading        : bool = False,
-                       tmp_dir          : str = "!/.cdxmp",  *,
+                       tmp_root_dir     : str|SubDir= "!/.cdxmp",  *,
                        verbose          : Context = Context.quiet,
                        parallel_kwargs  : dict = {} ):
         """
-        Initialize a multi-processing pool. Thin wrapper aroud joblib.parallel for cdxbasics.verbose.Context() output
+        Initialize a multi-processing pool. Thin wrapper aroud joblib.parallel for cdxcore.verbose.Context() output
         """
+        tmp_dir_ext            = unique_hash8( uuid.getnode(), os.getpid(), get_thread_id(), datetime.datetime.now() )
         num_workers            = int(num_workers)
-        self._tmp_dir          = SubDir(tmp_dir, ext='')
+        tmp_root_dir           = SubDir(tmp_root_dir) if not tmp_root_dir is None else None
+        self._tmp_dir          = tmp_root_dir(tmp_dir_ext, ext='') if not tmp_root_dir is None else None
         self._verbose          = verbose if not verbose is None else Context("quiet")
         self._threading        = threading
-        assert num_workers > 0, ("'num_workers' must be positive", num_workers)
+
+        if num_workers < 0:
+            num_workers = max( self.cpu_count() + num_workers + 1, 1 )
         
-        with self._verbose.write_t(f"Launching {num_workers} processes with temporary path '{self.tmp_path}'... ", end='') as tme:
+        path_info = f" with temporary directory '{self.tmp_path}'" if not self.tmp_path is None else ''
+        with self._verbose.write_t(f"Launching {num_workers} processes{path_info}... ", end='') as tme:
             self._pool = joblib_Parallel( n_jobs=num_workers, 
                                           backend="loky" if not threading else "threading", 
                                           return_as="generator_unordered", 
-                                          temp_folder=self.tmp_path, **parallel_kwargs)
+                                          temp_folder=self.tmp_path,
+                                          **parallel_kwargs)
             self._verbose.write(f"done; this took {tme}.", head=False)
 
     def __del__(self):
         self.terminate()
 
     @property
-    def tmp_path(self) -> str:
-        return self._tmp_dir.path
+    def tmp_path(self) -> str|None:
+        """ Path to the temporary directory for this object. """
+        return self._tmp_dir.path if not self._tmp_dir is None else None
     @property
     def is_threading(self) -> bool:
+        """ Whether we are threading or mulit-processing. """
         return self._threading
+    
+    @staticmethod
+    def cpu_count( only_physical_cores : bool = False ) -> int:
+        """
+        Return the number of physical CPUs.
+        
+        Parameters
+        ----------
+            only_physical_cores : boolean, optional
+            
+                If ``True``, does not take hyperthreading / SMT logical cores into account.
+                Default is ``False``.
+        
+        Returns
+        -------
+            cpus : int
+                Count
+        """
+        return cpu_count(only_physical_cores=only_physical_cores)
 
     def terminate(self):
         """
-        Stop the current parallel pool, and delete any temporary files.
+        Stop the current parallel pool, and delete any temporary files (if managed by ``JCPool``).
         """
         if not self._pool is None:
             tme = Timer()
@@ -301,14 +476,41 @@ class JCPool( object ):
             self._pool = None
             self._verbose.write(f"Shut down parallel pool. This took {tme}.")
         gc.collect()
-        self._tmp_dir.eraseEverything(keepDirectory=True)
+        if not self._tmp_dir is None:
+            dir_name = self._tmp_dir.path
+            self._tmp_dir.delete_everything(keep_directory=False)
+            self._verbose.write(f"Deleted temporary directoru {dir_name}.")
 
     def context( self, verbose : Context, verbose_interval : float = None ):
         """
-        Return a cdxbasics.verbose.Context object whose 'channel' is a queue towards a parallel thread.
-        As a result the worker process is able to use 'verbose' as if it were in-process
+        Parallel processing ``Context`` object.
+        
+        This function returns a :class:`cdxcore.verbose.Context` object whose ``channel`` is a queue towards a utility thread
+        which will outout all messages to ``verbose``.
+        As a result a worker process is able to use ``verbose`` as if it were in-process
+        
+        A standard usage pattern is:
+            
+        .. code-block:: python
+            :emphasize-lines: 13, 14
+
+            from cdxcore.verbose import Context
+            from cdxcore.jcpool import JCPool
+            import time as time 
+            import numpy as np
+    
+            pool    = JCPool( num_workers=4 )   # global pool. Reuse where possible
+            
+            def f( x, verbose : Context ):
+                verbose.write(f"Found {x}")     # <- text "Found 1" etc will be sent
+                return x                        #    to main thread via Queue 
+             
+            verbose = Context("all")
+            with pool.context( verbose ) as verbose:
+                for x in pool.parallel( pool.delayed(f)( x=x, verbose=verbose(1) ) for x in [1,2,3,4] ):
+                    verbose.write(f"Returned {x}")                    
         
-        See help(JCPool) for usage patterns.
+        See :class:`cdxcore.jcpool.JCPool` for more usage patterns.
         """
         if self._threading:
             return verbose
@@ -317,8 +519,8 @@ class JCPool( object ):
                                          verbose_interval=verbose_interval )
 
     @staticmethod
-    def validate( F : Callable, args : list, kwargs : Mapping ):
-        """ Check that 'args' and 'kwargs' do not contain Context objects without channel """
+    def _validate( F : Callable, args : list, kwargs : Mapping ):
+        """ Check that ``args`` and ``kwargs`` do not contain ``Context`` objects without channel """
         for k, v in enumerate(args):
             if isinstance(v, Context) and not isinstance(v.channel, ParallelContextChannel):
                 raise RuntimeError(f"Argument #{k} for {F.__qualname__} is a Context object, but its channel is not set to 'ParallelContextChannel'. Use JPool.context().")
@@ -328,22 +530,29 @@ class JCPool( object ):
 
     def delayed(self, F : Callable):
         """
-        Decorate a function F for parallel execution.
-        Synthatical sugar aroud joblib.delayed().
-        Checks that there are no Context arguments without ParallelContextChannel present.
+        Decorate a function for parallel execution.
+        
+        This decorate adds minor synthatical sugar on top of :func:`joblib.delayed`
+        (which in turn is discussed `here <https://joblib.readthedocs.io/en/latest/parallel.html#parallel>`__).
+
+        When called, this decorator checks that no :class:`cdxcore.verbose.Context`
+        arguments are passed to the pooled function which have no ``ParallelContextChannel`` present. In other words,
+        the function detects if the user forgot to use :meth:`cdxcore.jcpool.JCPool.context`.
         
         Parameters
         ----------
-            F : function.
+            F : Callable
+                Function.
             
         Returns
         -------
-            Decorated function.
+            wrapped F : Callable
+                Decorated function.
         """
         if self._threading:
-            return jl_delayed(F)
+            return _jl_delayed(F)
         def delayed_function( *args, **kwargs ):
-            JCPool.validate( F, args, kwargs )
+            JCPool._validate( F, args, kwargs )
             return F, args, kwargs # mimic joblin.delayed()
         try:
             delayed_function = functools.wraps(F)(delayed_function)
@@ -351,61 +560,84 @@ class JCPool( object ):
             " functools.wraps fails on some callable objects "
         return delayed_function
 
-    def parallel(self, jobs : Iterable) -> Iterable:
+    def parallel(self, jobs : Sequence|Mapping) -> Iterable:
         """
-        Process 'jobs' in parallel using the current multiprocessing pool.
-        All (function) values of 'jobs' must be generated using self.delayed.
-        See help(JCPool) for usage patterns.
+        Process a number of jobs in parallel using the current multiprocessing pool.
+        
+        All functions used in ``jobs`` must have been decorated using :dec:`cdxcore.jcpool.JCPool.delayed`.
+        
+        This function returns an iterator which yields results as soon as they 
+        are computed.
+        
+        If ``jobs`` is a ``Sequence`` you can also use
+        :meth:`cdxcore.jcpool.JCPool.parallel_to_list` to retrieve
+        a :class:`list` of all results upon completion of the last job. Similarly, if ``jobs`` 
+        is a ``Mapping``, use :meth:`cdxcore.jcpool.JCPool.parallel_to_dict` to retrieve
+        a :class:`dict` of results upon completion of the last job.
         
         Parameters
         ----------
-            jobs:
-                can be a sequence, a generator, or a dictionary.
-                Each function value must have been generated using JCPool.delayed()
+            jobs :  Sequence | Mapping
+                Can be a :class:`Sequence` containing ``Callable`` functions,
+                or a :class:`Mapping` whose values are ``Callable`` functions.
+
+                Each ``Callable`` used as part of either must
+                have been decorated with :dec:`cdxcore.jcpool.JCPool.delayed`.
                 
         Returns
         -------
-            An iterator which yields results as soon as they are available.   
-            If 'jobs' is a dictionary, then the resutling iterator will generate tuples with the first
-            element equal to the dictionary key of the respective function job.
+            parallel : Iterator
+                An iterator which yields results as soon as they are available.   
+                If ``jobs`` is a :class:`Mapping`, then the resutling iterator will generate tuples with the first
+                element equal to the mapping key of the respective function job. This function will *not*
+                return a dictionary.
         """
         return _parallel( self._pool, jobs )
 
-    def parallel_to_dict(self, jobs : Mapping) -> Mapping:
+    def parallel_to_dict(self, jobs : Mapping) -> dict:
         """
-        Process 'jobs' in parallel using the current multiprocessing pool.
-        All values of the dictionary 'jobs' must be generated using self.delayed.
-        This function awaits the calculation of all elements of 'jobs' and
-        returns a dictionary with the results.
+        Process a number of jobs in parallel using the current multiprocessing pool,
+        and return all results in a dictionary upon completion.
+        
+        This function awaits the calculation of all elements of ``jobs`` and
+        returns a :class:`dict` with the results.
         
-        See help(JCPool) for usage patterns.
-
         Parameters
         ----------
-            jobs:
-                A dictionary where all (function) values must have been generated using JCPool.delayed.
+            jobs : Mapping
+                A dictionary where all (function) values must have been decorated
+                with :dec:`cdxcore.jcpool.JCPool.delayed`.
                 
         Returns
         -------
-            A dictionary with results.
-            If 'jobs' is an OrderedDict, then this function will return an OrderedDict
-            with the same order as 'jobs'.
+            Results : dict
+                A dictionary with results.
+                
+                If ``jobs`` is an :class:`OrderedDict`, then this function will return an :class:`OrderedDict`
+                with the same order as ``jobs``. Otherwise the elements of the ``dict`` returned
+                by this function are in completion order.
         """
         return _parallel_to_dict( self._pool, jobs )
                 
     def parallel_to_list(self, jobs : Sequence ) -> Sequence:
         """
-        Call parallel() and convert the resulting generator into a list.
-
+        Process a number of jobs in parallel using the current multiprocessing pool,
+        and return all results in a list upon completion.
+        
+        This function awaits the calculation of all elements of ``jobs`` and
+        returns a :class:`list` with the results.
+        
         Parameters
         ----------
-            jobs:
-                can be a sequence, a generator, or a dictionary.
-                Each function value must have been generated using JCPool.delayed()
+            jobs : Sequence 
+                An sequence of ``Callable`` functions, each of which 
+                must have been decorated
+                with :dec:`cdxcore.jcpool.JCPool.delayed`.
                 
         Returns
         -------
-            An list with the results in order of the input.
+            Results : list
+                A list with results, in the order of ``jobs``.
         """
         return _parallel_to_list( self._pool, jobs )
 

cdxcore/subdir.py

@@ -1786,7 +1786,7 @@ class SubDir(object):
         # write to temp file, then rename into target file
         # this reduces collision when i/o operations are slow
         full_file_name = self.full_file_name(file,ext=ext)
-        tmp_file     = unique_hash48( [ file, uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() ] )
+        tmp_file     = unique_hash48( file, uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() )
         tmp_i        = 0
         fullTmpFile  = self.full_file_name(tmp_file,ext="tmp" if not ext=="tmp" else "_tmp")
         while os.path.exists(fullTmpFile):

{cdxcore-0.1.10.dist-info → cdxcore-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdxcore
-Version: 0.1.10
+Version: 0.1.11
 Summary: Basic Python Tools; upgraded cdxbasics
 Author-email: Hans Buehler <github@buehler.london>
 License-Expression: MIT

{cdxcore-0.1.10.dist-info → cdxcore-0.1.11.dist-info}/RECORD

@@ -1,20 +1,21 @@
-cdxcore/__init__.py,sha256=DzMRN-6SAKCjFPYC72fGXBql03834TWP2QnjtPSb_vc,127
+cdxcore/__init__.py,sha256=QkwdykAwr_r64uUVCGkEd6qZ7xPa-uKA44rrlJGMtKI,127
 cdxcore/config.py,sha256=YnIEJVFtMZ5EHlwzaB2JsSkCPhWPvEw9QSpe6mof_V4,98472
-cdxcore/crman.py,sha256=jOw40Bh6PkmEiabA7OS9NsqF-9viam3CBiKzPwxVkVw,5693
+cdxcore/crman.py,sha256=83oKpuzNy98ObpmUeJg2McGRxaLk2AP5lN1yd1F9ueQ,5759
 cdxcore/err.py,sha256=SIJMBXKXYI_hiysv5iSPRy0w_BbxyTDPbNEs56Y94Rk,14541
-cdxcore/jcpool.py,sha256=OzoXWBJKWaDgJm1OeUj9ERz9skvwslGOwNq0wbKtcFM,17222
+cdxcore/jcpool.py,sha256=uQyOBmOz7EPBsXU96J6Zb2Y-YmA2GwfsRYj6NzFEPKc,27335
 cdxcore/pretty.py,sha256=iUpgUCwmI8yb5O-WZFJEk3QvNYcj_GIFHUgZ5lK8F2I,17082
 cdxcore/pretty.py_bak.py,sha256=JgWr5044HzCNGG0wKSAWlWiPRs7-bNzkwiKH0T3n0to,28658
-cdxcore/subdir.py,sha256=ceL-Od5NAlJX3f4mYgwki5EMzbs-LpG5M2cikC8eFIE,173868
+cdxcore/subdir.py,sha256=NScdtAG-Wrt0D7_FcpO62qzII4bS9ABXFerkILPg4uE,173864
 cdxcore/uniquehash.py,sha256=g-D8pqPIppSdRq5QfdE5aP3paZ-NkXWHfnn-uNB7fmg,50648
 cdxcore/util.py,sha256=0fp0EzeZvnje1Q7SUcgB_JtKpsYgGTfvlHVfq0mE_ug,31930
 cdxcore/verbose.py,sha256=nKNoZQwl3eF1zBf-JZwPC-lL9d_o5mJsDsSUMixTMLw,29882
 cdxcore/version.py,sha256=m30oI2Ortg44dKSim-sIoeh9PioD1FWsSfVEP5rubhk,27173
-cdxcore-0.1.10.dist-info/licenses/LICENSE,sha256=M-cisgK9kb1bqVRJ7vrCxHcMQQfDxdY3c2YFJJWfNQg,1090
+cdxcore-0.1.11.dist-info/licenses/LICENSE,sha256=M-cisgK9kb1bqVRJ7vrCxHcMQQfDxdY3c2YFJJWfNQg,1090
 docs/source/conf.py,sha256=Owctibh5XcSpSNcrpOr3ROIDjoklmFVrMhu8cOSe50o,4180
 tests/test_config.py,sha256=0U9vFIKDex0Il-7Vc_C4saAuXoHIsdQ8YhhS8AO7FQI,15950
-tests/test_crman.py,sha256=jYDxqF__iq3fEjaZQoq66CNChWRoR79Ntyng5mr3sIA,1698
+tests/test_crman.py,sha256=hek6a-51-i6o5XhDa1vqFjUKYJggJ3pnb0Am0wunhwY,1692
 tests/test_err.py,sha256=VbVmbaB6o49G-n3t7yuJ4M0d9pyUQyJuVDqK-xRrLo8,3458
+tests/test_jcpool.py,sha256=vgKt5wTz1BXt8ruubtE2qINIlqH507IoTC1lpP7nejQ,4733
 tests/test_pretty.py,sha256=5TmF7c1TRDSN-YR5yo04SiLJiW3bZaxpXHJ-4ZEO8hg,11952
 tests/test_subdir.py,sha256=tO-zoOIKQtZEMpQM-tsrisyLRmMH8txCSOzh6jPRhYY,11721
 tests/test_uniquehash.py,sha256=ldoQLT77R7odMAok4Yo3jmiUIH3VPHKoSiSLKbbM_mo,24907
@@ -29,7 +30,7 @@ tmp/npio.py,sha256=4Kwp5H4MgKHkOEhu4UJ5CcwpM7Pm8UFkaoL5FvOEFRI,10310
 tmp/sharedarray.py,sha256=JuHuSlxA0evD0a-bEZgTFrfdlVPMgzfQNgfSjr1212w,11484
 up/git_message.py,sha256=EfSH7Pit3ZoCiRqSMwRCUN_QyuwreU4LTIyGSutBlm4,123
 up/pip_modify_setup.py,sha256=Esaml4yA9tFsqxLhk5bWSwvKCURONjQqfyChgFV2TSY,1584
-cdxcore-0.1.10.dist-info/METADATA,sha256=CR4ZGaCL36hx08u3KnBdB5TWKM8Sknk2W_KsCptAv4g,754
-cdxcore-0.1.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cdxcore-0.1.10.dist-info/top_level.txt,sha256=phNSwCyJFe7UP2YMoi8o6ykhotatlIbJHjTp9EHM51k,26
-cdxcore-0.1.10.dist-info/RECORD,,
+cdxcore-0.1.11.dist-info/METADATA,sha256=q8JLhUt0fOG-sxNyd_8GJKg-DBhFwjBOVaEpo00bVbE,754
+cdxcore-0.1.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cdxcore-0.1.11.dist-info/top_level.txt,sha256=phNSwCyJFe7UP2YMoi8o6ykhotatlIbJHjTp9EHM51k,26
+cdxcore-0.1.11.dist-info/RECORD,,

tests/test_crman.py

@@ -49,6 +49,4 @@ class Test(unittest.TestCase):
         self.assertEqual( crman.current, "" )
             
 if __name__ == '__main__':
-    unittest.main()
-
-
+    unittest.main()

tests/test_jcpool.py

@@ -0,0 +1,119 @@
+
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr 14 21:24:52 2020
+@author: hansb
+"""
+
+import unittest as unittest
+
+def import_local():
+    """
+    In order to be able to run our tests manually from the 'tests' directory
+    we force import from the local package.
+    We also force reloading all modules to make sure we are not running old code.
+    """
+    me = "cdxcore"
+    import os
+    import sys
+    cwd = os.getcwd()
+    if cwd[-len(me):] == me:
+        return
+    assert cwd[-5:] == "tests",("Expected current working directory to be in a 'tests' directory", cwd[-5:], "from", cwd)
+    assert cwd[-6] in ['/', '\\'],("Expected current working directory 'tests' to be lead by a '\\' or '/'", cwd[-6:], "from", cwd)
+    sys.path.insert( 0, cwd[:-6] )
+
+    # reload modules
+    import importlib as imp
+    modules = sys.modules.copy()
+    for name, mdata in modules.items():
+        if name[:len(me)] == me:
+            imp.reload(mdata)
+            print("Reloaded", name)
+#import_local()
+
+from cdxcore.jcpool import JCPool, Context
+import numpy as np
+
+class Test(unittest.TestCase):
+
+    def test_pool(self):
+        
+        self.maxDiff = None
+        
+        pool    = JCPool(2)
+        
+        class Channel(object):
+            """ utility to collect all traced messages """
+            def __init__(self):
+                self.messages = []
+            def __call__(self, msg, flush):
+                self.messages.append( msg )
+        
+        def f( ticker, tdata, verbose : Context ):
+            # some made up results
+            q  = np.quantile( tdata, 0.35, axis=0 )
+            tx = q[0]
+            ty = q[1]
+            # not in a unittest --> time.sleep( np.exp(tdata[0,0]) )
+            verbose.write(f"Result for {ticker}: {tx:.2f}, {ty:.2f}")
+            return tx, ty
+        
+        np.random.seed(1231)
+        tickerdata =\
+         { 'SPY': np.random.normal(size=(1000,2)),
+           'GLD': np.random.normal(size=(1000,2)), 
+           'BTC': np.random.normal(size=(1000,2))
+         } 
+
+        # iterator mode        
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            for ticker, tx, ty in pool.parallel(
+                        { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                        for ticker, tdata in tickerdata.items() } ):
+                verbose.report(1,f"Returned {ticker} {tx:.2f}, {ty:.2f}")
+        verbose_main.write("Analysis done")
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 01:   Returned BTC -0.38, -0.42\n', '00: 01:   Returned GLD -0.47, -0.42\n', '00: 01:   Returned SPY -0.42, -0.41\n', '00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+
+        # dict mode
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            l = pool.parallel_to_dict(
+                        { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                          for ticker, tdata in tickerdata.items() } )
+        verbose_main.write("Analysis done")
+        self.assertEqual( type(l), dict )
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+
+        # list mode            
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            l = pool.parallel_to_list(
+                        pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                        for ticker, tdata in tickerdata.items() )
+        verbose_main.write("Analysis done")
+        self.assertEqual( type(l), list )
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+            
+if __name__ == '__main__':
+    unittest.main()
+
+
+
+