cdxcore 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

cdxcore/subdir.py

@@ -339,7 +339,6 @@ Documentation
 -------------
 """
 
-
 import os as os
 import uuid as uuid
 import threading as threading
@@ -355,11 +354,9 @@ from enum import Enum
 from functools import update_wrapper
         
 import json as json
-import jsonpickle as jsonpickle
-import jsonpickle.ext.numpy as jsonpickle_numpy
 import gzip as gzip
 import blosc as blosc
-
+import sys as sys
 from .err import verify, error, warn, fmt as txtfmt
 from .pretty import PrettyObject
 from .verbose import Context
@@ -372,7 +369,16 @@ from .uniquehash import unique_hash48, UniqueLabel, NamedUniqueHash
 :meta private:
 compression
 """
-jsonpickle_numpy.register_handlers()
+
+def _import_jsonpickle():
+    """ For some dodgy reason importing `jsonpickle` normally causes my tests to fail with a recursion error """
+    jsonpickle = sys.modules.get('jsonpickle', None)
+    if jsonpickle is None:
+        import jsonpickle as jsonpickle
+        import jsonpickle.ext.numpy as jsonpickle_numpy
+        jsonpickle_numpy.register_handlers()
+    return jsonpickle
+
 BLOSC_MAX_BLOCK = 2147483631
 BLOSC_MAX_USE   = 1147400000 # ... blosc really cannot handle large files
 #
@@ -1518,6 +1524,7 @@ class SubDir(object):
                             return ok
                         # read
                         if fmt == Format.JSON_PICKLE:
+                            jsonpickle = _import_jsonpickle()
                             return jsonpickle.decode( f.read() )
                         else:
                             assert fmt == Format.JSON_PLAIN, ("Internal error: unknown Format", fmt)
@@ -1779,7 +1786,7 @@ class SubDir(object):
         # write to temp file, then rename into target file
         # this reduces collision when i/o operations are slow
         full_file_name = self.full_file_name(file,ext=ext)
-        tmp_file     = unique_hash48( [ file, uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() ] )
+        tmp_file     = unique_hash48( file, uuid.getnode(), os.getpid(), threading.get_ident(), datetime.datetime.now() )
         tmp_i        = 0
         fullTmpFile  = self.full_file_name(tmp_file,ext="tmp" if not ext=="tmp" else "_tmp")
         while os.path.exists(fullTmpFile):
@@ -1935,6 +1942,7 @@ class SubDir(object):
                         if not version is None:
                             f.write("# " + version + "\n")
                         if fmt == Format.JSON_PICKLE:
+                            jsonpickle = _import_jsonpickle()
                             f.write( jsonpickle.encode(obj) )
                         else:
                             assert fmt == Format.JSON_PLAIN, ("Internal error: invalid Format", fmt)

cdxcore/version.py

@@ -498,7 +498,7 @@ class Version(object):
         """
         Compute hash for use with :class:`cdxcore.uniquehash.UniqueHash`.
         """
-        return self.unique_id(max_len=uniqueHash.length)
+        return self.unique_id(max_len=unique_hash.length)
     
 # =======================================================
 # @version

{cdxcore-0.1.9.dist-info → cdxcore-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdxcore
-Version: 0.1.9
+Version: 0.1.11
 Summary: Basic Python Tools; upgraded cdxbasics
 Author-email: Hans Buehler <github@buehler.london>
 License-Expression: MIT

cdxcore-0.1.11.dist-info/RECORD

@@ -0,0 +1,36 @@
+cdxcore/__init__.py,sha256=QkwdykAwr_r64uUVCGkEd6qZ7xPa-uKA44rrlJGMtKI,127
+cdxcore/config.py,sha256=YnIEJVFtMZ5EHlwzaB2JsSkCPhWPvEw9QSpe6mof_V4,98472
+cdxcore/crman.py,sha256=83oKpuzNy98ObpmUeJg2McGRxaLk2AP5lN1yd1F9ueQ,5759
+cdxcore/err.py,sha256=SIJMBXKXYI_hiysv5iSPRy0w_BbxyTDPbNEs56Y94Rk,14541
+cdxcore/jcpool.py,sha256=uQyOBmOz7EPBsXU96J6Zb2Y-YmA2GwfsRYj6NzFEPKc,27335
+cdxcore/pretty.py,sha256=iUpgUCwmI8yb5O-WZFJEk3QvNYcj_GIFHUgZ5lK8F2I,17082
+cdxcore/pretty.py_bak.py,sha256=JgWr5044HzCNGG0wKSAWlWiPRs7-bNzkwiKH0T3n0to,28658
+cdxcore/subdir.py,sha256=NScdtAG-Wrt0D7_FcpO62qzII4bS9ABXFerkILPg4uE,173864
+cdxcore/uniquehash.py,sha256=g-D8pqPIppSdRq5QfdE5aP3paZ-NkXWHfnn-uNB7fmg,50648
+cdxcore/util.py,sha256=0fp0EzeZvnje1Q7SUcgB_JtKpsYgGTfvlHVfq0mE_ug,31930
+cdxcore/verbose.py,sha256=nKNoZQwl3eF1zBf-JZwPC-lL9d_o5mJsDsSUMixTMLw,29882
+cdxcore/version.py,sha256=m30oI2Ortg44dKSim-sIoeh9PioD1FWsSfVEP5rubhk,27173
+cdxcore-0.1.11.dist-info/licenses/LICENSE,sha256=M-cisgK9kb1bqVRJ7vrCxHcMQQfDxdY3c2YFJJWfNQg,1090
+docs/source/conf.py,sha256=Owctibh5XcSpSNcrpOr3ROIDjoklmFVrMhu8cOSe50o,4180
+tests/test_config.py,sha256=0U9vFIKDex0Il-7Vc_C4saAuXoHIsdQ8YhhS8AO7FQI,15950
+tests/test_crman.py,sha256=hek6a-51-i6o5XhDa1vqFjUKYJggJ3pnb0Am0wunhwY,1692
+tests/test_err.py,sha256=VbVmbaB6o49G-n3t7yuJ4M0d9pyUQyJuVDqK-xRrLo8,3458
+tests/test_jcpool.py,sha256=vgKt5wTz1BXt8ruubtE2qINIlqH507IoTC1lpP7nejQ,4733
+tests/test_pretty.py,sha256=5TmF7c1TRDSN-YR5yo04SiLJiW3bZaxpXHJ-4ZEO8hg,11952
+tests/test_subdir.py,sha256=tO-zoOIKQtZEMpQM-tsrisyLRmMH8txCSOzh6jPRhYY,11721
+tests/test_uniquehash.py,sha256=ldoQLT77R7odMAok4Yo3jmiUIH3VPHKoSiSLKbbM_mo,24907
+tests/test_util.py,sha256=DZ6AlPFDNNlkqP5MlM1BUwmBJvEj4WCFqZcdh_Isflw,19955
+tests/test_verbose.py,sha256=7JGCLKHU1HovO6UYSLLcJQxjaZxYJejS1fl8O3Sgk9w,5037
+tests/test_version.py,sha256=eq7bNT0GefbUkwEzo658UUxgBCiR7CDhuIxAmnnI-qQ,4658
+tmp/deferred.py,sha256=TirvzxYXWnZkOWuEMB7qsf6auilfy4VD_zPknYCjrnw,9401
+tmp/dynaplot.py,sha256=kwrH_WccpJcfS7n_gzdAr4QxQobvIZZrrxgdsKLKfj0,48552
+tmp/filelock.py,sha256=HqnHZhSCESaOA3ClrdWPW_GZpyo7c3VRSEofAV-khKM,18137
+tmp/np.py,sha256=2MynhiaTfmx984Gz7TwfZH3t7GCmCAQiyeWzDDCL6_k,47686
+tmp/npio.py,sha256=4Kwp5H4MgKHkOEhu4UJ5CcwpM7Pm8UFkaoL5FvOEFRI,10310
+tmp/sharedarray.py,sha256=JuHuSlxA0evD0a-bEZgTFrfdlVPMgzfQNgfSjr1212w,11484
+up/git_message.py,sha256=EfSH7Pit3ZoCiRqSMwRCUN_QyuwreU4LTIyGSutBlm4,123
+up/pip_modify_setup.py,sha256=Esaml4yA9tFsqxLhk5bWSwvKCURONjQqfyChgFV2TSY,1584
+cdxcore-0.1.11.dist-info/METADATA,sha256=q8JLhUt0fOG-sxNyd_8GJKg-DBhFwjBOVaEpo00bVbE,754
+cdxcore-0.1.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cdxcore-0.1.11.dist-info/top_level.txt,sha256=phNSwCyJFe7UP2YMoi8o6ykhotatlIbJHjTp9EHM51k,26
+cdxcore-0.1.11.dist-info/RECORD,,

{cdxcore-0.1.9.dist-info → cdxcore-0.1.11.dist-info}/top_level.txt

@@ -1,6 +1,5 @@
 cdxcore
 docs
-docs2
 tests
 tmp
 up

tests/test_config.py

@@ -4,7 +4,6 @@ Created on Tue Apr 14 21:24:52 2020
 @author: hansb
 """
 
-
 import unittest as unittest
 import dataclasses as dataclasses
 import sys as sys
@@ -12,7 +11,6 @@ import os as os
 import pickle as pickle
 import tempfile as tempfile
 import shutil as shutil
-sys.setrecursionlimit(100)
 
 def import_local():
     """
@@ -37,7 +35,7 @@ def import_local():
         if name[:len(me)] == me:
             imp.reload(mdata)
             print("Reloaded", name)
-import_local()
+#import_local()
 
 from cdxcore.config import Config, Int, Float
 from cdxcore.pretty import PrettyObject as pdct

tests/test_crman.py

@@ -29,7 +29,7 @@ def import_local():
         if name[:len(me)] == me:
             imp.reload(mdata)
             print("Reloaded", name)
-import_local()
+#import_local()
 
 from cdxcore.crman import CRMan
 
@@ -49,6 +49,4 @@ class Test(unittest.TestCase):
         self.assertEqual( crman.current, "" )
             
 if __name__ == '__main__':
-    unittest.main()
-
-
+    unittest.main()

tests/test_err.py

@@ -30,7 +30,7 @@ def import_local():
         if name[:len(me)] == me:
             imp.reload(mdata)
             print("Reloaded", name)
-import_local()
+#import_local()
 
 from cdxcore.err import fmt, error, verify, warn, warn_if
 

tests/test_jcpool.py

@@ -0,0 +1,119 @@
+
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr 14 21:24:52 2020
+@author: hansb
+"""
+
+import unittest as unittest
+
+def import_local():
+    """
+    In order to be able to run our tests manually from the 'tests' directory
+    we force import from the local package.
+    We also force reloading all modules to make sure we are not running old code.
+    """
+    me = "cdxcore"
+    import os
+    import sys
+    cwd = os.getcwd()
+    if cwd[-len(me):] == me:
+        return
+    assert cwd[-5:] == "tests",("Expected current working directory to be in a 'tests' directory", cwd[-5:], "from", cwd)
+    assert cwd[-6] in ['/', '\\'],("Expected current working directory 'tests' to be lead by a '\\' or '/'", cwd[-6:], "from", cwd)
+    sys.path.insert( 0, cwd[:-6] )
+
+    # reload modules
+    import importlib as imp
+    modules = sys.modules.copy()
+    for name, mdata in modules.items():
+        if name[:len(me)] == me:
+            imp.reload(mdata)
+            print("Reloaded", name)
+#import_local()
+
+from cdxcore.jcpool import JCPool, Context
+import numpy as np
+
+class Test(unittest.TestCase):
+
+    def test_pool(self):
+        
+        self.maxDiff = None
+        
+        pool    = JCPool(2)
+        
+        class Channel(object):
+            """ utility to collect all traced messages """
+            def __init__(self):
+                self.messages = []
+            def __call__(self, msg, flush):
+                self.messages.append( msg )
+        
+        def f( ticker, tdata, verbose : Context ):
+            # some made up results
+            q  = np.quantile( tdata, 0.35, axis=0 )
+            tx = q[0]
+            ty = q[1]
+            # not in a unittest --> time.sleep( np.exp(tdata[0,0]) )
+            verbose.write(f"Result for {ticker}: {tx:.2f}, {ty:.2f}")
+            return tx, ty
+        
+        np.random.seed(1231)
+        tickerdata =\
+         { 'SPY': np.random.normal(size=(1000,2)),
+           'GLD': np.random.normal(size=(1000,2)), 
+           'BTC': np.random.normal(size=(1000,2))
+         } 
+
+        # iterator mode        
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            for ticker, tx, ty in pool.parallel(
+                        { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                        for ticker, tdata in tickerdata.items() } ):
+                verbose.report(1,f"Returned {ticker} {tx:.2f}, {ty:.2f}")
+        verbose_main.write("Analysis done")
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 01:   Returned BTC -0.38, -0.42\n', '00: 01:   Returned GLD -0.47, -0.42\n', '00: 01:   Returned SPY -0.42, -0.41\n', '00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+
+        # dict mode
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            l = pool.parallel_to_dict(
+                        { ticker: pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                          for ticker, tdata in tickerdata.items() } )
+        verbose_main.write("Analysis done")
+        self.assertEqual( type(l), dict )
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+
+        # list mode            
+        channel      = Channel()
+        verbose_main = Context("all", channel=channel)
+        
+        verbose_main.write("Launching analysis")
+        with pool.context( verbose_main ) as verbose:
+            l = pool.parallel_to_list(
+                        pool.delayed(f)( ticker=ticker, tdata=tdata, verbose=verbose(2) )
+                        for ticker, tdata in tickerdata.items() )
+        verbose_main.write("Analysis done")
+        self.assertEqual( type(l), list )
+
+        l = sorted( channel.messages )
+        self.assertEqual( str(l), r"['00: 02:     Result for BTC: -0.38, -0.42\n', '00: 02:     Result for GLD: -0.47, -0.42\n', '00: 02:     Result for SPY: -0.42, -0.41\n', '00: Analysis done\n', '00: Launching analysis\n']")
+            
+if __name__ == '__main__':
+    unittest.main()
+
+
+
+

tests/test_pretty.py

@@ -37,7 +37,7 @@ def import_local():
         if name[:len(me)] == me:
             imp.reload(mdata)
             print("Reloaded", name)
-import_local()
+#import_local()
 
 from cdxcore.pretty import PrettyObject, Sequence
 

tests/test_subdir.py

@@ -28,7 +28,7 @@ def import_local():
     for name, mdata in modules.items():
         if name[:len(me)] == me:
             imp.reload(mdata)
-import_local()
+#import_local()
     
 """
 Imports

tests/test_uniquehash.py

@@ -32,7 +32,7 @@ def import_local():
     for name, mdata in modules.items():
         if name[:len(me)] == me:
             imp.reload(mdata)
-import_local()
+#import_local()
     
 """
 Imports

tests/test_util.py

@@ -31,7 +31,7 @@ def import_local():
     for name, mdata in modules.items():
         if name[:len(me)] == me:
             imp.reload(mdata)
-import_local()
+#import_local()
 
 from cdxcore.util import is_function, is_atomic, is_float, is_filename
 from cdxcore.util import fmt, fmt_seconds, fmt_list, fmt_dict, fmt_big_number, fmt_digits, fmt_big_byte_number, fmt_datetime, fmt_date, fmt_time, fmt_timedelta, fmt_filename, DEF_FILE_NAME_MAP

tests/test_verbose.py

@@ -29,7 +29,7 @@ def import_local():
         if name[:len(me)] == me:
             imp.reload(mdata)
             print("Reloaded", name)
-import_local()
+#import_local()
 
 from cdxcore.verbose import Context
 from cdxcore.uniquehash import unique_hash32 as unique_hash

tests/test_version.py

@@ -28,7 +28,7 @@ def import_local():
     for name, mdata in modules.items():
         if name[:len(me)] == me:
             imp.reload(mdata)
-import_local()
+#import_local()
     
 """
 Imports

cdxcore-0.1.9.dist-info/RECORD

@@ -1,36 +0,0 @@
-cdxcore/__init__.py,sha256=a9wRC3XcpvMapllTNzJzr0MMUqBnYzI_lnm18xrxjj0,126
-cdxcore/config.py,sha256=YnIEJVFtMZ5EHlwzaB2JsSkCPhWPvEw9QSpe6mof_V4,98472
-cdxcore/crman.py,sha256=jOw40Bh6PkmEiabA7OS9NsqF-9viam3CBiKzPwxVkVw,5693
-cdxcore/err.py,sha256=SIJMBXKXYI_hiysv5iSPRy0w_BbxyTDPbNEs56Y94Rk,14541
-cdxcore/pretty.py,sha256=iUpgUCwmI8yb5O-WZFJEk3QvNYcj_GIFHUgZ5lK8F2I,17082
-cdxcore/pretty.py_bak.py,sha256=JgWr5044HzCNGG0wKSAWlWiPRs7-bNzkwiKH0T3n0to,28658
-cdxcore/subdir.py,sha256=bVABx_XpzZxmjoUQe0Brt2cEakhUONP1YA4tjPQ_LTk,173453
-cdxcore/uniquehash.py,sha256=g-D8pqPIppSdRq5QfdE5aP3paZ-NkXWHfnn-uNB7fmg,50648
-cdxcore/util.py,sha256=0fp0EzeZvnje1Q7SUcgB_JtKpsYgGTfvlHVfq0mE_ug,31930
-cdxcore/verbose.py,sha256=nKNoZQwl3eF1zBf-JZwPC-lL9d_o5mJsDsSUMixTMLw,29882
-cdxcore/version.py,sha256=tsLHRIgYYqrcqGJqz-bMEJuixWtTg1AQB_QwJrf6VDE,27172
-cdxcore-0.1.9.dist-info/licenses/LICENSE,sha256=M-cisgK9kb1bqVRJ7vrCxHcMQQfDxdY3c2YFJJWfNQg,1090
-docs/source/conf.py,sha256=Owctibh5XcSpSNcrpOr3ROIDjoklmFVrMhu8cOSe50o,4180
-docs2/source/conf.py,sha256=qxnJSrm3fV-llfvYlM0adpVD5pE_-8f8KqxH_ywe5UA,1276
-tests/test_config.py,sha256=JtIWCt0Q2PmmADfOGL42anONvuNowtxLK312xvWmRD4,15979
-tests/test_crman.py,sha256=hJlhuUuDrhzzJ3-ILUu_JCH842BJx9VGpEBCGrr4LOU,1697
-tests/test_err.py,sha256=AUfUUnksdbcHEuY-boyTW7fJHJ_w13zYpaPYrqwWbSo,3457
-tests/test_pretty.py,sha256=AJOI4jFnol_pY_wzF9baPNzx_J5pViVzZcRkWMDU4HY,11951
-tests/test_subdir.py,sha256=heLvzVl3qpgLq_rj4BY5odn6ZWk978kum1SznKQ4JMw,11720
-tests/test_uniquehash.py,sha256=kxmH5slL4NxmcCS2OOdbzqu9G2C3sm2SR4HkfIyromU,24906
-tests/test_util.py,sha256=MnM3EFp4p8MGw1qnhtBryHikHOt__zfOm65OMzN0zWE,19954
-tests/test_verbose.py,sha256=0EOVK5TWrwQwAjv08enaOTQZ6-_SQFMKXi3Nw6nSfMI,5036
-tests/test_version.py,sha256=xPF6eXD7J9VB_L1FGK5kgjXkrfD9gqHdj03zc9s6XIs,4657
-tmp/deferred.py,sha256=TirvzxYXWnZkOWuEMB7qsf6auilfy4VD_zPknYCjrnw,9401
-tmp/dynaplot.py,sha256=kwrH_WccpJcfS7n_gzdAr4QxQobvIZZrrxgdsKLKfj0,48552
-tmp/filelock.py,sha256=HqnHZhSCESaOA3ClrdWPW_GZpyo7c3VRSEofAV-khKM,18137
-tmp/jcpool.py,sha256=OzoXWBJKWaDgJm1OeUj9ERz9skvwslGOwNq0wbKtcFM,17222
-tmp/np.py,sha256=2MynhiaTfmx984Gz7TwfZH3t7GCmCAQiyeWzDDCL6_k,47686
-tmp/npio.py,sha256=4Kwp5H4MgKHkOEhu4UJ5CcwpM7Pm8UFkaoL5FvOEFRI,10310
-tmp/sharedarray.py,sha256=JuHuSlxA0evD0a-bEZgTFrfdlVPMgzfQNgfSjr1212w,11484
-up/git_message.py,sha256=EfSH7Pit3ZoCiRqSMwRCUN_QyuwreU4LTIyGSutBlm4,123
-up/pip_modify_setup.py,sha256=Esaml4yA9tFsqxLhk5bWSwvKCURONjQqfyChgFV2TSY,1584
-cdxcore-0.1.9.dist-info/METADATA,sha256=Bz8HlL06uVWBnx7Ie2ogBTgyk_RffCMbFC5QBt3MCNo,753
-cdxcore-0.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cdxcore-0.1.9.dist-info/top_level.txt,sha256=eG4oLGRzNN40tCfK5tJbzfT_I2M-9IPGVpbRwFWE9Wk,32
-cdxcore-0.1.9.dist-info/RECORD,,

docs2/source/conf.py

@@ -1,35 +0,0 @@
-# conf.py
-def set_path(source = "cdxcore"):
-    import os, sys
-    root_path = os.path.split(
-                os.path.split(  
-                  os.path.split( __file__ )[0] # 'source
-                  )[0] # 'docs'
-                )[0] # 'packag
-    assert root_path[-len(source):] == source, f"Conf.py '{__file__}': invalid source path '{root_path}'. Call 'make html' from the docs directory"
-    sys.path.insert(0, root_path)  # so your package is importable
-    
-
-project = "cdxcore"
-author = "Hans Buehler"
-release = "0.1.0"
-
-set_path(project)
-
-extensions = [
-    "sphinx.ext.autodoc",           # core: import & docstrings
-    "sphinx.ext.autosummary",       # summary tables with links
-    "sphinx.ext.napoleon",          # Google/NumPy style docstrings
-    "sphinx_autodoc_typehints",     # nicer type hints formatting
-    "sphinx.ext.mathjax",           # math rendering
-]
-
-autosummary_generate = True         # make _autosummary files
-autodoc_typehints = "description"   # show types in doc body instead of signature
-autodoc_member_order = "bysource"   # preserve order in source file
-napoleon_google_docstring = True
-napoleon_numpy_docstring = True
-napoleon_use_param = True
-napoleon_use_rtype = False
-autoclass_content = "both"
-