datastock 0.0.33__py3-none-any.whl → 0.0.34__py3-none-any.whl

datastock/_generic_utils.py

@@ -1,8 +1,14 @@
 
 
+import itertools as itt
+from copy import deepcopy
+
+
 import numpy as np
 import scipy.sparse as scpsp
 import astropy.units as asunits
+from functools import reduce  # forward compatibility for Python 3
+import operator
 
 
 from . import _generic_check
@@ -407,8 +413,14 @@ def compare_dict(
     return _compare_dict_verb_return(dkeys, returnas, verb)
 
 
-def compare_obj(obj0=None, obj1=None, returnas=None, verb=None):
-    """ Compare thje content of 2 instances """
+def compare_obj(
+    obj0=None,
+    obj1=None,
+    excluded=None,
+    returnas=None,
+    verb=None,
+):
+    """ Compare the content of 2 instances """
 
     # -----------
     # Check class
@@ -423,8 +435,8 @@ def compare_obj(obj0=None, obj1=None, returnas=None, verb=None):
     # Check
 
     return compare_dict(
-        d0=obj0.to_dict(),
-        d1=obj1.to_dict(),
+        d0=obj0.to_dict(excluded=excluded, returnas='values', copy=False),
+        d1=obj1.to_dict(excluded=excluded, returnas='values', copy=False),
         dname=None,
         returnas=returnas,
         verb=verb,
@@ -437,34 +449,255 @@ def compare_obj(obj0=None, obj1=None, returnas=None, verb=None):
 # ###############################################################
 
 
-def flatten_dict(
-    din=None,
-    parent_key=None,
+def to_dict(
+    coll=None,
+    flatten=None,
     sep=None,
+    excluded=None,
+    # copy vs ref
     asarray=None,
-    with_types=None,
-    type_str=None,
+    copy=None,
+    # dtypes
+    returnas=None,
 ):
-    """ Return a flattened version of the input dict """
 
     # ------------
     # check inputs
 
-    # sep
-    sep = _generic_check._check_var(
-        sep, 'sep',
-        default='.',
+    flatten = _generic_check._check_var(
+        flatten, 'flatten',
+        default=True,
+        types=bool,
+    )
+
+    returnas = _generic_check._check_var(
+        returnas, 'returnas',
+        default='types',
         types=str,
+        allowed=['types', 'values', 'both', 'blended'],
+    )
+
+    # ----------------------
+    # get flat key/type tree
+
+    dtypes, sep = flatten_dict_keys(
+        din=coll,
+        parent_key=None,
+        sep=sep,
+        excluded=excluded,
+    )
+
+    if returnas == 'types':
+        if flatten is False:
+            return reshape_dict(dtypes, sep=sep)
+        else:
+            return dtypes
+
+    # ---------------------------
+    # Get list of dict attributes
+
+    dout = dict_from_dtypes(
+        coll,
+        dtypes=dtypes,
+        flatten=flatten,
+        sep=sep,
+        asarray=asarray,
+        copy=copy,
     )
 
+    # ---------
+    # return
+
+    if returnas == 'blended':
+        return _blend_dicts(
+            dbase=dout,
+            dextra=dtypes,
+            extra_key='__type',
+            flatten=flatten,
+        )
+    if returnas == 'both':
+        return dtypes, dout
+    else:
+        return dout
+
+
+def _flatten_dict_check(
+    din=None,
+    parent_key=None,
+    sep=None,
+    excluded=None,
+):
+    # ------------
+    # check inputs
+
+    # sep
+    if sep is not None:
+        sep = _generic_check._check_var(
+            sep, 'sep',
+            default='.',
+            types=str,
+        )
+
     # parent_key
     if parent_key is not None:
         parent_key = _generic_check._check_var(
             parent_key, 'parent_key',
-            default='.',
-            types=str,
+            default=('',),
+            types=(str, tuple),
         )
 
+    # excluded
+    if excluded is not None:
+        if isinstance(excluded, str):
+            excluded = ((excluded,),)
+
+        if not isinstance(excluded, (list, tuple)):
+            msg = "Arg excluded must be a tuple of tuples of str!"
+            raise Exception(msg)
+
+        if any([isinstance(ss, (str, list)) for ss in excluded]):
+            excluded = tuple([
+                tuple(ss) if isinstance(ss, list)
+                else (ss if isinstance(ss, tuple) else (ss,))
+                for ss in excluded
+            ])
+
+        c0 = (
+            isinstance(excluded, tuple)
+            and all([isinstance(tt, tuple) for tt in excluded])
+            and all([all([isinstance(ss, str) for ss in tt]) for tt in excluded])
+        )
+        if not c0:
+            msg = "Arg excluded must be a tuple of tuples of str!"
+            raise Exception(msg)
+
+    return parent_key, sep, excluded
+
+
+def flatten_dict_keys(
+    din=None,
+    parent_key=None,
+    sep=None,
+    excluded=None,
+):
+    """ Return a flattened version of the input dict keys"""
+
+    # ------------
+    # check inputs
+    # ------------
+
+    parent_key, sep, excluded = _flatten_dict_check(
+        din=din,
+        parent_key=parent_key,
+        sep=sep,
+        excluded=excluded,
+    )
+
+    # ------------
+    # top level
+    # ------------
+
+    if isinstance(din, dict):
+
+        dkeys = {}
+        for k0, v0 in din.items():
+
+            # key
+            if parent_key is None:
+                key = (k0,)
+            else:
+                key = tuple([k1 for k1 in parent_key] + [k0])
+
+            # value
+            if isinstance(v0, dict):
+                dkeys.update(
+                    flatten_dict_keys(
+                        v0,
+                        key,
+                        sep=None,
+                        excluded=excluded,
+                    )[0]
+                )
+
+            else:
+
+                # get class
+                if excluded is None or key not in excluded:
+                    dkeys[key] = v0.__class__.__name__
+
+    else:
+        dkeys = {}
+        lk0 = [
+            k0 for k0 in dir(din)
+            if k0 != '__dict__'
+            and '__dlinks' not in k0
+            and not (
+                hasattr(din.__class__, k0)
+                and isinstance(getattr(din.__class__, k0), property)
+            )
+            and isinstance(getattr(din, k0), dict)
+        ]
+        for k0 in lk0:
+            dkeys.update(flatten_dict_keys(
+                getattr(din, k0),
+                parent_key=(k0,),
+                sep=None,
+                excluded=excluded,
+            )[0])
+
+    # ---------------------
+    # format keys using sep
+    # ---------------------
+
+    if sep is not None:
+
+        # --------------------
+        # safety check vs sep
+
+        # dict of non-conform keys
+        dkout = {
+            k0: v0 for k0, v0 in dkeys.items()
+            if any([sep in k1 for k1 in k0])
+        }
+
+        # error msg
+        if len(dkout) > 0:
+            lstr = [f"\t- {k0}: {v0}" for k0, v0 in dkout.items()]
+            msg = (
+                f"The following keys already have the desired sep '{sep}':\n"
+                + '\n'.join(lstr)
+            )
+            raise Exception(msg)
+
+        # ----------
+        # formatting
+
+        dkeys = {sep.join(k0): v0 for k0, v0 in dkeys.items()}
+
+    return dkeys, sep
+
+
+def getFromDict(dataDict, mapList):
+    return reduce(operator.getitem, mapList, dataDict)
+
+
+def setInDict(dataDict, mapList, value):
+    getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value
+
+
+def dict_from_dtypes(
+    coll=None,
+    dtypes=None,
+    flatten=None,
+    sep=None,
+    asarray=None,
+    copy=None,
+):
+    """ Assumes dtypes is flat """
+
+    # -------------------------
+    # check inputs
+
     # asarray
     asarray = _generic_check._check_var(
         asarray, 'asarray',
@@ -472,99 +705,120 @@ def flatten_dict(
         types=bool,
     )
 
-    # with_types
-    with_types = _generic_check._check_var(
-        with_types, 'with_types',
+    copy = _generic_check._check_var(
+        copy, 'copy',
         default=False,
         types=bool,
     )
 
-    # with_types
-    type_str = _generic_check._check_var(
-        type_str, 'type_str',
-        default='__type',
-        types=str,
-    )
+    # -------------------------
+    # get flat / unflat version
 
-    # --------
-    # flatten
+    # keys
+    if sep is None:
+        lkeys = sorted(dtypes.keys())
+    else:
+        lkeys = sorted([k0.split(sep) for k0 in dtypes.keys()])
 
-    if with_types:
-        dtypes = {}
+    # -----------
+    # build
 
-    # --------
-    # flatten
+    # initialize dout
+    if flatten is True:
+        dout = {}
+    else:
+        dout = dict(reshape_dict(dtypes, sep=sep))
 
-    items = []
-    for k0, v0 in din.items():
+    # loop on all keys from (flat) dtypes
+    for k0 in lkeys:
 
-        # key
-        if parent_key is None:
-            key = k0
-        else:
-            key = f'{parent_key}{sep}{k0}'
-
-        # value
-        if isinstance(v0, dict):
-            items.extend(
-                flatten_dict(
-                    v0,
-                    key,
-                    sep=sep,
-                    asarray=asarray,
-                    with_types=with_types,
-                    type_str=type_str,
-                ).items()
-            )
+        # get value from coll
+        for ii, k1 in enumerate(k0):
+            if ii == 0:
+                out = getattr(coll, k1)
+            else:
+                out = out[k1]
+
+        # asarray
+        if asarray is True:
+            out = np.asarray(out)
 
+        # set value in dout
+        if flatten is True:
+            if sep is None:
+                dout[k0] = out
+            else:
+                dout[sep.join(k0)] = out
         else:
+            setInDict(dout, k0, out)
+
+    # ---------------
+    # prepare output
 
-            # get class
-            cc = v0.__class__.__name__
+    if copy is True:
+        return deepcopy(dout)
+    else:
+        return dout
 
-            # store type?
-            if with_types:
-                dtypes[f'{key}{type_str}'] = cc
 
-            # asarray
-            if asarray:
+def _blend_dicts(
+    dbase=None,
+    dextra=None,
+    extra_key=None,
+    flatten=None,
+):
 
-                # astropy unit to str
-                if 'Unit' in cc:
-                    v0 = str(v0)
+    # -------------
+    # check inputs
 
-                items.append((key, np.asarray(v0)))
+    extra_key = _generic_check._check_var(
+        extra_key, 'extra_key',
+        default='__type',
+        types=str,
+    )
 
-            else:
-                items.append((key, v0))
+    assert isinstance(dbase, dict) and isinstance(dextra, dict)
 
-    # --------
-    # return
+    # ------------
+    # blend
+
+    if flatten is True:
+
+        for k0, v0 in dextra.items():
+            dbase[f"{k0}{extra_key}"] = v0
 
-    if with_types:
-        dout = dict(items)
-        dout.update(dtypes)
-        return dout
     else:
-        return dict(items)
+        msg = "Blended dict not implement for flatten = False"
+        raise NotImplementedError(msg)
+
+    return dbase
 
 
 def _reshape_dict(k0, v0, dinit={}, sep=None):
     """ Populate dinit """
 
-    lk = k0.split(sep)
-    k0 = k0 if len(lk) == 1 else lk[0]
+    assert isinstance(dinit, dict), dinit
+
+    if sep is None:
+        lk = k0
+    else:
+        lk = k0.split(sep)
+
+    k0 = lk[0]
 
     if len(lk) == 2:
         if k0 not in dinit.keys():
             dinit[k0] = {}
-        assert isinstance(dinit[k0], dict)
+        assert isinstance(dinit[k0], dict), (k0, dinit[k0])
         dinit[k0].update({lk[1]: v0})
 
     elif len(lk) > 2:
         if k0 not in dinit.keys():
             dinit[k0] = {}
-        _reshape_dict(sep.join(lk[1:]), v0, dinit=dinit[k0], sep=sep)
+        assert isinstance(dinit[k0], dict), (k0, dinit[k0])
+
+        knew = lk[1:] if sep is None else sep.join(lk[1:])
+        _reshape_dict(knew, v0, dinit=dinit[k0], sep=sep)
 
     else:
         assert k0 not in dinit.keys()
@@ -577,18 +831,20 @@ def reshape_dict(din, sep=None):
     # ------------
     # check inputs
 
-    sep = _generic_check._check_var(
-        sep, 'sep',
-        default='.',
-        types=str,
-    )
+    if sep is not None:
+        sep = _generic_check._check_var(
+            sep, 'sep',
+            default='.',
+            types=str,
+        )
 
-    # ------------
+    # ------------------------
     # Get all individual keys
 
     dout = {}
     for k0, v0 in din.items():
         _reshape_dict(k0, v0, dinit=dout, sep=sep)
+
     return dout
 
 
@@ -601,7 +857,7 @@ def reshape_dict(din, sep=None):
 def KnuthMorrisPratt(text, pattern):
 
     """ Yields all starting positions of copies of the pattern in the sequence
-    
+
     Calling conventions are similar to string.find, but its arguments can be
     lists or iterators, not just strings, it returns all matches, not just
     the first one, and it does not need the whole text in memory at once.

datastock/_saveload.py

@@ -15,6 +15,9 @@ from . import _generic_check
 from . import _generic_utils
 
 
+_KEY_SEP = '--sep--'
+
+
 # #################################################################
 # #################################################################
 #                   Save
@@ -23,6 +26,7 @@ from . import _generic_utils
 
 def save(
     dflat=None,
+    sep=None,
     name=None,
     path=None,
     clsname=None,
@@ -80,6 +84,9 @@ def save(
     dt = dtm.datetime.now().strftime("%Y%m%d-%H%M%S")
     name = f'{clsname}_{name}_{user}_{dt}.npz'
 
+    # add sep
+    dflat[_KEY_SEP] = sep
+
     # save
     pfe = os.path.join(path, name)
     np.savez(pfe,  **dflat)
@@ -147,6 +154,17 @@ def load(
 
     dflat = dict(np.load(pfe, allow_pickle=allow_pickle))
 
+    # ------------------------------
+    # load sep from file if exists
+
+    if _KEY_SEP in dflat.keys():
+        # new
+        sep = str(dflat[_KEY_SEP])
+        del dflat[_KEY_SEP]
+    else:
+        # back-compatibility
+        sep = [k0 for k0 in dflat.keys() if k0.startswith('_dref')][0][5]
+
     # ----------
     # reshape
 
@@ -181,13 +199,19 @@ def load(
             dout[k0] = None
         elif typ == 'ndarray':
             dout[k0] = dflat[k0]
+        elif any([ss in typ for ss in ['csc_', 'bsr_', 'coo_', 'csr_', 'dia_', 'dok_', 'lil_']]):
+            assert typ in type(dflat[k0]).__name__
+            dout[k0] = dflat[k0]
         elif 'Unit' in typ:
             dout[k0] = asunits.Unit(v0.tolist())
         elif typ == 'type':
             dout[k0] = dflat[k0]
         else:
             msg = (
-                f"Don't know how to deal with dflat['{k0}']: {typ}"
+                f"Don't know how to deal with dflat['{k0}']:\n"
+                f"\t- typ: {typ}\n"
+                f"\t- type: {type(dflat[k0])}\n"
+                f"\t- value: {dflat[k0]}\n"
             )
             raise Exception(msg)
 

datastock/version.py

@@ -1,2 +1,2 @@
 # Do not edit, pipeline versioning governed by git tags!
-__version__ = '0.0.33'
+__version__ = '0.0.34'

{datastock-0.0.33.dist-info → datastock-0.0.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datastock
-Version: 0.0.33
+Version: 0.0.34
 Summary: A python library for generic class and data handling
 Home-page: https://github.com/ToFuProject/datastock
 Author: Didier VEZINET

{datastock-0.0.33.dist-info → datastock-0.0.34.dist-info}/RECORD

@@ -2,9 +2,9 @@ _updateversion.py,sha256=OR6OJJozaHWzu7NWjKu5ERi0IyYqR61KrWvzf7kfoto,951
 datastock/_DataCollection_utils.py,sha256=hHf6HvGKMmM-psx3fj9QcY1TEmKrAtTdkRokH7SFqoo,7143
 datastock/__init__.py,sha256=MAUJgQG1Mz4Jq9Eu70T9phcV83iFjzLC1qjlqJZDbq4,183
 datastock/_class.py,sha256=Az9PS3aSskiPMb1ekt78Y2ynBujYVc_cDjJxW9xH9g4,47
-datastock/_class0.py,sha256=oV6Ar7in66OnjsDK7RRhe_OSt-ifBKul-x2NbucjF98,5442
+datastock/_class0.py,sha256=QULjNJke13jJrGLIeM7SWHZVziorDK_KCIlqq8LgS9U,5883
 datastock/_class1.py,sha256=l4W0atz_Q-ykO45eONzDtXkwCkcPKwuf2EuD9Ztwak4,29435
-datastock/_class1_binning.py,sha256=G1rbejlwoDRYANI_LY4QOKWA-BEikddVM3-EklROoZQ,28336
+datastock/_class1_binning.py,sha256=LWHv2LIfgZfSFWYwqdcN0DKpNe6q7Go3sxfcJqmzTrI,28085
 datastock/_class1_check.py,sha256=QN0o5Z9XOZAUIoe-dXTVNmbkj3l_v7U4lawcn8N0Yrw,52436
 datastock/_class1_compute.py,sha256=9ka4MXqTb2Gwe8ZnZR31rd5QDc7-nxuhFqXpQSn_sss,28211
 datastock/_class1_domain.py,sha256=TudSRoLedzU2qGOU6R1_Lx99MuqJzG0rOZl8-HZZfTU,6252
@@ -17,7 +17,7 @@ datastock/_direct_calls.py,sha256=EHFwI2mGMDqGz8_Bv2BseMBX4J8dSdE_RcNX3pt0ZYY,18
 datastock/_export_dataframe.py,sha256=fy-uJR3EhDlHvd9ls1EQna_C8fyha1jCJLu1DTKTkdo,1576
 datastock/_find_plateau.py,sha256=sqnAuy0361DXkqBb_Lo1MmIGjn35tnKFvcv6MW6hifs,2685
 datastock/_generic_check.py,sha256=nfD4PIInb63uiwioknFf78dE5lGpgSxoOvT5BgP2nts,24443
-datastock/_generic_utils.py,sha256=iSZa_XOhOJW1Rtr5kH2dxcs_2WaAxcuFls_eEKgOOTw,15382
+datastock/_generic_utils.py,sha256=1RTAcMHVTmBRyz0kvpIBemV-ULxluxA01zMssuafZdk,21243
 datastock/_plot_BvsA_as_distribution.py,sha256=fpRhlbv3Bk96buANC46Brc9hdLxkOAsoKpE5A9pohG0,15389
 datastock/_plot_BvsA_as_distribution_check.py,sha256=7OkzsxqXvJBe6uc_7IiZNwMYWryyI1vKUYYRCkPfz7Q,13002
 datastock/_plot_as_array.py,sha256=s3Jl0B_1gJ-SewSGfyjXrtzMG1utvcq-Vyb0Vmn-rzI,75861
@@ -26,13 +26,13 @@ datastock/_plot_as_profile1d.py,sha256=rxMpLL-QMHqHiAYjH7gDXZOUkMVz3ps8SlUUVNXK8
 datastock/_plot_correlations.py,sha256=ITOypu_AEoKl0ihxocV-JVTXIHqut6p9TfG-xZmQysc,10175
 datastock/_plot_misc.py,sha256=XixTi2CiihKjtQP0TRycH0b25caWN1m35DgpsDeiWZE,21729
 datastock/_plot_text.py,sha256=wQPqjfpLyIioS2JeOt3E9C9HgYUJ49YEoOgRuKYvAR8,3143
-datastock/_saveload.py,sha256=Oi42AYgF_KOCKaXgQL8HSh1zWUYN_RQ-un970dSNL1c,10981
-datastock/version.py,sha256=erWDbcyWQZO8LL8XfxpUCuIDY3AzE8WGLJTibLSQ9BU,80
+datastock/_saveload.py,sha256=Ycu8-OXeak4gqPOJMCE_oeBSZRqdxsqSJs84ZMK4UQg,11663
+datastock/version.py,sha256=ZPzh1gh33VoRYnkFgdCZ0tQKhQLqL6wrNOmoy2vEm0M,80
 datastock/tests/__init__.py,sha256=teOo2xP0IO7PQMuMDmum61XVHe2TuxW3BiHiL73X8jQ,35
 datastock/tests/test_01_DataStock.py,sha256=rIDlqIGGR6gCFtjAw66G9CF3-Mne95XF0eXtUU7wYfc,15260
 datastock/tests/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datastock-0.0.33.dist-info/LICENSE,sha256=V1uXqi3vxR0QhB4QdFyjkynl6jpN4wZmlB5EMYJk0NM,1068
-datastock-0.0.33.dist-info/METADATA,sha256=u5e7ReycLQE31ItS-ZsBdbgMeZaf5rfsmJxOQ40_FgE,8666
-datastock-0.0.33.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datastock-0.0.33.dist-info/top_level.txt,sha256=BzJsLLK_zZw13WQCoMhC74qWVKalnVCjBxdPXvJn7HQ,25
-datastock-0.0.33.dist-info/RECORD,,
+datastock-0.0.34.dist-info/LICENSE,sha256=V1uXqi3vxR0QhB4QdFyjkynl6jpN4wZmlB5EMYJk0NM,1068
+datastock-0.0.34.dist-info/METADATA,sha256=tFeEmOoG0f9tFVraH_cu38XphCD8rcBL7ufMj5JCBls,8666
+datastock-0.0.34.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datastock-0.0.34.dist-info/top_level.txt,sha256=BzJsLLK_zZw13WQCoMhC74qWVKalnVCjBxdPXvJn7HQ,25
+datastock-0.0.34.dist-info/RECORD,,