datastock 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl

datastock/_saveload.py

@@ -3,6 +3,8 @@
 import os
 import getpass
 import datetime as dtm
+import itertools as itt
+import warnings
 
 
 import numpy as np
@@ -13,6 +15,9 @@ from . import _generic_check
 from . import _generic_utils
 
 
+_KEY_SEP = '--sep--'
+
+
 # #################################################################
 # #################################################################
 #                   Save
@@ -21,6 +26,7 @@ from . import _generic_utils
 
 def save(
     dflat=None,
+    sep=None,
     name=None,
     path=None,
     clsname=None,
@@ -78,6 +84,9 @@ def save(
     dt = dtm.datetime.now().strftime("%Y%m%d-%H%M%S")
     name = f'{clsname}_{name}_{user}_{dt}.npz'
 
+    # add sep
+    dflat[_KEY_SEP] = sep
+
     # save
     pfe = os.path.join(path, name)
     np.savez(pfe,  **dflat)
@@ -145,6 +154,17 @@ def load(
 
     dflat = dict(np.load(pfe, allow_pickle=allow_pickle))
 
+    # ------------------------------
+    # load sep from file if exists
+
+    if _KEY_SEP in dflat.keys():
+        # new
+        sep = str(dflat[_KEY_SEP])
+        del dflat[_KEY_SEP]
+    else:
+        # back-compatibility
+        sep = [k0 for k0 in dflat.keys() if k0.startswith('_dref')][0][5]
+
     # ----------
     # reshape
 
@@ -179,13 +199,19 @@ def load(
             dout[k0] = None
         elif typ == 'ndarray':
             dout[k0] = dflat[k0]
+        elif any([ss in typ for ss in ['csc_', 'bsr_', 'coo_', 'csr_', 'dia_', 'dok_', 'lil_']]):
+            assert typ in type(dflat[k0]).__name__
+            dout[k0] = dflat[k0]
         elif 'Unit' in typ:
             dout[k0] = asunits.Unit(v0.tolist())
         elif typ == 'type':
             dout[k0] = dflat[k0]
         else:
             msg = (
-                f"Don't know how to deal with dflat['{k0}']: {typ}"
+                f"Don't know how to deal with dflat['{k0}']:\n"
+                f"\t- typ: {typ}\n"
+                f"\t- type: {type(dflat[k0])}\n"
+                f"\t- value: {dflat[k0]}\n"
             )
             raise Exception(msg)
 
@@ -210,156 +236,235 @@ def load(
 
 
 def get_files(
-    path=None,
-    patterns=None,
-    pfe=None,
-    dpath=None,
+    dpfe=None,
+    returnas=None,
+    strict=None,
 ):
-    """ Return a dict of path keys associated to list of file names
-
-    A pfe is a str describing the path, file name and extension
+    """ Return a dict of path keys associated to list of file names, or a list
+
+    dpfe can be:
+        - str: a valid file path
+        - list of str: a list of valid file paths
+        - dict with:
+            keys = valid path str
+            values =
+                - str: valid file names in the associated path
+                - str: pattern to be found in the files names in that path
+                - list of str: list of the above (file names or patterns)
 
-    If pfe is provided, it is just checked
+    If pattern is (or contains) tuples, the str in tuples are exclusive
 
-    If path / patterns is provided, return all files in path matching patterns
+    """
 
-    If dpath is provided, must be a dict with:
-        - keys: valid path
-        - values: dict with 'patterns' or 'pfe'
+    # ------------------------------------------
+    # check inputs
+    # ------------------------------------------
 
-    If pattern is (or contains) tuples, the str in tuples are exclusive
+    # -----------------
+    # check returnas
 
-    """
+    returnas = _generic_check._check_var(
+        returnas, 'returnas',
+        allowed=[dict, list],
+        default=list,
+    )
 
-    # ------
-    # pick
+    # -----------------
+    # check dpfe
 
     lc = [
-        pfe is not None,
-        patterns is not None,
-        dpath is not None,
+        isinstance(dpfe, (str, tuple)),
+        isinstance(dpfe, list) and all([isinstance(pp, (str, tuple)) for pp in dpfe]),
+        isinstance(dpfe, dict) and all([isinstance(pp, str) for pp in dpfe.keys()])
     ]
 
-    if np.sum(lc) != 1:
-        msg = "Please provide pfe xor pattern xor case!"
+    if not any(lc):
+        msg = (
+            "Please provide dpfe as\n"
+            "\t- str: a valid file path\n"
+            "\t- dict with:\n"
+            "\t\tkeys = valid path str\n"
+            "\t\tvalues =\n"
+            "\t\t\t- str: valid file names in the associated path\n"
+            "\t\t\t- str: pattern to be found in the files names in that path\n"
+            "\t\t\t- list of str: list of the above (file names or patterns)\n"
+        )
         raise Exception(msg)
 
+    # --------------------------------------------
+    # sort cases
+    # --------------------------------------------
+
     # -----------
-    # check path
+    # str or list
+
+    # str
+    if lc[0]:
+        dpfe = [dpfe]
+        lc[0] = False
+        lc[1] = True
+
+    # list of str
+    if lc[1]:
+        # call check on list of files
+        lpfe = _get_files_from_path(
+            lpfe=dpfe,
+            path=None,
+            strict=strict,
+        )
+        dpfe = None
+
+    # -----------
+    # dict
+
+    if lc[2]:
+
+        dout = {}
+        for k0, v0 in dpfe.items():
+
+            # back-compatibility
+            if isinstance(v0, dict):
+                if v0.get('patterns') is not None:
+                    v0 = v0['patterns']
+                elif v0.get('pfe') is not None:
+                    v0 = v0['pfe']
+                else:
+                    msg = ()
+
+            # call pfe / pattern recognition
+            lpfe = _get_files_from_path(
+                lpfe=v0,
+                path=k0,
+                strict=strict,
+            )
+            dout[k0] = [os.path.split(pfe)[1] for pfe in lpfe]
+
+        lpfe = None
+        dpfe = dout
+
+    # -------------------------------------------------
+    # returnas
+    # -------------------------------------------------
+
+    if returnas is list:
+        if lpfe is None:
+            lpfe = list(itt.chain.from_iterable([
+                [os.path.join(k0, v1) for v1 in v0]
+                for k0, v0 in dpfe.items()
+            ]))
+        out = lpfe
 
-    if path is not None:
-        if not (isinstance(path, str) and os.path.isdir(path)):
-            msg = f"Arg path must be a valid path name!\n{path}"
-            return Exception(msg)
     else:
-        path = './'
-    path = os.path.abspath(path)
+        if dpfe is None:
+            lpath = {pfe.split()[0] for pfe in lpfe}
+            dpfe = {
+                path: [pfe for pfe in lpfe if path == pfe.split()[0]]
+                for path in lpath
+            }
+        out = dpfe
 
-    # -----------
-    # check pfe
+    return out
 
-    if isinstance(pfe, str):
-        pfe = [pfe]
 
-    if pfe is not None:
+def _get_files_from_path(
+    lpfe=None,
+    path=None,
+    strict=None,
+):
 
-        err = False
-        assert isinstance(pfe, (list, tuple))
-        lout = [pp for pp in pfe if not os.path.isfile(pp)]
+    # -----------------
+    # preliminary check
 
-        # check that each file exists
-        if len(lout) == len(pfe):
-            pfe = [os.path.join(path, pp) for pp in pfe]
-            lout = [pp for pp in pfe if not os.path.isfile(pp)]
-            if len(lout) > 0:
-                err = True
-        elif len(lout) > 0:
-            err = True
+    if isinstance(lpfe, (str, tuple)):
+        lpfe = [lpfe]
 
-        # Exception
-        if err is True:
-            msg = f"The following files do not exist:\n{lout}"
-            raise Exception(msg)
+    if not all([isinstance(pfe, (str, tuple)) for pfe in lpfe]):
+        msg = (
+            "Please provide a list of str (pfe or patterns)!\n"
+            f"\t- Provided: {lpfe}"
+        )
+        raise Exception(msg)
 
-    # ---------------------------
-    # check pattern
+    # path
+    if path is None:
+        path = os.path.abspath('.')
+    if not os.path.isdir(path):
+        msg = f"Provided path is not valid!\n\t- path: {path}"
+        raise Exception(msg)
 
-    if patterns is not None:
+    # strict
+    strict = _generic_check._check_var(
+        strict, 'strict',
+        types=bool,
+        default=True,
+    )
+
+    # ---------------
+    # pfe vs patterns
 
-        if isinstance(patterns, (str, tuple)):
-            patterns = [patterns]
+    lc = [
+        any([os.path.isfile(pfe) for pfe in lpfe if isinstance(pfe, str)]),
+        any([os.path.isfile(os.path.join(path, pfe)) for pfe in lpfe if isinstance(pfe, str)]),
+    ]
+
+    # ---------------------
+    # valid pfe
 
-        if not all([isinstance(pp, (str, tuple)) for pp in patterns]):
-            msg = f"Arg patterns must be a list of str / tuple!\n{patterns}"
+    if lc[0] or lc[1]:
+
+        if lc[0]:
+            out = [pfe for pfe in lpfe if os.path.isfile(pfe)]
+            lfail = [pfe for pfe in lpfe if not os.path.isfile(pfe)]
+
+        else:
+            out = [
+                os.path.join(path, pfe) for pfe in lpfe
+                if os.path.isfile(os.path.join(path, pfe))
+            ]
+            lfail = [
+                os.path.join(path, pfe) for pfe in lpfe
+                if not os.path.isfile(os.path.join(path, pfe))
+            ]
+
+        if len(lfail) > 0:
+            msg = (
+                "The following files do not exist:\n"
+                + "\n".join([f"\t- {k0}" for k0 in lfail])
+            )
+            if strict is True:
+                raise Exception(msg)
+            else:
+                warnings.warn(msg)
+
+    # ---------------------
+    # patterns
+
+    else:
+
+        if not all([isinstance(pp, (str, tuple)) for pp in lpfe]):
+            msg = f"Arg patterns must be a list of str / tuple!\n{lpfe}"
             raise Exception(msg)
 
-        pfe = sorted([
+        out = sorted([
             os.path.join(path, ff) for ff in os.listdir(path)
             if os.path.isfile(os.path.join(path, ff))
             if all([
                     p0 in ff if isinstance(p0, str)
                     else all([p1 not in ff for p1 in p0])
-                    for p0 in patterns
+                    for p0 in lpfe
                 ])
         ])
 
-    # ---------------------
-    # format pfe into dpfe
-
-    if dpath is None:
-
-        lpf = [os.path.split(ff) for ff in pfe]
-        lpu = sorted(set([ff[0] for ff in lpf]))
-
-        dpfe = {
-            os.path.abspath(k0): [ff[1] for ff in lpf if ff[0] == k0]
-            for k0 in lpu
-        }
-
-    # ----------------
-    # check case
-
-    if dpath is not None:
-
-        # check format
-        c0 = (
-            isinstance(dpath, dict)
-            and all([
-                isinstance(k0, str)
-                and os.path.isdir(k0)
-                and (
-                    isinstance(v0, (str, list))
-                    or (
-                        isinstance(v0, dict)
-                        and isinstance(v0.get('patterns', ''), (list, str, tuple))
-                        and isinstance(v0.get('pfe', ''), (list, str))
-                    )
-                )
-                for k0, v0 in dpath.items()
-            ])
-        )
-        if not c0:
+        # safety check
+        if len(out) == 0:
             msg = (
-                "Arg dpath must be a dict with:\n"
-                "\t- keys: valid paths\n"
-                "\t- values: dict with 'patterns' xor 'pfe'\n"
-                f"Provided:\n{dpath}"
+                "The following list of files is empty:\n"
+                f"\t- lpfe = {lpfe}\n"
+                f"\t- path: {path}"
             )
-            raise Exception(msg)
-
-        # str => patterns
-        for k0, v0 in dpath.items():
-            if isinstance(v0, (str, list)):
-                dpath[k0] = {'patterns': v0}
-
-        # append list of files
-        dpfe = {}
-        for k0, v0 in dpath.items():
-            dpfe.update(get_files(
-                path=k0,
-                pfe=v0.get('pfe'),
-                patterns=v0.get('patterns'),
-                dpath=None,
-            ))
+            if strict is True:
+                raise Exception(msg)
+            else:
+                warnings.warn(msg)
 
-    return dpfe
+    return out

datastock/version.py

@@ -1,2 +1,2 @@
 # Do not edit, pipeline versioning governed by git tags!
-__version__ = '0.0.32'
+__version__ = '0.0.34'

{datastock-0.0.32.dist-info → datastock-0.0.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datastock
-Version: 0.0.32
+Version: 0.0.34
 Summary: A python library for generic class and data handling
 Home-page: https://github.com/ToFuProject/datastock
 Author: Didier VEZINET

{datastock-0.0.32.dist-info → datastock-0.0.34.dist-info}/RECORD

@@ -2,9 +2,9 @@ _updateversion.py,sha256=OR6OJJozaHWzu7NWjKu5ERi0IyYqR61KrWvzf7kfoto,951
 datastock/_DataCollection_utils.py,sha256=hHf6HvGKMmM-psx3fj9QcY1TEmKrAtTdkRokH7SFqoo,7143
 datastock/__init__.py,sha256=MAUJgQG1Mz4Jq9Eu70T9phcV83iFjzLC1qjlqJZDbq4,183
 datastock/_class.py,sha256=Az9PS3aSskiPMb1ekt78Y2ynBujYVc_cDjJxW9xH9g4,47
-datastock/_class0.py,sha256=oV6Ar7in66OnjsDK7RRhe_OSt-ifBKul-x2NbucjF98,5442
+datastock/_class0.py,sha256=QULjNJke13jJrGLIeM7SWHZVziorDK_KCIlqq8LgS9U,5883
 datastock/_class1.py,sha256=l4W0atz_Q-ykO45eONzDtXkwCkcPKwuf2EuD9Ztwak4,29435
-datastock/_class1_binning.py,sha256=G1rbejlwoDRYANI_LY4QOKWA-BEikddVM3-EklROoZQ,28336
+datastock/_class1_binning.py,sha256=LWHv2LIfgZfSFWYwqdcN0DKpNe6q7Go3sxfcJqmzTrI,28085
 datastock/_class1_check.py,sha256=QN0o5Z9XOZAUIoe-dXTVNmbkj3l_v7U4lawcn8N0Yrw,52436
 datastock/_class1_compute.py,sha256=9ka4MXqTb2Gwe8ZnZR31rd5QDc7-nxuhFqXpQSn_sss,28211
 datastock/_class1_domain.py,sha256=TudSRoLedzU2qGOU6R1_Lx99MuqJzG0rOZl8-HZZfTU,6252
@@ -17,7 +17,7 @@ datastock/_direct_calls.py,sha256=EHFwI2mGMDqGz8_Bv2BseMBX4J8dSdE_RcNX3pt0ZYY,18
 datastock/_export_dataframe.py,sha256=fy-uJR3EhDlHvd9ls1EQna_C8fyha1jCJLu1DTKTkdo,1576
 datastock/_find_plateau.py,sha256=sqnAuy0361DXkqBb_Lo1MmIGjn35tnKFvcv6MW6hifs,2685
 datastock/_generic_check.py,sha256=nfD4PIInb63uiwioknFf78dE5lGpgSxoOvT5BgP2nts,24443
-datastock/_generic_utils.py,sha256=iSZa_XOhOJW1Rtr5kH2dxcs_2WaAxcuFls_eEKgOOTw,15382
+datastock/_generic_utils.py,sha256=1RTAcMHVTmBRyz0kvpIBemV-ULxluxA01zMssuafZdk,21243
 datastock/_plot_BvsA_as_distribution.py,sha256=fpRhlbv3Bk96buANC46Brc9hdLxkOAsoKpE5A9pohG0,15389
 datastock/_plot_BvsA_as_distribution_check.py,sha256=7OkzsxqXvJBe6uc_7IiZNwMYWryyI1vKUYYRCkPfz7Q,13002
 datastock/_plot_as_array.py,sha256=s3Jl0B_1gJ-SewSGfyjXrtzMG1utvcq-Vyb0Vmn-rzI,75861
@@ -26,13 +26,13 @@ datastock/_plot_as_profile1d.py,sha256=rxMpLL-QMHqHiAYjH7gDXZOUkMVz3ps8SlUUVNXK8
 datastock/_plot_correlations.py,sha256=ITOypu_AEoKl0ihxocV-JVTXIHqut6p9TfG-xZmQysc,10175
 datastock/_plot_misc.py,sha256=XixTi2CiihKjtQP0TRycH0b25caWN1m35DgpsDeiWZE,21729
 datastock/_plot_text.py,sha256=wQPqjfpLyIioS2JeOt3E9C9HgYUJ49YEoOgRuKYvAR8,3143
-datastock/_saveload.py,sha256=8SM5AdRbcy2KZfDEq5Uesvf55DXzLvPxTMUUsZqhhmk,8885
-datastock/version.py,sha256=DoX-bepDn_EGe6LwIcMmvJjqwD_t5S7ZUGxeWnZ0YXA,80
+datastock/_saveload.py,sha256=Ycu8-OXeak4gqPOJMCE_oeBSZRqdxsqSJs84ZMK4UQg,11663
+datastock/version.py,sha256=ZPzh1gh33VoRYnkFgdCZ0tQKhQLqL6wrNOmoy2vEm0M,80
 datastock/tests/__init__.py,sha256=teOo2xP0IO7PQMuMDmum61XVHe2TuxW3BiHiL73X8jQ,35
 datastock/tests/test_01_DataStock.py,sha256=rIDlqIGGR6gCFtjAw66G9CF3-Mne95XF0eXtUU7wYfc,15260
 datastock/tests/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datastock-0.0.32.dist-info/LICENSE,sha256=V1uXqi3vxR0QhB4QdFyjkynl6jpN4wZmlB5EMYJk0NM,1068
-datastock-0.0.32.dist-info/METADATA,sha256=7V40HnzvYSYxee3fNKOk7FVtvwpVgABiz7ETfUAulZQ,8666
-datastock-0.0.32.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-datastock-0.0.32.dist-info/top_level.txt,sha256=BzJsLLK_zZw13WQCoMhC74qWVKalnVCjBxdPXvJn7HQ,25
-datastock-0.0.32.dist-info/RECORD,,
+datastock-0.0.34.dist-info/LICENSE,sha256=V1uXqi3vxR0QhB4QdFyjkynl6jpN4wZmlB5EMYJk0NM,1068
+datastock-0.0.34.dist-info/METADATA,sha256=tFeEmOoG0f9tFVraH_cu38XphCD8rcBL7ufMj5JCBls,8666
+datastock-0.0.34.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datastock-0.0.34.dist-info/top_level.txt,sha256=BzJsLLK_zZw13WQCoMhC74qWVKalnVCjBxdPXvJn7HQ,25
+datastock-0.0.34.dist-info/RECORD,,

{datastock-0.0.32.dist-info → datastock-0.0.34.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any