sdata 0.22.5__tar.gz → 0.23.0__tar.gz

{sdata-0.22.5/sdata.egg-info → sdata-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sdata
-Version: 0.22.5
+Version: 0.23.0
 Summary: a structured data format
 Home-page: https://github.com/lepy/sdata
 Author: Lepy

{sdata-0.22.5 → sdata-0.23.0}/sdata/__init__.py

@@ -1,7 +1,7 @@
 # -*-coding: utf-8-*-
 from __future__ import division
 
-__version__ = '0.22.5'
+__version__ = '0.23.0'
 __revision__ = None
 __version_info__ = tuple([int(num) for num in __version__.split('.')])
 

{sdata-0.22.5 → sdata-0.23.0}/sdata/data.py

@@ -52,6 +52,7 @@ class Data(object):
 
     SDATA_VERSION = "!sdata_version"
     SDATA_NAME = "!sdata_name"
+    SDATA_SNAME = "!sdata_sname"
     SDATA_UUID = "!sdata_uuid"
     SDATA_SUUID = "!sdata_suuid"
     SDATA_CTIME = "!sdata_ctime"
@@ -59,9 +60,10 @@ class Data(object):
     SDATA_PARENT = "!sdata_parent"
     SDATA_CLASS = "!sdata_class"
     SDATA_PROJECT = "!sdata_project"
+    SDATA_URL = "!sdata_url"
 
     SDATA_ATTRIBUTES = [SDATA_VERSION, SDATA_NAME, SDATA_UUID, SDATA_SUUID, SDATA_CLASS, SDATA_PARENT, SDATA_PROJECT,
-                        SDATA_CTIME, SDATA_MTIME]
+                        SDATA_CTIME, SDATA_MTIME, SDATA_URL]
 
     def __init__(self, **kwargs):
         """create Data object
@@ -94,12 +96,14 @@ class Data(object):
         # set default sdata attributes
         self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
         self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
+        self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
         self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
         self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
         self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
         self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
         self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
         self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
+        self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
 
         metadata = kwargs.get("metadata")
         if metadata is not None:
@@ -150,10 +154,19 @@ class Data(object):
             # logger.info("uuid new")
             self._set_uuid(uuid.uuid4())
 
-        self.gen_suuid()
+        if "suuid" not in kwargs:
+            self.gen_suuid()
+        else:
+            suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
+            self.suuid = kwargs.get("suuid")
+            self.sname = suuid.sname
+
+        if "sname" in kwargs:
+            self.sname = kwargs.get("sname")
 
     def gen_suuid(self):
         self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).idstr
+        self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
 
     def gen_uuid_from_state(self):
         """generate the same uuid for the same data
@@ -330,6 +343,15 @@ class Data(object):
 
     suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
 
+
+    def _get_sname(self):
+        return self.metadata.get(self.SDATA_SNAME).value
+
+    def _set_sname(self, value):
+        self.metadata.set_attr(self.SDATA_SNAME, value)
+
+    sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
+
     def _get_uuid(self):
         return self.metadata.get(self.SDATA_UUID).value
         # return self._uuid
@@ -969,7 +991,6 @@ class Data(object):
         exportlines.append(self.metadata.to_csv_header(prefix="#;", sep=";", filepath=None))
         if self.df is not None:
             exportlines.append(self.df.to_csv(sep=";"))
-
         exportstr = "".join(exportlines)
 
         if filepath is None:
@@ -988,16 +1009,21 @@ class Data(object):
         :return: sdata.Data
         """
         data = cls()
-        if filepath:
-            df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
-            sio = open(filepath, "r")
-        elif s is not None:
-            sio = StringIO(s)
-            pd.read_csv(sio, sep=";", comment="#")
-            sio.seek(0)
-        else:
-            logger.error("data.from_csv: no csv data available")
+        df = None
+        try:
+            if filepath:
+                df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
+                sio = open(filepath, "r")
+            elif s is not None:
+                sio = StringIO(s)
+                pd.read_csv(sio, sep=";", comment="#")
+                sio.seek(0)
+            else:
+                logger.error("data.from_csv: no csv data available")
+        except Exception as exp:
+            logging.error(f"{exp}")
             raise
+            return
 
         attritute_list = []
         for line in sio:

{sdata-0.22.5 → sdata-0.23.0}/sdata/image.py

@@ -7,6 +7,7 @@ from sdata.timestamp import TimeStamp
 from sdata.data import Data
 from sdata.suuid import SUUID
 from sdata.metadata import Metadata, Attribute
+import sdata.contrib
 import sdata.contrib.piexif
 import sdata.contrib.piexif.helper
 import json
@@ -25,11 +26,22 @@ class Image(Data):
         highly experimental"""
 
 
-    def __init__(self, **kwargs):
+    def __init__(self, url, **kwargs):
         """Image Object"""
+
+        # url = kwargs.get("url", "")
+        load = kwargs.get("load", False)
+        try:
+            name = os.path.basename(url)
+        except Exception as exp:
+            name = None
+        if "name" not in kwargs:
+            kwargs["name"] = name
         Data.__init__(self, **kwargs)
-        self.url = kwargs.get("url", "")
+        self.url = url
         self.img = None
+        if load is True:
+            self.load()
 
     @classmethod
     def _from_filepath(cls, filepath, **kwargs):
@@ -38,7 +50,10 @@ class Image(Data):
         """
         project = kwargs.get("project")
         suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
-        data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
+        kwargs["uuid"] = suuid.huuid
+        kwargs["url"] = filepath
+
+        data = cls(name=os.path.basename(filepath), **kwargs)
         return data
 
     @classmethod
@@ -47,13 +62,21 @@ class Image(Data):
 
         """
         project = kwargs.get("project")
-        suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
+        class_name = "Image" #cls.__class__.__name__
+        suuid = SUUID.from_file(class_name, filepath, ns_name=project)
+        kwargs["sname"] = suuid.sname
+        kwargs["uuid"] = suuid.huuid
+        kwargs["suuid"] = suuid.idstr
+        kwargs["url"] = filepath
+        name = os.path.basename(filepath)
+
+
         if filepath.lower().endswith(".png"):
             data = cls.from_png(filepath, **kwargs)
         elif filepath.lower().endswith(".jpg"):
             data = cls.from_jpg(filepath, **kwargs)
         else:
-            data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
+            data = cls(name=name, **kwargs)
         return data
 
     def get_sha3_256(self, filepath):
@@ -78,6 +101,15 @@ class Image(Data):
             self.img = PIL.Image.open(self.url)
         return self.img
 
+    def to_numpy(self):
+        """returns a NumPy array created from the color channels of the given Image object
+
+            array([[[192, 190, 187],
+                    [193, 190, 187],
+                    [193, 190, 185],
+        """
+        return np.array(self.img)
+
     @classmethod
     def from_png(cls, filepath, **kwargs):
         """load png image and metadata
@@ -93,12 +125,28 @@ class Image(Data):
             data.metadata = data.metadata.from_json(d)
         return data
 
+    @staticmethod
+    def _get_metadata(filepath, **kwargs):
+        """
+
+        """
+        if filepath.lower().endswith(".png"):
+            d = Image._get_png_metadata(filepath, **kwargs)
+        elif filepath.lower().endswith(".jpg"):
+            d = Image._get_png_metadata(filepath, **kwargs)
+        else:
+            d = {}
+        return d
+
     @staticmethod
     def _get_png_metadata(filepath, **kwargs):
         """load metadata json dict from img.info
         """
         img = PIL.Image.open(filepath)
-        d = json.loads(img.info.get("sdata"))
+        try:
+            d = json.loads(img.info.get("sdata"))
+        except Exception as exp:
+            d = {}
         return d
 
     @classmethod
@@ -111,9 +159,12 @@ class Image(Data):
             return
         data = cls._from_filepath(filepath, **kwargs)
         # data.load()
-        d = cls._get_jpg_metadata(filepath)
-        if d is not None:
-            data.metadata = data.metadata.from_json(d)
+        try:
+            d = cls._get_jpg_metadata(filepath)
+            if d is not None:
+                data.metadata = data.metadata.from_json(d)
+        except Exception as exp:
+            logging.debug(exp)
         return data
 
     @staticmethod
@@ -130,7 +181,7 @@ class Image(Data):
             d = json.loads(json_string)
         return d
 
-    def save(self, filepath, rename=True, random=True, **kwargs):
+    def saveas(self, filepath, rename=True, random=True, **kwargs):
         """save image
 
         """
@@ -138,17 +189,22 @@ class Image(Data):
         if rename is True:
             basename = os.path.basename(filepath)
             other.rename(basename, random=random)
+        other.update_mtime()
+        other.save(filepath)
+        return other
 
-        if other.img is None:
-            other.load()
+    def save(self, filepath, **kwargs):
 
-        if filepath.lower().endswith(".png"):
-            other._save_png(filepath)
+        if self.img is None:
+            self.load()
+
+        if filepath.endswith(".png"):
+            self._save_png(filepath)
         elif filepath.lower().endswith(".jpg"):
-            other._save_jpg(filepath)
+            self._save_jpg(filepath)
         else:
             logging.warning(f"metadata not supported for {filepath}")
-            other.img.save(filepath)
+            self.img.save(filepath)
 
     def _save_png(self, filepath):
         """save png with metadata

{sdata-0.22.5 → sdata-0.23.0}/sdata/metadata.py

@@ -56,6 +56,7 @@ class Attribute(object):
     """Attribute class"""
 
     DTYPES = {'float': float, 'int': int, 'str': str, 'timestamp': TimeStamp, "bool": bool}
+    DTYPES_INV = {v: k for k, v in DTYPES.items()}
 
     def __init__(self, name, value, **kwargs):
         """Attribute
@@ -158,6 +159,8 @@ s
         """
         if value is None:
             return None
+        if value in self.DTYPES_INV.keys():
+            value = self.DTYPES_INV.get(value, "str")
         elif "float" in value:
             value = "float"
         elif "int" in value:
@@ -259,7 +262,7 @@ class Metadata(object):
         * type (int, str, float, bool, timestamp)
         """
 
-    ATTRIBUTEKEYS = ["name", "value", "dtype", "unit", "description", "label", "required"]
+    ATTRIBUTEKEYS = ["name", "value", "unit", "dtype", "description", "label", "required"]
 
     def __init__(self, **kwargs):
         """Metadata class
@@ -307,7 +310,7 @@ class Metadata(object):
             attr = name # name is the Attribute!
         else:
             attr = self.get_attr(prefix + name) or Attribute(name, value, **kwargs)
-        for key in ["dtype", "unit", "description", "label", "required"]:
+        for key in ["unit", "dtype", "description", "label", "required"]:
             if key in kwargs:
                 if key in kwargs:
                     # print("!!!", attr, key, kwargs.get(key))
@@ -355,16 +358,21 @@ class Metadata(object):
             pass
         return str(value), "str"
 
-    def update_from_dict(self, d):
+    def update_from_dict(self, d, guess_dtype=True):
         """set attributes from dict
 
         :param d: dict
         :return:
         """
         for k, v in d.items():
-            value, dtype = self.guess_dtype_from_value(v)
+            if guess_dtype is False:
+                value = v
+                dtype = None
+            else:
+                value, dtype = self.guess_dtype_from_value(v)
             if dtype in ["float", "int", "bool"]:
-                v = {"name":k, "value":value, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                v = {"name":k, "value":value}
+                # v = {"name":k, "value":value, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
             elif isinstance(v, (str,)):
                 v = {"name":k, "value":v, "dtype":"str", "unit":"", "description":"", "label":"", "required":False}
             elif hasattr(v, "keys"):
@@ -375,7 +383,8 @@ class Metadata(object):
                      "label":v.get("label", ""), "required":v.get("required", False)}
             else:
                 v, dtype = self.guess_dtype_from_value(v)
-                v = {"name":k, "value":v, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                # v = {"name":k, "value":v, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                v = {"name":k, "value":v, "dtype":dtype}
             self.set_attr(**v)
 
     @classmethod
@@ -449,6 +458,9 @@ class Metadata(object):
     @classmethod
     def from_dataframe(cls, df):
         """create metadata from dataframe"""
+        df.unit.fillna("", inplace=True)
+        df.label.fillna("", inplace=True)
+        df.description.fillna("", inplace=True)
         d = df.to_dict(orient='index')
         metadata = cls.from_dict(d)
         return metadata
@@ -533,17 +545,17 @@ class Metadata(object):
     def from_list(cls, mlist):
         """create metadata from a list of Attribute values
 
-        [['force_x', 1.2, 'float', 'kN', 'force in x-direction'],
-         ['force_y', 3.1, 'float', 'N', 'force in y-direction', 'label', True]]
+        [['force_x', 1.2, 'kN', 'float', 'force in x-direction'],
+         ['force_y', 3.1, 'N', 'float', 'force in y-direction', 'label', True]]
          """
         metadata = cls()
         for alist in mlist:
             if len(alist) < 2:
                 logger.error("Metadata.from_list skip {}".format(alist))
             else:
-                alist.extend(["", "", "", ""])
-                #["name", "value", "dtype", "unit", "description"]
-                metadata.add(alist[0], alist[1], dtype=alist[2], unit=alist[3], description=alist[4],
+                alist.extend(["", "", "", "", ""])
+                #["name", "value", "unit", "dtype", "description", "label", "required"]
+                metadata.add(alist[0], alist[1], unit=alist[2], dtype=alist[3], description=alist[4],
                              label=alist[5], required=alist[6])
         return metadata
 
@@ -618,6 +630,31 @@ class Metadata(object):
     def __getitem__(self, name):
         return self.get(name)
 
+    def __setitem__(self, name, value):
+        """update value in Attribute or create new Attribute"""
+        attr = self.get(name)
+
+        if attr is None:
+            attr = Attribute(name=name, value=value, dtype=type(value).__name__)
+            self._attributes[attr.name] = attr
+        else:
+            dtype = attr.guess_dtype(value)
+            attr.dtype = dtype
+            attr.value = dtype(value)
+
+    def __contains__(self, item):
+        return item in self.attributes.keys()
+
+    def pop(self, item):
+        try:
+            return self._attributes.pop(item)
+        except KeyError as exp:
+            return None
+
+    def __iter__(self):
+        for x in self.attributes.items():
+            yield x
+
     @property
     def sha3_256(self):
         """Return a new SHA3 hash object with a hashbit length of 32 bytes.

{sdata-0.22.5 → sdata-0.23.0}/sdata/suuid.py

@@ -1,6 +1,7 @@
 import uuid
 import base64
 import hashlib
+import os
 
 """
 SUUID
@@ -54,7 +55,14 @@ class SUUID:
         if name is None:
             name = ""
 
-        self.name = name
+        self.name = self._clean_name(name)
+
+    @staticmethod
+    def _clean_name(name):
+        rs = ["|", ";"]
+        for r in rs:
+            name = name.replace(r, "")
+        return name
 
     def __str__(self):
         return f"({self.class_name}|{self.name}|{self.huuid})"
@@ -88,6 +96,11 @@ class SUUID:
         s = "".join([self.huuid, self.class_name + "|" + self.name])
         return base64.b64encode(s.encode())
 
+    @property
+    def sname(self):
+        s = "|".join([self.class_name, self.name, self.huuid])
+        return s
+
     @property
     def suuid(self):
         s = "".join([self.huuid, self.class_name + "|" + self.name])
@@ -149,6 +162,7 @@ class SUUID:
         :param ns_string: e.g. projectname "MyProject"
         :return: suuid
         """
+        name = cls._clean_name(name)
         uid = cls.get_uuid_from_name(name=name, ns_name=ns_name)
         suuid = cls(class_name=class_name, name=name, huuid=uid.hex)
         return suuid
@@ -167,6 +181,7 @@ class SUUID:
         :param ns_string: e.g. projectname "MyProject"
         :return: suuid
         """
+        name = cls._clean_name(name)
         suuid = cls.from_name(class_name, name=name, ns_name=ns_name)
         return suuid.id
 
@@ -188,7 +203,9 @@ class SUUID:
         sh = hashlib.sha3_256()
         with open(filepath, "rb") as fh:
             sh.update(fh.read())
+        basename = os.path.basename(filepath)
         suuid = cls.from_name(class_name=class_name, name=sh.hexdigest(), ns_name=ns_name)
+        suuid.name = basename
         return suuid
 
     @classmethod
@@ -210,7 +227,7 @@ class SUUID:
     @classmethod
     def from_suuid_bytes(cls, bytestring):
         """
-        :param bytestring: b'NTcxYzNlY2E1NGYwNWNlY2E4MzFlOGNkNjYxNjMwOWFDTE5fQ1M=\n'
+        :param bytestring: 'MTNhNjMxMWNjZDFlNDFiOGI2MzJhNGY0ZTIzMjdiNTFQYXJ0fG15dGVzdG5hbWUwMQ=='
         """
         id_string = base64.b64decode(bytestring).decode()
         return cls.from_idstr(id_string)
@@ -218,13 +235,23 @@ class SUUID:
     @classmethod
     def from_suuid_str(cls, string):
         """
-        :param string: 'NTcxYzNlY2E1NGYwNWNlY2E4MzFlOGNkNjYxNjMwOWFDTE5fQ1M=\n'
+        :param string: 'MTNhNjMxMWNjZDFlNDFiOGI2MzJhNGY0ZTIzMjdiNTFQYXJ0fG15dGVzdG5hbWUwMQ=='
 
         """
 
         id_string = base64.b64decode(string.encode()).decode()
         return cls.from_idstr(id_string)
 
+
+    @classmethod
+    def from_suuid_sname(cls, s):
+        """
+        :param string: 'Part|mytestname01|22a476e75a68486a92764eb830395923'
+
+        """
+        class_name, name, huuid = s.split("|")
+        return cls(class_name=class_name, name=name, huuid=huuid)
+
     @classmethod
     def from_idstr(cls, id_string):
         try:

{sdata-0.22.5 → sdata-0.23.0/sdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sdata
-Version: 0.22.5
+Version: 0.23.0
 Summary: a structured data format
 Home-page: https://github.com/lepy/sdata
 Author: Lepy