sdata 0.22.7__tar.gz → 0.23.1__tar.gz

{sdata-0.22.7/sdata.egg-info → sdata-0.23.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sdata
-Version: 0.22.7
+Version: 0.23.1
 Summary: a structured data format
 Home-page: https://github.com/lepy/sdata
 Author: Lepy

{sdata-0.22.7 → sdata-0.23.1}/sdata/__init__.py

@@ -1,7 +1,7 @@
 # -*-coding: utf-8-*-
 from __future__ import division
 
-__version__ = '0.22.7'
+__version__ = '0.23.1'
 __revision__ = None
 __version_info__ = tuple([int(num) for num in __version__.split('.')])
 

{sdata-0.22.7 → sdata-0.23.1}/sdata/data.py

@@ -52,6 +52,7 @@ class Data(object):
 
     SDATA_VERSION = "!sdata_version"
     SDATA_NAME = "!sdata_name"
+    SDATA_SNAME = "!sdata_sname"
     SDATA_UUID = "!sdata_uuid"
     SDATA_SUUID = "!sdata_suuid"
     SDATA_CTIME = "!sdata_ctime"
@@ -95,12 +96,14 @@ class Data(object):
         # set default sdata attributes
         self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
         self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
+        self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
         self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
         self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
         self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
         self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
         self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
         self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
+        self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
 
         metadata = kwargs.get("metadata")
         if metadata is not None:
@@ -151,10 +154,19 @@ class Data(object):
             # logger.info("uuid new")
             self._set_uuid(uuid.uuid4())
 
-        self.gen_suuid()
+        if "suuid" not in kwargs:
+            self.gen_suuid()
+        else:
+            suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
+            self.suuid = kwargs.get("suuid")
+            self.sname = suuid.sname
+
+        if "sname" in kwargs:
+            self.sname = kwargs.get("sname")
 
     def gen_suuid(self):
         self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).idstr
+        self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
 
     def gen_uuid_from_state(self):
         """generate the same uuid for the same data
@@ -177,7 +189,7 @@ class Data(object):
     def __eq__(self, other):
         """compare Data checksum
         
-        :param other: sdata.Data objecet
+        :param other: sdata.Data object
         :return: True or False
         """
         if not isinstance(other, self.__class__):
@@ -331,6 +343,15 @@ class Data(object):
 
     suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
 
+
+    def _get_sname(self):
+        return self.metadata.get(self.SDATA_SNAME).value
+
+    def _set_sname(self, value):
+        self.metadata.set_attr(self.SDATA_SNAME, value)
+
+    sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
+
     def _get_uuid(self):
         return self.metadata.get(self.SDATA_UUID).value
         # return self._uuid

{sdata-0.22.7 → sdata-0.23.1}/sdata/image.py

@@ -12,6 +12,8 @@ import sdata.contrib.piexif
 import sdata.contrib.piexif.helper
 import json
 import os
+from io import BytesIO
+import base64
 import hashlib
 try:
     import PIL
@@ -47,25 +49,89 @@ class Image(Data):
     def _from_filepath(cls, filepath, **kwargs):
         """read image from file
 
+        :param fielpath: image file path
+        :return: sdata.Image
         """
         project = kwargs.get("project")
         suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
-        data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
+        kwargs["uuid"] = suuid.huuid
+        kwargs["url"] = filepath
+        data = cls(name=os.path.basename(filepath), **kwargs)
         return data
 
+    @classmethod
+    def from_image(cls, name, img, **kwargs):
+        """read image from PIL.Image
+
+        """
+        project = kwargs.get("project", "")
+        parent = kwargs.get("parent", "")
+        class_name = "Image" #cls.__class__.__name__
+
+        sh = hashlib.sha3_256()
+        sh.update(img.tobytes())
+        suuid = SUUID.from_name(class_name=class_name, name=sh.hexdigest(), ns_name=project)
+        suuid.project = project
+        kwargs["parent"] = parent
+        kwargs["project"] = suuid.project
+        kwargs["sname"] = suuid.sname
+        kwargs["uuid"] = suuid.huuid
+        kwargs["suuid"] = suuid.idstr
+        kwargs["url"] = sdata.osname(name)
+        # name = os.path.basename(filepath)
+        data = cls(name=name, **kwargs)
+        data.img = img
+        return data
+
+
+    @classmethod
+    def from_bytes(cls, name, image_data, **kwargs):
+        """read image from file
+
+        """
+        project = kwargs.get("project")
+        class_name = "Image" #cls.__class__.__name__
+        suuid = SUUID.from_str(class_name, name, ns_name=project)
+        kwargs["sname"] = suuid.sname
+        kwargs["uuid"] = suuid.huuid
+        kwargs["suuid"] = suuid.idstr
+        kwargs["url"] = name
+        data = cls(name=name, **kwargs)
+        data.img = PIL.Image.open(BytesIO(image_data))
+        try:
+            d = None
+            if name.lower().endswith(".png"):
+                d = cls._get_png_metadata(image_data)
+            elif name.lower().endswith(("jpg", "jpeg")):
+                d = cls._get_jpg_metadata(image_data)
+            if d is not None:
+                data.metadata = data.metadata.from_json(d)
+        except:
+            pass
+        return data
+
+
     @classmethod
     def from_file(cls, filepath, **kwargs):
         """read image from file
 
         """
         project = kwargs.get("project")
-        suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
+        class_name = "Image" #cls.__class__.__name__
+        suuid = SUUID.from_file(class_name, filepath, ns_name=project)
+        kwargs["sname"] = suuid.sname
+        kwargs["uuid"] = suuid.huuid
+        kwargs["suuid"] = suuid.idstr
+        kwargs["url"] = filepath
+        name = os.path.basename(filepath)
+
+
         if filepath.lower().endswith(".png"):
             data = cls.from_png(filepath, **kwargs)
-        elif filepath.lower().endswith(".jpg"):
+        elif filepath.lower().endswith(("jpg", "jpeg")):
             data = cls.from_jpg(filepath, **kwargs)
         else:
-            data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
+            data = cls(name=name, **kwargs)
         return data
 
     def get_sha3_256(self, filepath):
@@ -90,6 +156,15 @@ class Image(Data):
             self.img = PIL.Image.open(self.url)
         return self.img
 
+    def to_numpy(self):
+        """returns a NumPy array created from the color channels of the given Image object
+
+            array([[[192, 190, 187],
+                    [193, 190, 187],
+                    [193, 190, 185],
+        """
+        return np.array(self.img)
+
     @classmethod
     def from_png(cls, filepath, **kwargs):
         """load png image and metadata
@@ -139,9 +214,12 @@ class Image(Data):
             return
         data = cls._from_filepath(filepath, **kwargs)
         # data.load()
-        d = cls._get_jpg_metadata(filepath)
-        if d is not None:
-            data.metadata = data.metadata.from_json(d)
+        try:
+            d = cls._get_jpg_metadata(filepath)
+            if d is not None:
+                data.metadata = data.metadata.from_json(d)
+        except Exception as exp:
+            logging.debug(exp)
         return data
 
     @staticmethod
@@ -205,4 +283,26 @@ class Image(Data):
         json_string = json.dumps(self.metadata.to_json())
         exif_dict = {"Exif": {sdata.contrib.piexif.ExifIFD.UserComment: sdata.contrib.piexif.helper.UserComment.dump(json_string)}}
         exif_bytes = sdata.contrib.piexif.dump(exif_dict)
-        self.img.save(filepath, "jpeg", exif=exif_bytes)
+        self.img.save(filepath, "jpeg", exif=exif_bytes)
+
+    def get_image_download_link(self, img, filename, text):
+
+        json_string = json.dumps(self.metadata.to_json())
+        buffered = BytesIO()
+
+        if filename.endswith(".png"):
+            metadaten = PIL.PngImagePlugin.PngInfo()
+            metadaten.add_text("sdata", json_string)
+            img.save(buffered, "PNG", pnginfo=metadaten)
+        elif filename.lower().endswith(".jpg"):
+            exif_dict = {"Exif": {
+                sdata.contrib.piexif.ExifIFD.UserComment: sdata.contrib.piexif.helper.UserComment.dump(json_string)}}
+            exif_bytes = sdata.contrib.piexif.dump(exif_dict)
+            img.save(buffered, format="JPEG", exif=exif_bytes)
+        else:
+            logging.warning(f"metadata not supported for {filepath}")
+            img.save(buffered)
+
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+        href = f'<a href="data:file/txt;base64,{img_str}" download="{filename}">{text}</a>'
+        return href

{sdata-0.22.7 → sdata-0.23.1}/sdata/metadata.py

@@ -262,7 +262,7 @@ class Metadata(object):
         * type (int, str, float, bool, timestamp)
         """
 
-    ATTRIBUTEKEYS = ["name", "value", "dtype", "unit", "description", "label", "required"]
+    ATTRIBUTEKEYS = ["name", "value", "unit", "dtype", "description", "label", "required"]
 
     def __init__(self, **kwargs):
         """Metadata class
@@ -310,7 +310,7 @@ class Metadata(object):
             attr = name # name is the Attribute!
         else:
             attr = self.get_attr(prefix + name) or Attribute(name, value, **kwargs)
-        for key in ["dtype", "unit", "description", "label", "required"]:
+        for key in ["unit", "dtype", "description", "label", "required"]:
             if key in kwargs:
                 if key in kwargs:
                     # print("!!!", attr, key, kwargs.get(key))
@@ -358,16 +358,21 @@ class Metadata(object):
             pass
         return str(value), "str"
 
-    def update_from_dict(self, d):
+    def update_from_dict(self, d, guess_dtype=True):
         """set attributes from dict
 
         :param d: dict
         :return:
         """
         for k, v in d.items():
-            value, dtype = self.guess_dtype_from_value(v)
+            if guess_dtype is False:
+                value = v
+                dtype = None
+            else:
+                value, dtype = self.guess_dtype_from_value(v)
             if dtype in ["float", "int", "bool"]:
-                v = {"name":k, "value":value, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                v = {"name":k, "value":value}
+                # v = {"name":k, "value":value, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
             elif isinstance(v, (str,)):
                 v = {"name":k, "value":v, "dtype":"str", "unit":"", "description":"", "label":"", "required":False}
             elif hasattr(v, "keys"):
@@ -378,7 +383,8 @@ class Metadata(object):
                      "label":v.get("label", ""), "required":v.get("required", False)}
             else:
                 v, dtype = self.guess_dtype_from_value(v)
-                v = {"name":k, "value":v, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                # v = {"name":k, "value":v, "dtype":dtype, "unit":"", "description":"", "label":"", "required":False}
+                v = {"name":k, "value":v, "dtype":dtype}
             self.set_attr(**v)
 
     @classmethod
@@ -452,6 +458,9 @@ class Metadata(object):
     @classmethod
     def from_dataframe(cls, df):
         """create metadata from dataframe"""
+        df.unit.fillna("", inplace=True)
+        df.label.fillna("", inplace=True)
+        df.description.fillna("", inplace=True)
         d = df.to_dict(orient='index')
         metadata = cls.from_dict(d)
         return metadata
@@ -621,6 +630,31 @@ class Metadata(object):
     def __getitem__(self, name):
         return self.get(name)
 
+    def __setitem__(self, name, value):
+        """update value in Attribute or create new Attribute"""
+        attr = self.get(name)
+
+        if attr is None:
+            attr = Attribute(name=name, value=value, dtype=type(value).__name__)
+            self._attributes[attr.name] = attr
+        else:
+            dtype = attr.guess_dtype(value)
+            attr.dtype = dtype
+            attr.value = dtype(value)
+
+    def __contains__(self, item):
+        return item in self.attributes.keys()
+
+    def pop(self, item):
+        try:
+            return self._attributes.pop(item)
+        except KeyError as exp:
+            return None
+
+    def __iter__(self):
+        for x in self.attributes.items():
+            yield x
+
     @property
     def sha3_256(self):
         """Return a new SHA3 hash object with a hashbit length of 32 bytes.

{sdata-0.22.7 → sdata-0.23.1}/sdata/suuid.py

@@ -1,6 +1,7 @@
 import uuid
 import base64
 import hashlib
+import os
 
 """
 SUUID
@@ -54,7 +55,14 @@ class SUUID:
         if name is None:
             name = ""
 
-        self.name = name
+        self.name = self._clean_name(name)
+
+    @staticmethod
+    def _clean_name(name):
+        rs = ["|", ";"]
+        for r in rs:
+            name = name.replace(r, "")
+        return name
 
     def __str__(self):
         return f"({self.class_name}|{self.name}|{self.huuid})"
@@ -88,6 +96,11 @@ class SUUID:
         s = "".join([self.huuid, self.class_name + "|" + self.name])
         return base64.b64encode(s.encode())
 
+    @property
+    def sname(self):
+        s = "|".join([self.class_name, self.name, self.huuid])
+        return s
+
     @property
     def suuid(self):
         s = "".join([self.huuid, self.class_name + "|" + self.name])
@@ -149,6 +162,7 @@ class SUUID:
         :param ns_string: e.g. projectname "MyProject"
         :return: suuid
         """
+        name = cls._clean_name(name)
         uid = cls.get_uuid_from_name(name=name, ns_name=ns_name)
         suuid = cls(class_name=class_name, name=name, huuid=uid.hex)
         return suuid
@@ -167,6 +181,7 @@ class SUUID:
         :param ns_string: e.g. projectname "MyProject"
         :return: suuid
         """
+        name = cls._clean_name(name)
         suuid = cls.from_name(class_name, name=name, ns_name=ns_name)
         return suuid.id
 
@@ -188,7 +203,9 @@ class SUUID:
         sh = hashlib.sha3_256()
         with open(filepath, "rb") as fh:
             sh.update(fh.read())
+        basename = os.path.basename(filepath)
         suuid = cls.from_name(class_name=class_name, name=sh.hexdigest(), ns_name=ns_name)
+        suuid.name = basename
         return suuid
 
     @classmethod
@@ -210,7 +227,7 @@ class SUUID:
     @classmethod
     def from_suuid_bytes(cls, bytestring):
         """
-        :param bytestring: b'NTcxYzNlY2E1NGYwNWNlY2E4MzFlOGNkNjYxNjMwOWFDTE5fQ1M=\n'
+        :param bytestring: 'MTNhNjMxMWNjZDFlNDFiOGI2MzJhNGY0ZTIzMjdiNTFQYXJ0fG15dGVzdG5hbWUwMQ=='
         """
         id_string = base64.b64decode(bytestring).decode()
         return cls.from_idstr(id_string)
@@ -218,13 +235,23 @@ class SUUID:
     @classmethod
     def from_suuid_str(cls, string):
         """
-        :param string: 'NTcxYzNlY2E1NGYwNWNlY2E4MzFlOGNkNjYxNjMwOWFDTE5fQ1M=\n'
+        :param string: 'MTNhNjMxMWNjZDFlNDFiOGI2MzJhNGY0ZTIzMjdiNTFQYXJ0fG15dGVzdG5hbWUwMQ=='
 
         """
 
         id_string = base64.b64decode(string.encode()).decode()
         return cls.from_idstr(id_string)
 
+
+    @classmethod
+    def from_suuid_sname(cls, s):
+        """
+        :param string: 'Part|mytestname01|22a476e75a68486a92764eb830395923'
+
+        """
+        class_name, name, huuid = s.split("|")
+        return cls(class_name=class_name, name=name, huuid=huuid)
+
     @classmethod
     def from_idstr(cls, id_string):
         try:

{sdata-0.22.7 → sdata-0.23.1/sdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sdata
-Version: 0.22.7
+Version: 0.23.1
 Summary: a structured data format
 Home-page: https://github.com/lepy/sdata
 Author: Lepy