sdata 0.22.5__tar.gz → 0.25.0__tar.gz

{sdata-0.22.5/sdata.egg-info → sdata-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sdata
-Version: 0.22.5
+Version: 0.25.0
 Summary: a structured data format
 Home-page: https://github.com/lepy/sdata
 Author: Lepy
@@ -9,7 +9,6 @@ Maintainer: Lepy
 Maintainer-email: lepy@tuta.io
 License: MIT/Apache-2.0
 Keywords: open data
-Platform: UNKNOWN
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -26,6 +25,15 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Description-Content-Type: text/markdown
 License-File: LICENSE-MIT
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: tabulate
+Requires-Dist: xlrd
+Requires-Dist: openpyxl
+Requires-Dist: xlsxwriter
+Requires-Dist: pytz
+Requires-Dist: requests
+Requires-Dist: Pillow
 
 
 [![PyPI version](https://badge.fury.io/py/sdata.svg)](https://badge.fury.io/py/sdata)
@@ -128,5 +136,3 @@ Try to paste some Excel-Data in the forms ...
 * [sdata](https://doi.org/10.5281/zenodo.4311396)
     * Ingolf Lepenies. (2020, December 8). sdata - a structured data format (Version 0.8.4). Zenodo. http://doi.org/10.5281/zenodo.4311397
 
-
-

{sdata-0.22.5 → sdata-0.25.0}/sdata/__init__.py

@@ -1,7 +1,7 @@
 # -*-coding: utf-8-*-
 from __future__ import division
 
-__version__ = '0.22.5'
+__version__ = '0.25.0'
 __revision__ = None
 __version_info__ = tuple([int(num) for num in __version__.split('.')])
 
@@ -14,8 +14,8 @@ import uuid
 import logging
 
 # fix "ValueError: unsupported pickle protocol: 5"
-import pickle
-pickle.HIGHEST_PROTOCOL = 4
+#import pickle
+#pickle.HIGHEST_PROTOCOL = 4
 import pandas
 
 from sdata.metadata import Metadata, Attribute

{sdata-0.22.5 → sdata-0.25.0}/sdata/data.py

@@ -24,6 +24,7 @@ import json
 import hashlib
 import base64
 import requests
+import re
 from tabulate import tabulate
 from sdata.contrib.sqlitedict import SqliteDict
 
@@ -46,12 +47,31 @@ except:
 def uuid_from_str(name):
     return uuid.uuid3(uuid.NAMESPACE_DNS, name)
 
+def nameunit_from_colname(column_name):
+    """
+
+    """
+    #pattern = r"^(.*?)\s*\[(.+)\]$"
+    pattern = r"^(.*?)\s*\[([^\]]+)\]$"
+
+    # Anwenden des Regex
+    match = re.match(pattern, column_name)
+
+    if match:
+        name = match.group(1)  # Der Name der Spalte
+        unit = match.group(2)  # Die Einheit
+    else:
+        name = column_name
+        unit = "-"
+    return name.strip(), unit.strip()
+
 class Data(object):
     """Base sdata object"""
     ATTR_NAMES = []
 
     SDATA_VERSION = "!sdata_version"
     SDATA_NAME = "!sdata_name"
+    SDATA_SNAME = "!sdata_sname"
     SDATA_UUID = "!sdata_uuid"
     SDATA_SUUID = "!sdata_suuid"
     SDATA_CTIME = "!sdata_ctime"
@@ -59,9 +79,11 @@ class Data(object):
     SDATA_PARENT = "!sdata_parent"
     SDATA_CLASS = "!sdata_class"
     SDATA_PROJECT = "!sdata_project"
+    SDATA_URL = "!sdata_url"
+    SDATA_COLUMN = "!sdata_column"
 
     SDATA_ATTRIBUTES = [SDATA_VERSION, SDATA_NAME, SDATA_UUID, SDATA_SUUID, SDATA_CLASS, SDATA_PARENT, SDATA_PROJECT,
-                        SDATA_CTIME, SDATA_MTIME]
+                        SDATA_CTIME, SDATA_MTIME, SDATA_URL]
 
     def __init__(self, **kwargs):
         """create Data object
@@ -94,12 +116,15 @@ class Data(object):
         # set default sdata attributes
         self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
         self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
+        self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
         self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
         self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
         self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
         self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
         self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
         self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
+        self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
+        self.metadata.add(self.SDATA_PROJECT, "", dtype="str", description="project name")
 
         metadata = kwargs.get("metadata")
         if metadata is not None:
@@ -126,7 +151,8 @@ class Data(object):
         self.table = kwargs.get("table", None)
         self._description = ""
         self.description = kwargs.get("description", "")
-        self.project = kwargs.get("project", "")
+        self.project = kwargs.get("project", "N.N.")
+        self.metadata.add(self.SDATA_PROJECT, self.project, dtype="str", description="project name")
 
         if (kwargs.get("uuid")=="" or kwargs.get("uuid") is not None) and not self.metadata.get(self.SDATA_UUID).value and kwargs.get("uuid")!="hash":
             # logger.info("uuid in kwargs")
@@ -150,10 +176,23 @@ class Data(object):
             # logger.info("uuid new")
             self._set_uuid(uuid.uuid4())
 
-        self.gen_suuid()
+        if "suuid" not in kwargs:
+            self.gen_suuid()
+        else:
+            suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
+            self.suuid = kwargs.get("suuid")
+            self.sname = suuid.sname
+
+        if "sname" in kwargs:
+            self.sname = kwargs.get("sname")
+
+
+        if "parent" in kwargs:
+            self.metadata.add(self.SDATA_PARENT, kwargs.get("parent"))
 
     def gen_suuid(self):
         self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).idstr
+        self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
 
     def gen_uuid_from_state(self):
         """generate the same uuid for the same data
@@ -176,7 +215,7 @@ class Data(object):
     def __eq__(self, other):
         """compare Data checksum
         
-        :param other: sdata.Data objecet
+        :param other: sdata.Data object
         :return: True or False
         """
         if not isinstance(other, self.__class__):
@@ -197,6 +236,10 @@ class Data(object):
         self._set_uuid(suuid.huuid)
         return self
 
+    @property
+    def m(self):
+        return self.metadata
+
     def update_mtime(self):
         """update modification time
 
@@ -222,6 +265,13 @@ class Data(object):
             s.update(tablestr)
         return s.hexdigest()
 
+    @property
+    def sha3_256_metadata(self):
+        s = hashlib.sha3_256()
+        metadatastr = self.metadata.to_json().encode(errors="replace")
+        s.update(metadatastr)
+        return s.hexdigest()
+
     @property
     def sha3_256(self):
         """Return a SHA3 hash of the sData object with a hashbit length of 32 bytes.
@@ -238,8 +288,8 @@ class Data(object):
         metadatastr = self.metadata.to_json().encode(errors="replace")
         s.update(metadatastr)
         if self.table is not None:
-            tablestr = self.table.to_json().encode(errors="replace")
-            s.update(tablestr)
+            # tablestr = self.table.to_json().encode(errors="replace")
+            s.update(self.sha3_256_table.encode(errors="replace"))
         s.update(self.description.encode(errors="replace"))
         return s.hexdigest()
 
@@ -330,6 +380,15 @@ class Data(object):
 
     suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
 
+
+    def _get_sname(self):
+        return self.metadata.get(self.SDATA_SNAME).value
+
+    def _set_sname(self, value):
+        self.metadata.set_attr(self.SDATA_SNAME, value)
+
+    sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
+
     def _get_uuid(self):
         return self.metadata.get(self.SDATA_UUID).value
         # return self._uuid
@@ -496,6 +555,49 @@ class Data(object):
             data.to_folder(exportpath, dtype=dtype)
         return path
 
+    def set_column_metadata(self):
+        """convert 'Force [N]' -> 'Force', 'N'
+
+        add Attribute for each column
+
+            sdata_column_0: value="Force", unit="N"
+            sdata_column_1
+            ...
+            sdata_column_n
+
+
+        """
+        if self.df is None:
+            return
+        for i, col in enumerate(self.df.columns):
+            name, unit = nameunit_from_colname(col)
+            scolname = f"{self.SDATA_COLUMN}_{i}"
+            self.metadata.add(scolname, value=name, unit=unit, label=col)
+
+    def set_columnnames_from_metadata(self):
+        """Use '!sdata_column_*' to reformat columns names
+
+                '!sdata_column_0',  value='Force', unit='N' -> 'Force [N]'
+
+                > f'{name} [{unit}]'
+
+        """
+        if self.df is None:
+            return
+
+        cols = []
+        for i, col in enumerate(self.df.columns):
+            scolname = f"{self.SDATA_COLUMN}_{i}"
+            attr = self.metadata.get(scolname)
+            if attr is None:
+                name, unit = nameunit_from_colname(col)
+            else:
+                name = attr.value
+                unit = attr.unit
+            new_colname = f'{name} [{unit}]'
+            cols.append(new_colname)
+        self.df.columns = cols
+
     @classmethod
     def from_folder(cls, path):
         """sdata object instance
@@ -634,6 +736,14 @@ class Data(object):
     def mdf(self):
         return self.metadata.df
 
+    @property
+    def udf(self):
+        return self.metadata.udf
+
+    @property
+    def sdf(self):
+        return self.metadata.sdf
+
     @property
     def md(self):
         return self.metadata
@@ -728,6 +838,39 @@ class Data(object):
         """
         return [(x.name, x.dir()) for x in self.group.values()]
 
+    def to_parquet(self, filepath=None, **kwargs):
+        """
+
+        """
+        engine = kwargs.get("engine", "pyarrow")
+        df = self.df.copy()
+        #df["!sdata_index"] = df.index
+        #df.reset_index(inplace=True)
+        df.attrs = {"metadata": self.metadata.to_dict(),
+                    "description": self.description}
+        df.to_parquet(filepath, engine=engine)
+
+    @classmethod
+    def from_parquet(cls, filepath):
+        """load data from parquet file
+
+        :param filepath:
+        :return: Data object
+        """
+        try:
+            if os.path.exists(filepath):
+                df = pd.read_parquet(filepath)
+                tt = cls(name=filepath)
+                tt.table = df
+                attrs = df.attrs
+                tt.metadata = tt.metadata.from_dict(attrs["metadata"])
+                tt.description = attrs.get("description")
+                return tt
+            else:
+                raise Exception("parquet file '{}' not available".format(filepath))
+        except Exception as exp:
+            raise
+
     def to_xlsx_byteio(self):
         """get xlsx as byteio
 
@@ -838,36 +981,31 @@ class Data(object):
         """
         try:
             if os.path.exists(filepath):
-                wb = openpyxl.load_workbook(filename=filepath)
-                sheetnames = wb.sheetnames
-
                 tt = cls(name=filepath)
 
                 # read df
-                if "table" in sheetnames:
+                try:
                     tt.table = pd.read_excel(filepath, sheet_name="table", index_col=0)
-                else:
+                except:
                     logger.info("no table data in '{}'".format(filepath))
                 dfm = pd.read_excel(filepath, sheet_name="metadata")
-                dfm = dfm.set_index(dfm.name.values)
+                dfm.set_index(dfm.name.values, inplace=True)
                 # dfm["value"] = dfm["value"].replace(np.nan, None)
                 dfm["description"] = dfm["description"].replace(np.nan, '')
                 dfm["label"] = dfm["label"].replace(np.nan, '')
-                # print("!data.from_xlsx", dfm)
+
+                # read metadata
                 tt.metadata = tt.metadata.from_dataframe(dfm)
+                for attr in tt.metadata.sdata_attributes.values():
+                    if attr.value == "nan":
+                        attr.value = ""
 
                 # read description
-                if "description" in sheetnames:
-                    cells = []
-                    for cell in wb["description"]["A"]:
-                        if cell.value is not None:
-                            cells.append(cell.value)
-                        else:
-                            cells.append("")
-                    tt.description = "\n".join(cells)
-                else:
+                try:
+                    dfd = pd.read_excel(filepath, sheet_name="description", header=None)
+                    tt.description = "\n".join(dfd[0])
+                except:
                     logger.info("no description in '{}'".format(filepath))
-
                 return tt
             else:
                 raise Exception("excel file '{}' not available".format(filepath))
@@ -969,7 +1107,6 @@ class Data(object):
         exportlines.append(self.metadata.to_csv_header(prefix="#;", sep=";", filepath=None))
         if self.df is not None:
             exportlines.append(self.df.to_csv(sep=";"))
-
         exportstr = "".join(exportlines)
 
         if filepath is None:
@@ -987,17 +1124,26 @@ class Data(object):
         :param sep: separator (default=";")
         :return: sdata.Data
         """
+        if s is None and filepath is None:
+            logging.error("specify string if filepath")
+            return
         data = cls()
-        if filepath:
-            df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
-            sio = open(filepath, "r")
-        elif s is not None:
-            sio = StringIO(s)
-            pd.read_csv(sio, sep=";", comment="#")
-            sio.seek(0)
-        else:
-            logger.error("data.from_csv: no csv data available")
+        df = None
+        try:
+            if filepath:
+                df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
+                sio = open(filepath, "r")
+                sio.seek(0)
+            elif s is not None:
+                sio = StringIO(s)
+                df = pd.read_csv(sio, sep=";", comment="#")
+                sio.seek(0)
+            else:
+                logger.error("data.from_csv: no csv data available")
+        except Exception as exp:
+            logging.error(f"{exp}")
             raise
+            return
 
         attritute_list = []
         for line in sio:
@@ -1238,7 +1384,7 @@ class Data(object):
         :return: Data
         """
         data = copy.deepcopy(self)
-        data.metadata.add(self.SDATA_PARENT, self.uuid)
+        data.metadata.add(self.SDATA_PARENT, self.suuid)
         data.metadata.add(self.SDATA_UUID, self.gen_uuid())
         data.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str")
         if "uuid" in kwargs:

{sdata-0.22.5 → sdata-0.25.0}/sdata/experiments/__init__.py

@@ -21,6 +21,21 @@ class TestProgram(Data):
     """A sdata Testprogram
 
     """
+    ATTRIBUTES = [
+        ['material_name', '', 'str', '-', 'Der Name des Materials.', '', False],
+        ['material_norm_name', '', 'str', '-', 'Der normative Name des Materials.', '', False],
+        ['material_number_norm', '', 'str', '-', 'Die normative Nummer des Materials.', '', False],
+        ['material_manifacturer', '', 'str', '-', 'Die Hersteller des Materials.', '', False],
+        ['material_grade', '', 'str', '-', 'Der Zustand des Materials.', '', False],
+        ['material_coating', '', 'str', '-', 'Die Beschichtung des Materials.', '', False],
+        ['material_density', '', 'float', 'g/cm^3', 'Die Dichte des Materials.', '', False],
+        ['material_production_date', '', 'str', '-', 'Das Produktionsdatum des Materials.', '', False],
+        ['responsible_person', '', 'str', '-', 'Verantwortlicher des Testprogramms.', '', False],
+        ['responsible_person_phone', '', 'str', '-', 'Telefonnummer des Verantwortlichen vom Testprogramms.', '', False],
+        ['responsible_person_email', '', 'str', '-', 'Emailadressen des Verantwortlichen vom Testprogramms.', '', False],
+        ['responsible_person_organisation', '', 'str', '-', 'Organisation des Verantwortlichen vom Testprogramms.', '', False],
+        ['notes', '', 'str', '-', 'Notizen zum Testprogramm.', '', False],
+        ]
 
     def __init__(self, **kwargs):
         """
@@ -48,13 +63,26 @@ class TestProgram(Data):
         if kwargs.get("uuid_testprogram") is not None:
             self._set_uuid(kwargs.get("uuid_testprogram"))
 
+        for attrlist in self.ATTRIBUTES:
+            self.metadata.set_attr(attrlist[0], attrlist[1],
+                                   dtype=attrlist[2],
+                                   unit=attrlist[3],
+                                   description=attrlist[4],
+                                   label=attrlist[5],
+                                   required=attrlist[6])
+
+
     def gen_testseries(self, **kwargs):
         """generate TestSeries instance
 
         :param kwargs:
         :return: TestSeries()
         """
-        ts = TestSeries(**kwargs)
+        class_name = kwargs.get("class_name", TestSeries)
+        if "class_name" in kwargs:
+            kwargs.pop("class_name")
+
+        ts = class_name(**kwargs)
         ts.project = self.project
         ts.name_testprogram = self.name_testprogram
         ts.uuid_testprogram = self.uuid_testprogram
@@ -63,7 +91,7 @@ class TestProgram(Data):
 
     @property
     def longname(self):
-        return "_".join([self.name_testprogram])
+        return "|".join([self.name_testprogram])
 
 class TestSeries(TestProgram):
     """A sdata TestSeries
@@ -75,6 +103,8 @@ class TestSeries(TestProgram):
 
     SDATA_TESTTYPE = "!sdata_testtype"
 
+
+
     def __init__(self, **kwargs):
         """Test Series
 
@@ -185,7 +215,11 @@ class TestSeries(TestProgram):
         :param kwargs:
         :return: Test()
         """
-        t = Test(**kwargs)
+        class_name = kwargs.get("class_name", Test)
+        if "class_name" in kwargs:
+            kwargs.pop("class_name")
+
+        t = class_name(**kwargs)
         t.name_testprogram = self.name_testprogram
         t.uuid_testprogram = self.uuid_testprogram
         t.name_testseries = self.name_testseries
@@ -198,7 +232,7 @@ class TestSeries(TestProgram):
 
     @property
     def longname(self):
-        return "_".join([self.name_testprogram, self.name])
+        return "|".join([self.name_testprogram, self.name])
 
 
 class Test(TestSeries):
@@ -209,14 +243,15 @@ class Test(TestSeries):
     SDATA_TESTSERIES_NAME = "!sdata_testseries_name"
     SDATA_TESTSERIES_UUID = "!sdata_testseries_uuid"
 
+
     def __init__(self, **kwargs):
         """Test
 
         :param kwargs:
         """
         TestSeries.__init__(self, **kwargs)
-        self.metadata.add(name=self.SDATA_TESTSERIES_NAME, value="N.N.", dtype="str")
-        self.metadata.add(name=self.SDATA_TESTSERIES_UUID, value="", dtype="str")
+        self.metadata.add(name=self.SDATA_TESTSERIES_NAME, value="N.N.", dtype="str", description="name of the testseries")
+        self.metadata.add(name=self.SDATA_TESTSERIES_UUID, value="", dtype="str", description="uuid of the testseries")
 
         if kwargs.get("name") is not None:
             self._set_name(kwargs.get("name"))
@@ -235,6 +270,7 @@ class Test(TestSeries):
             # print("uuid_testprogram", kwargs.get("uuid_testprogram"), self._set_uuid_testprogram)
             self._set_uuid_testprogram(kwargs.get("uuid_testprogram"))
 
+
     def _get_uuid_testseries(self):
         return self.metadata.get(self.SDATA_TESTSERIES_UUID).value
 
@@ -284,7 +320,7 @@ class Test(TestSeries):
 
     @property
     def longname(self):
-        return "_".join([self.name_testprogram, self.name_testseries, self.name])
+        return "|".join([self.name_testprogram, self.name_testseries, self.name])