sdata 1.5.0__py3-none-any.whl

sdata/experiments/__init__.py

@@ -0,0 +1,331 @@
+# -*-coding: utf-8-*-
+import logging
+logger = logging.getLogger("sdata")
+import pandas as pd
+import numpy as np
+from sdata import Data
+from sdata.deprecated.data import Sdata_Uuid_Exeption
+import uuid
+
+# import sdata.experiments.ks2
+# import sdata.experiments.lapshear
+# import sdata.experiments.material
+#
+# # ["name", "default_value", "dtype", "unit", "description", "required"]
+# part_default_attributes = [
+#     ["part_id", None, "str", "-", "Part ID", True],
+# ]
+
+
+class TestProgram(Data):
+    """A sdata Testprogram
+
+    """
+    # Trotz "Test"-Präfix keine pytest-Testklasse (gilt via Vererbung auch für
+    # TestSeries/Test) -> unterdrückt PytestCollectionWarning.
+    __test__ = False
+
+    ATTRIBUTES = [
+        ['material_name', '', 'str', '-', 'Der Name des Materials.', '', False],
+        ['material_norm_name', '', 'str', '-', 'Der normative Name des Materials.', '', False],
+        ['material_number_norm', '', 'str', '-', 'Die normative Nummer des Materials.', '', False],
+        ['material_manifacturer', '', 'str', '-', 'Die Hersteller des Materials.', '', False],
+        ['material_grade', '', 'str', '-', 'Der Zustand des Materials.', '', False],
+        ['material_coating', '', 'str', '-', 'Die Beschichtung des Materials.', '', False],
+        ['material_density', '', 'float', 'g/cm^3', 'Die Dichte des Materials.', '', False],
+        ['material_production_date', '', 'str', '-', 'Das Produktionsdatum des Materials.', '', False],
+        ['responsible_person', '', 'str', '-', 'Verantwortlicher des Testprogramms.', '', False],
+        ['responsible_person_phone', '', 'str', '-', 'Telefonnummer des Verantwortlichen vom Testprogramms.', '', False],
+        ['responsible_person_email', '', 'str', '-', 'Emailadressen des Verantwortlichen vom Testprogramms.', '', False],
+        ['responsible_person_organisation', '', 'str', '-', 'Organisation des Verantwortlichen vom Testprogramms.', '', False],
+        ['notes', '', 'str', '-', 'Notizen zum Testprogramm.', '', False],
+        ]
+
+    def __init__(self, **kwargs):
+        """
+
+        :param kwargs:
+        """
+        Data.__init__(self, **kwargs)
+        self.metadata.add(name=self.SDATA_CLASS, value=self.__class__.__name__, dtype="str")
+
+        # if kwargs.get("uuid_testprogram") is not None:
+        #     try:
+        #         self.uuid_testprogram = kwargs.get("uuid_testprogram") # store given uuid str or generate a new uuid
+        #     except Sdata_Uuid_Exeption as exp:
+        #         if self.auto_correct is True:
+        #             logger.warning("got invald uuid -> generate a new uuid")
+        #             self.uuid_testprogram = uuid.uuid4().hex
+        #         else:
+        #             raise
+
+        self.name_testprogram = self.name
+        self.uuid_testprogram = self.uuid
+
+        if kwargs.get("name_testprogram") is not None:
+            self._set_name(kwargs.get("name_testprogram"))
+        if kwargs.get("uuid_testprogram") is not None:
+            self._set_uuid(kwargs.get("uuid_testprogram"))
+
+        for attrlist in self.ATTRIBUTES:
+            self.metadata.set_attr(attrlist[0], attrlist[1],
+                                   dtype=attrlist[2],
+                                   unit=attrlist[3],
+                                   description=attrlist[4],
+                                   label=attrlist[5],
+                                   required=attrlist[6])
+
+
+    def gen_testseries(self, **kwargs):
+        """generate TestSeries instance
+
+        :param kwargs:
+        :return: TestSeries()
+        """
+        class_name = kwargs.get("class_name", TestSeries)
+        if "class_name" in kwargs:
+            kwargs.pop("class_name")
+
+        ts = class_name(**kwargs)
+        ts.project = self.project
+        ts.name_testprogram = self.name_testprogram
+        ts.uuid_testprogram = self.uuid_testprogram
+        ts.metadata.add(self.SDATA_PARENT, self.uuid)
+        return ts
+
+    @property
+    def longname(self):
+        return "|".join([self.name_testprogram])
+
+class TestSeries(TestProgram):
+    """A sdata TestSeries
+
+    """
+
+    SDATA_TESTPROGRAM_NAME = "!sdata_testprogram_name"
+    SDATA_TESTPROGRAM_UUID = "!sdata_testprogram_uuid"
+
+    SDATA_TESTTYPE = "!sdata_testtype"
+
+
+
+    def __init__(self, **kwargs):
+        """Test Series
+
+        :param kwargs:
+        """
+        TestProgram.__init__(self, **kwargs)
+
+        self.metadata.add(name=self.SDATA_CLASS, value=self.__class__.__name__, dtype="str")
+        self.metadata.add(name=self.SDATA_TESTPROGRAM_NAME, value="N.N.", dtype="str", description="name of the testprogram")
+        self.metadata.add(name=self.SDATA_TESTPROGRAM_UUID, value="", dtype="str", description="uuid of the testprogram")
+        self.metadata.add(name=self.SDATA_TESTTYPE, value="", dtype="str", description="test type")
+
+        # if kwargs.get("name_testprogram") is not None:
+        #     self.name_testprogram = kwargs.get("name_testprogram")
+        #
+        # if kwargs.get("uuid_testseries") is not None:
+        #     try:
+        #         self.uuid_testseries = kwargs.get("uuid_testseries") # store given uuid str or generate a new uuid
+        #     except Sdata_Uuid_Exeption as exp:
+        #         if self.auto_correct is True:
+        #             logger.warning("got invald uuid -> generate a new uuid")
+        #             self.uuid_testseries = uuid.uuid4().hex
+        #         else:
+        #             raise
+
+        self.uuid_testseries = self.uuid
+        self.name_testseries = self.name
+
+        # if kwargs.get("name") is not None:
+        #     self._set_name(kwargs.get("name"))
+        # if kwargs.get("uuid") is not None:
+        #     self._set_uuid(kwargs.get("uuid"))
+
+        # if kwargs.get("name_testseries") is not None:
+        #     self.name_testseries = kwargs.get("name_testseries")
+
+        if kwargs.get("testtype") is not None:
+            self._set_testtype(kwargs.get("testtype"))
+        if kwargs.get("name_testseries") is not None:
+            self._set_name(kwargs.get("name_testseries"))
+        if kwargs.get("uuid_testseries") is not None:
+            self._set_uuid(kwargs.get("uuid_testseries"))
+        if kwargs.get("name") is not None:
+            self._set_name(kwargs.get("name"))
+        if kwargs.get("uuid") is not None:
+            self._set_uuid(kwargs.get("uuid"))
+        if kwargs.get("name_testprogram") is not None:
+            self._set_name_testprogram(kwargs.get("name_testprogram"))
+        if kwargs.get("uuid_testprogram") is not None:
+            self._set_uuid_testprogram(kwargs.get("uuid_testprogram"))
+
+    uuid_testseries = property(fget=TestProgram._get_uuid, fset=TestProgram._set_uuid, doc="uuid of the testseries")
+    name_testseries = property(fget=TestProgram._get_name, fset=TestProgram._set_name, doc="name of the testseries")
+
+    def _get_uuid_testprogram(self):
+        return self.metadata.get(self.SDATA_TESTPROGRAM_UUID).value
+        # return self._uuid
+
+    def _set_uuid_testprogram(self, value):
+        if isinstance(value, str):
+            try:
+                uuid.UUID(value)
+                self.metadata.set_attr(self.SDATA_TESTPROGRAM_UUID, uuid.UUID(value).hex)
+            except ValueError as exp:
+                logger.warning("data.uuid: %s" % exp)
+                raise Sdata_Uuid_Exeption("got invalid uuid str '{}'".format(str(value)))
+        elif isinstance(value, uuid.UUID):
+            self.metadata.set_attr(self.SDATA_TESTPROGRAM_UUID, value.hex)
+        else:
+            logger.error("Data.uuid: invalid uuid '{}'".format(value))
+            raise Exception("Data.uuid: invalid uuid '{}'".format(value))
+
+    uuid_testprogram = property(fget=_get_uuid_testprogram, fset=_set_uuid_testprogram, doc="uuid of the testprogram")
+
+    def _get_name_testprogram(self):
+        # return self._name
+        return self.metadata.get(self.SDATA_TESTPROGRAM_NAME).value
+
+    def _set_name_testprogram(self, value):
+        if isinstance(value, str):
+            try:
+                self.metadata.set_attr(self.SDATA_TESTPROGRAM_NAME, str(value)[:256])
+            except ValueError as exp:
+                logger.warning("data.name_testprogram: %s" % exp)
+        else:
+            # self._name = str(value)[:256]
+            self.metadata.set_attr(self.SDATA_TESTPROGRAM_NAME, str(value)[:256])
+
+    name_testprogram = property(fget=_get_name_testprogram, fset=_set_name_testprogram, doc="name of the testprogram")
+
+    def _get_testtype(self):
+        return self.metadata.get(self.SDATA_TESTTYPE).value
+
+    def _set_testtype(self, value):
+        if isinstance(value, str):
+            try:
+                self.metadata.set_attr(self.SDATA_TESTTYPE, str(value)[:256])
+            except ValueError as exp:
+                logger.warning("data.testtype: %s" % exp)
+        else:
+            self.metadata.set_attr(self.SDATA_TESTTYPE, str(value)[:256])
+
+    testtype = property(fget=_get_testtype, fset=_set_testtype, doc="testtype of the testseries")
+
+    def gen_test(self, **kwargs):
+        """generate Test instance
+
+        :param kwargs:
+        :return: Test()
+        """
+        class_name = kwargs.get("class_name", Test)
+        if "class_name" in kwargs:
+            kwargs.pop("class_name")
+
+        t = class_name(**kwargs)
+        t.name_testprogram = self.name_testprogram
+        t.uuid_testprogram = self.uuid_testprogram
+        t.name_testseries = self.name_testseries
+        t.uuid_testseries = self.uuid_testseries
+        t.project = self.project
+        t.testtype = self.testtype
+        t.metadata.add(self.SDATA_PARENT, self.uuid)
+
+        return t
+
+    @property
+    def longname(self):
+        return "|".join([self.name_testprogram, self.name])
+
+
+class Test(TestSeries):
+    """A sdata Test
+
+    """
+
+    SDATA_TESTSERIES_NAME = "!sdata_testseries_name"
+    SDATA_TESTSERIES_UUID = "!sdata_testseries_uuid"
+
+
+    def __init__(self, **kwargs):
+        """Test
+
+        :param kwargs:
+        """
+        TestSeries.__init__(self, **kwargs)
+        self.metadata.add(name=self.SDATA_TESTSERIES_NAME, value="N.N.", dtype="str", description="name of the testseries")
+        self.metadata.add(name=self.SDATA_TESTSERIES_UUID, value="", dtype="str", description="uuid of the testseries")
+
+        if kwargs.get("name") is not None:
+            self._set_name(kwargs.get("name"))
+        else:
+            self._set_name("N.N.")
+        if kwargs.get("testtype") is not None:
+            # print("testtype", kwargs.get("testtype"), self._set_testtype)
+            self._set_testtype(kwargs.get("testtype"))
+        if kwargs.get("name_testseries") is not None:
+            self._set_name_testseries(kwargs.get("name_testseries"))
+        if kwargs.get("uuid_testseries") is not None:
+            self._set_uuid_testseries(kwargs.get("uuid_testseries"))
+        if kwargs.get("name_testprogram") is not None:
+            self._set_name_testprogram(kwargs.get("name_testprogram"))
+        if kwargs.get("uuid_testprogram") is not None:
+            # print("uuid_testprogram", kwargs.get("uuid_testprogram"), self._set_uuid_testprogram)
+            self._set_uuid_testprogram(kwargs.get("uuid_testprogram"))
+
+
+    def _get_uuid_testseries(self):
+        return self.metadata.get(self.SDATA_TESTSERIES_UUID).value
+
+    def _set_uuid_testseries(self, value):
+        if isinstance(value, str):
+            try:
+                uuid.UUID(value)
+                self.metadata.set_attr(self.SDATA_TESTSERIES_UUID, uuid.UUID(value).hex)
+            except ValueError as exp:
+                logger.warning("data.uuid: %s" % exp)
+                raise Sdata_Uuid_Exeption("got invalid uuid str '{}'".format(str(value)))
+        elif isinstance(value, uuid.UUID):
+            self.metadata.set_attr(self.SDATA_TESTSERIES_UUID, value.hex)
+        else:
+            logger.error("Data.uuid: invalid uuid '{}'".format(value))
+            raise Exception("Data.uuid: invalid uuid '{}'".format(value))
+
+    uuid_testseries = property(fget=_get_uuid_testseries, fset=_set_uuid_testseries, doc="uuid of the testseries")
+
+    def _get_name_testseries(self):
+        return self.metadata.get(self.SDATA_TESTSERIES_NAME).value
+
+    def _set_name_testseries(self, value):
+        if isinstance(value, str):
+            try:
+                self.metadata.set_attr(self.SDATA_TESTSERIES_NAME, str(value)[:256])
+            except ValueError as exp:
+                logger.warning("data.name_testseries: %s" % exp)
+        else:
+            self.metadata.set_attr(self.SDATA_TESTSERIES_NAME, str(value)[:256])
+
+    name_testseries = property(fget=_get_name_testseries, fset=_set_name_testseries, doc="name of the testseries")
+
+    def _get_testtype(self):
+        return self.metadata.get(self.SDATA_TESTTYPE).value
+
+    def _set_testtype(self, value):
+        if isinstance(value, str):
+            try:
+                self.metadata.set_attr(self.SDATA_TESTTYPE, str(value)[:256])
+            except ValueError as exp:
+                logger.warning("data.testtype: %s" % exp)
+        else:
+            self.metadata.set_attr(self.SDATA_TESTTYPE, str(value)[:256])
+
+    testtype = property(fget=_get_testtype, fset=_set_testtype, doc="testtype of the testseries")
+
+    @property
+    def longname(self):
+        return "|".join([self.name_testprogram, self.name_testseries, self.name])
+
+
+
+__all__ = []

sdata/experiments/blob0.py

@@ -0,0 +1,131 @@
+# -*-coding: utf-8-*-
+
+import logging
+import collections
+import pandas as pd
+import numpy as np
+from sdata.timestamp import TimeStamp
+from sdata.base import Base
+from sdata.metadata import Metadata, Attribute
+import json
+import os
+import hashlib
+
+
+class Blob(Base):
+    """Binary Large Object as reference
+
+    .. warning::
+
+        highly experimental"""
+
+    VAULT_TYPES = ["filesystem", "hdf5", "db", "www"]
+
+    def __init__(self, **kwargs):
+        """Binary Large Object
+
+        :param name:
+        :param url:
+        :param filetype:
+        :param vault:
+        :param kwargs:
+
+        """
+        super().__init__(**kwargs)
+        self.metadata.add("blob_name", kwargs.get("name", ""))
+        self.metadata.add("blob_type", "unknown")
+        self.url = kwargs.get("url", "")
+        self.metadata.add("blob_sha1", self.sha1)
+        self.metadata.add("blob_md5", self.md5)
+        self.metadata.add("vault", kwargs.get("vault", ""))
+        self.metadata.add("blob_filetype", kwargs.get("filetype", "unknown"))
+
+
+    def _get_url(self):
+        # return self._name
+        return self.metadata.get("blob_url").value
+
+    def _set_url(self, value):
+        if isinstance(value, str):
+            try:
+                self.metadata.set_attr("blob_url", str(value)[:256])
+            except ValueError as exp:
+                logging.warning("blob.url: %s" % exp)
+        else:
+            # self._name = str(value)[:256]
+            self.metadata.set_attr("blob_url", str(value)[:256])
+
+    url = property(fget=_get_url, fset=_set_url, doc="url of the blob")
+
+    def exists(self, vault="filesystem"):
+        """Test whether a object under the blob.url exists.
+
+        :param vault:
+        :return:
+        """
+        if vault == "filesystem":
+            if os.path.exists(self.url):
+                return True
+            else:
+                return False
+        else:
+            return False
+
+    @property
+    def sha1(self):
+        """calculate the sha1 hash of the blob
+
+        :return: sha1
+        """
+        hash = hashlib.sha1()
+        if self.url and os.path.exists(self.url):
+            with open(self.url, "rb") as fh:
+                self.update_hash(fh, hash)
+            return hash.hexdigest()
+        else:
+            logging.error("can't open external url '{}'".format(str(self.url)))
+
+    @property
+    def md5(self):
+        """calculate the md5 hash of the blob
+
+        :return: sha1
+        """
+        hash = hashlib.md5()
+        if self.url and os.path.exists(self.url):
+            with open(self.url, "rb") as fh:
+                self.update_hash(fh, hash)
+            return hash.hexdigest()
+        else:
+            logging.error("can't open external url '{}'".format(str(self.url)))
+
+    def update_hash(self, fh, hashobject, buffer_size = 65536):
+        """A hash represents the object used to calculate a checksum of a
+        string of information.
+
+        .. code-block:: python
+
+            hashobject = hashlib.md5()
+            df = pd.DataFrame([1,2,3])
+            url = "/tmp/blob.csv"
+            df.to_csv(url)
+            blob = sdata.Blob(url=url)
+            fh = open(url, "rb")
+            blob.update_hash(fh, hashobject)
+            hashobject.hexdigest()
+
+        :param fh: file handle
+        :param hashobject: hash object, e.g. hashlib.sha1()
+        :param buffer_size: buffer size (default buffer_size=65536)
+        :return: hashobject
+        """
+
+
+        with fh as f:
+            while True:
+                data = f.read(buffer_size)
+                if not data:
+                    break
+                hash.update(data)
+        return hash
+

sdata/experiments/file.py

@@ -0,0 +1,42 @@
+from sdata import SUUID, Base
+import os
+
+class File(Base):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def exists(self, path):
+        filepath = os.path.join(path, self.name)
+        return os.path.exists(path)
+
+class Directory(Base):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.path = kwargs.get("path", os.getcwd())
+        self.files = {}
+
+    def register_files(self, path):
+        self.path = path
+        for f in os.listdir(self.path):
+            self.add(File(name=f))
+
+    def add(self, f):
+        """
+
+        """
+        self.files[f.name] = f
+
+    def ls(self):
+        """
+
+        """
+        return list(self.files.keys())
+
+if __name__ == '__main__':
+    f = File(name="a.txt")
+    print(f)
+
+    d = Directory(name="/folder")
+    d.add(f)
+    print(d.ls())
+    print(d)

sdata/experiments/filegroup.py

@@ -0,0 +1,144 @@
+import logging
+from typing import Any, Dict, List, Optional
+from sdata.base import Base
+logger = logging.getLogger(__name__)
+
+class FileGroup(Base):
+    """
+    A derived class from Base that manages a group of files.
+    Stores file metadata in self.data['files'] as a dictionary where:
+    - Keys are unique file identifiers (e.g., filenames or custom keys).
+    - Values are dictionaries containing 'name', 'url' (relative or absolute path),
+      'filetype' (e.g., 'pdf', 'txt'), and 'reference' (e.g., file content as bytes,
+      or a file handle, or any other reference to the file data).
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        """
+        Initialize FileGroup.
+
+        :param kwargs: Keyword arguments passed to Base.__init__.
+        """
+        super().__init__(**kwargs)
+        # Initialize the files dictionary if not already present in data
+        if 'files' not in self.data:
+            self.data['files'] = {}
+
+    def add_file(
+            self,
+            key: str,
+            name: str,
+            url: str,
+            filetype: str,
+            reference: Any,
+            overwrite: bool = False
+    ) -> None:
+        """
+        Add a file's metadata to the group.
+
+        :param key: Unique key for the file (e.g., filename or custom identifier).
+        :param name: The name of the file.
+        :param url: Relative or absolute path to the file.
+        :param filetype: The type of the file (e.g., 'pdf', 'csv', 'jpg').
+        :param reference: Reference to the file, e.g., bytes content, file object, or other data.
+        :param overwrite: If True, overwrite existing entry with the same key.
+        :raises ValueError: If key already exists and overwrite is False.
+        """
+        if key in self.data['files'] and not overwrite:
+            raise ValueError(f"File with key '{key}' already exists. Set overwrite=True to replace.")
+
+        self.data['files'][key] = {
+            'name': name,
+            'url': url,
+            'filetype': filetype,
+            'reference': reference
+        }
+        logger.debug(f"Added file '{key}' to {self.sname}")
+
+    def get_file(self, key: str) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve a file's metadata by key.
+
+        :param key: The unique key for the file.
+        :return: Dictionary with file metadata if found, else None.
+        """
+        return self.data['files'].get(key)
+
+    def remove_file(self, key: str) -> None:
+        """
+        Remove a file from the group by key.
+
+        :param key: The unique key for the file.
+        :raises KeyError: If the key does not exist.
+        """
+        if key not in self.data['files']:
+            raise KeyError(f"File with key '{key}' not found.")
+        del self.data['files'][key]
+        logger.debug(f"Removed file '{key}' from {self.sname}")
+
+    def list_files(self) -> List[str]:
+        """
+        List all file keys in the group.
+
+        :return: List of file keys.
+        """
+        return list(self.data['files'].keys())
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Extend Base.to_dict to include the files in data.
+        Note: If 'reference' contains non-serializable objects (e.g., file handles),
+        they may need custom handling for serialization.
+        """
+        return super().to_dict()
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> 'FileGroup':
+        """
+        Create a FileGroup instance from a dictionary.
+        Assumes 'data' contains 'files' if previously serialized.
+        """
+        instance = super().from_dict(d)
+        # Ensure 'files' is initialized if missing
+        if 'files' not in instance.data:
+            instance.data['files'] = {}
+        return instance
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+
+    # Erstelle eine FileGroup-Instanz (z.B. für ein Projekt "Dokumentensammlung")
+    group = FileGroup(name="Dokumentensammlung", description="Eine Sammlung von PDF- und Text-Dateien.")
+
+    # Füge Dateien hinzu (hier mit Beispieldaten; 'reference' könnte z.B. Bytes oder ein File-Objekt sein)
+    group.add_file(
+        key="doc1",
+        name="Bericht.pdf",
+        url="/pfad/zum/bericht.pdf",  # Relativ- oder Absolutpfad
+        filetype="pdf",
+        reference=b"Beispiel-Inhalt als Bytes"  # Hier Bytes als Platzhalter; könnte auch ein File-Handle sein
+    )
+
+    group.add_file(
+        key="doc2",
+        name="Notizen.txt",
+        url="C:\\Dokumente\\notizen.txt",  # Absolutpfad-Beispiel
+        filetype="txt",
+        reference="Text-Inhalt als String"  # String als Referenz
+    )
+
+    # Versuche, eine Datei zu überschreiben (mit overwrite=True)
+    group.add_file(
+        key="doc1",
+        name="Aktualisierter_Bericht.pdf",
+        url="/neuer/pfad/zum/bericht.pdf",
+        filetype="pdf",
+        reference=b"Neuer Inhalt",
+        overwrite=True
+    )
+
+    # Liste alle Datei-Keys auf
+    print("Verfügbare Dateien:", group.list_files())  # Ausgabe: ['doc1', 'doc2']
+
+    # Hole Metadaten einer spezifischen Datei
+    file_info = group.get_file("doc1")