sdata 1.5.0__py3-none-any.whl

sdata/iolib/pud.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+
+import logging
+logger = logging.getLogger("sdata")
+import os
+import uuid
+import numpy as np
+import pandas as pd
+from sdata.iolib import PID
+from sdata import Data
+
+class Pud(Data):
+    """run object, e.g. single tension test simulation"""
+
+    ATTRIBUTES = ["material_norm_name",
+                  "material_number_norm",
+                  "material_name",
+                  "test",
+                  "sample_ident_number",
+                  "sample_geometry",
+                  "sample_direction",
+                  "nominal_pre_deformation <%>",
+                  "actual_pre_deformation <%>",
+                  "direction_of_pre_deformation",
+                  "heat_treatment",
+                  "actual_sample_width_<mm>",
+                  "actual_sample_thickness_<mm>",
+                  "actual_gauge_length_<mm>",
+                  "nominal_testing_temperature_<K>",
+                  "nominal_testing_speed_<m/s>",
+                  "order",
+                  "date_of_test_<dd.mm.yyyy>",
+                  "tester",
+                  "place_of_test",
+                  "remark",
+                  "data",
+                  ]
+
+    def __init__(self, **kwargs):
+        """"""
+        Data.__init__(self, **kwargs)
+        # for attr in self.ATTRIBUTES:
+        #     self.set_attr(attr, None)
+
+
+    @classmethod
+    def from_file(cls, filepath):
+        """read pud file
+
+            .. code-block:: none
+
+                WERKSTOFF_NORM_NAME                    = HC340LA
+                WERKSTOFFNUMMER_NORM                   =
+                MATERIALNAME                           = HC340LA
+                PRUEFUNG                               = FLIESSKURVE
+                PROBENIDENTNUMMER                      = id0815
+                PROBENGEOMETRIE                        =
+                ENTNAHMERICHTUNG                       = Quer (90deg)
+                VORVERFORMUNG_SOLL <%>                 = 0
+                VORVERFORMUNG_IST  <%>                 =
+                VORVERFORMUNGSRICHTUNG                 = Unverformt
+                WAERMEBEHANDLUNG                       = O
+                PROBENBREITE_IST <mm>                  = 20.014
+                PROBENDICKE_IST <mm>                   = 0.751
+                MESSLAENGE_IST <mm>                    = 80.0
+                MESSLAENGE_IST_FD <mm>                 = 80.0
+                PRUEFTEMPERATUR_SOLL <K>               = 293
+                PRUEFGESCHWINDIGKEIT_SOLL  <mm/s>      = 0.32
+                PRUEFGESCHWINDIGKEIT_IST   <mm/s>      = 0.32
+                DEHNRATE_SOLL <1/s>                    = 0.004
+                DEHNRATE_IST  <1/s>                    = 0.004
+                AUFTRAG                                =
+                PRUEFDATUM  <tt.mm.jjjj>               = 19.03.2017
+                PRUEFER                                = Otto
+                PRUEFSTELLE                            = SALZGITTER AG
+                BEMERKUNG                              = ASL 2009-056
+                DATEN                                  =
+                ZEIT <s>; KRAFT <N>; WEG <mm>; BREITENAENDERUNG <mm>; WEG_FD <mm>
+                1.2372;192.181;-0.0235;0.0012;-0.0235
+                1.2772;198.325;-0.0231;0.0012;-0.0231
+                1.2972;201.397;-0.0227;0.0012;-0.0227
+                1.3172;205.152;-0.0224;0.0013;-0.0224
+                1.3572;211.638;-0.022;0.0013;-0.022
+                1.3972;218.123;-0.0213;0.0013;-0.0213
+        """
+
+        if not os.path.exists(filepath):
+            raise Exception("File not exists '{}'".format(filepath))
+
+        filename = os.path.split(filepath)[-1]
+        tt = cls(name=filename.upper().replace(".TXT", ""))
+
+        df = None
+        # https://docs.python.org/2/library/codecs.html#standard-encodings
+        for encoding in ['ISO-8859-1', "ascii", "utf-8", "iso8859_1", "latin1", 'ISO-8859-1', 'cp1252', "iso8859_2", ]:
+
+            try:
+                logging.debug("try {}".format(encoding))
+                df = pd.read_csv(filepath, encoding=encoding, sep="=", header=None)
+                if df is not None:
+                    break
+            except:  # pragma: no cover - Encoding-Fallback
+                pass
+
+        startindex = df[df[0].str.startswith("DATEN")].iloc[0].name + 1
+        attributes = df.loc[:startindex - 2]
+        attributes.columns = ["key", "value"]
+
+        table = {
+            0xe4     : u'ae',
+            ord(u'ö'): u'oe',
+            ord(u'ü'): u'ue',
+            ord(u'ß'): u'ss',
+            ord(u'Ä'): u'AE',
+            ord(u'Ö'): u'OE',
+            ord(u'Ü'): u'UE',
+            ord(u'ß'): u'SS',
+        }
+
+        for row, attr in attributes.iterrows():
+            s = attr.key  # .decode('utf8')
+            attr.key = s.translate(table)
+            try:
+                tt.metadata.set_attr(attr.key.strip(), attr.value.strip())
+            except AttributeError as exp:
+                tt.metadata.set_attr(attr.key.strip(), attr.value)
+            except UnicodeEncodeError as exp:  # pragma: no cover - seltener Encoding-Fehler
+                print("UnicodeEncodeError {}".format(attr.key))
+                raise
+
+        table = df.loc[startindex:].copy()
+        table = table[0].str.split(";", expand=True)
+        table.columns = [x.strip() for x in table.iloc[0].values]
+        table = table.drop(index=startindex)
+        for col in table.columns:
+            table[col] = table[col].astype(float)
+
+        if 'KRAFT <kN>' in table.columns:  # pragma: no cover - kN-Einheit selten
+            table['KRAFT <N>'] = table['KRAFT <kN>'] * 1e3
+
+        # print(table)
+        # print(attributes)
+        # mapper={"Kraft <N>":"force",
+        #    'Weg <mm>':"displacement",
+        #    "Zeit <s>":"time",
+        #    "Breitenaenderung <mm>":"displacement_y",
+        #     "KRAFT <N>":"force",
+        #    'WEG <mm>':"displacement",
+        #    "ZEIT <s>":"time",
+        #    "BREITENAENDERUNG <mm>":"displacement_y"}
+        #
+        # table = table.rename(columns=mapper)
+
+        attr = tt.metadata.get("PROBENIDENTNUMMER")
+        tt.uuid = uuid.uuid5(uuid.NAMESPACE_DNS, str(attr.name.upper()))
+
+        tt.table = table
+
+        l0 = tt.metadata.get('MESSLAENGE_IST <mm>', np.nan)
+        width = tt.metadata.get('PROBENBREITE_IST <mm>', np.nan)
+        thickness = tt.metadata.get('PROBENDICKE_IST <mm>', np.nan)
+        actual_testing_temperature = tt.metadata.get('PRUEFTEMPERATUR_SOLL <K>', np.nan)
+        sample_direction = tt.metadata.get('ENTNAHMERICHTUNG', np.nan)
+        nominal_pre_deformation = tt.metadata.get('VORVERFORMUNG_SOLL <%>', np.nan)
+        actual_pre_deformation = tt.metadata.get('VORVERFORMUNG_IST  <%>', np.nan)
+        nominal_strain_rate = tt.metadata.get('DEHNRATE_SOLL <1/s>', np.nan)
+        actual_strain_rate = tt.metadata.get('DEHNRATE_IST  <1/s>', np.nan)
+        place_of_test = tt.metadata.get('PRUEFSTELLE', np.nan)
+
+        mapper = {"Kraft <N>"            : "force",
+                  'Weg <mm>'             : "displacement",
+                  "Zeit <s>"             : "time",
+                  "Breitenaenderung <mm>": "displacement_y",
+                  "KRAFT <N>"            : "force",
+                  'WEG <mm>'             : "displacement",
+                  "ZEIT <s>"             : "time",
+                  "BREITENAENDERUNG <mm>": "displacement_y"}
+
+        try:
+            l0 = float(l0)
+        except Exception as exp:
+            l0 = np.nan
+
+        try:
+            width = float(width)
+        except Exception as exp:
+            width = np.nan
+
+        try:
+            thickness = float(thickness)
+        except Exception as exp:
+            thickness = np.nan
+
+        try:
+            actual_testing_temperature = float(actual_testing_temperature)
+        except Exception as exp:
+            actual_testing_temperature = np.nan
+
+        try:
+            sample_direction = tt.metadata.get_orientation(sample_direction)
+        except Exception as exp:
+            sample_direction = np.nan
+
+        try:
+            nominal_pre_deformation = float(nominal_pre_deformation) / 100.
+        except Exception as exp:
+            nominal_pre_deformation = np.nan
+
+        try:
+            actual_pre_deformation = float(actual_pre_deformation) / 100.
+        except Exception as exp:
+            actual_pre_deformation = np.nan
+
+        try:
+            nominal_strain_rate = float(nominal_strain_rate)
+        except Exception as exp:
+            nominal_strain_rate = np.nan
+
+        try:
+            actual_strain_rate = float(actual_strain_rate)
+        except Exception as exp:
+            actual_strain_rate = np.nan
+
+        # try:
+        #     place_of_test = str(place_of_test)
+        # except Exception as exp:
+        #     place_of_test = "?"hash
+
+        area = width * thickness
+        tt.metadata.set_attr("l0", l0)
+        tt.metadata.set_attr("width", width)
+        tt.metadata.set_attr("thickness", thickness)
+        tt.metadata.set_attr("area", area)
+        tt.metadata.set_attr("actual_testing_temperature", actual_testing_temperature)
+        tt.metadata.set_attr("sample_direction", sample_direction)
+        tt.metadata.set_attr("nominal_pre_deformation", nominal_pre_deformation)
+        tt.metadata.set_attr("actual_pre_deformation", actual_pre_deformation)
+        tt.metadata.set_attr("nominal_strain_rate", nominal_strain_rate)
+        tt.metadata.set_attr("actual_strain_rate", actual_strain_rate)
+        tt.metadata.set_attr("place_of_test", place_of_test)
+
+        return tt

sdata/iolib/reader.py

@@ -0,0 +1,262 @@
+"""Einheitliche Reader-/Source-Abstraktion für :class:`sdata.sclass.dataframe.DataFrame`.
+
+RFC 0009 — das symmetrische Gegenstück zu :mod:`sdata.iolib.writer` (RFC 0007).
+Ein gemeinsamer ``read()``-Vertrag, unter dem Quellen (Parquet-Datei/Objektspeicher,
+JSON1-SQLite-Store, relationales SQL) austauschbar sind; die ``require_*``-Prüfung
+aus dem Writer dient hier als **Eingangs**-Validierung einer Pipeline.
+
+Konkrete Reader:
+
+* :class:`ParquetReader` — eine fsspec-URI, ein Objekt (``_sdata`` reist im Format mit).
+* :class:`StoreReader`   — Selektor-Zugriff (``suuid``/``sname``/Record-ID) und Iteration
+  über einen :class:`~sdata.iolib.json1sqlitestore.JSON1SQLiteStore`.
+* :class:`SqlReader`     — verlustfreier Snapshot aus der ``sdata_dataframe_meta``-Tabelle
+  des :class:`~sdata.iolib.writer.SqlWriter`; ``fresh=True`` liest zusätzlich den
+  **aktuellen** Stand der Datentabelle.
+
+``read()`` gibt das :class:`DataFrame` selbst zurück — kein ``ReadReceipt``-Pendant:
+das Objekt trägt Identität (``suuid``/``sname``) und Provenienz bereits in seinen
+Metadaten (RFC 0009 §8).
+"""
+from __future__ import annotations
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Iterator, Protocol, Tuple, runtime_checkable, TYPE_CHECKING
+
+import pandas as pd
+
+from sdata.iolib.writer import check_contract, _safe_identifier, _SQL_META_TABLE
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from sdata.sclass.dataframe import DataFrame
+
+__all__ = [
+    "DataFrameReader",
+    "BaseDataFrameReader",
+    "ParquetReader",
+    "StoreReader",
+    "SqlReader",
+    "read_group",
+]
+
+#: Jüngste Metazeile zu einem Selektor (suuid **oder** sname); Literal-SQL mit
+#: qmark-Parametern — keine Identifier-Interpolation (wie im Writer, RFC 0007 C1).
+_SQL_META_SELECT = (
+    "SELECT target_table, sdata FROM sdata_dataframe_meta "
+    "WHERE suuid = ? OR sname = ? ORDER BY rowid DESC LIMIT 1"
+)
+
+
+def read_group(reader: "DataFrameReader", keys, *, name: str = "group"):
+    """Mehrere Mitglieder aus **einer** Quelle zu einer
+    :class:`~sdata.sclass.dataframegroup.DataFrameGroup` zusammenführen (RFC 0011).
+
+    :param reader: eine beliebige :class:`DataFrameReader`-Quelle.
+    :param keys: Iterable von Selektoren (suuid/sname/…), je einer pro Mitglied.
+    :param name: Name der erzeugten Gruppe.
+    :return: eine ``DataFrameGroup`` mit den gelesenen Mitgliedern (Schlüssel = Selektor).
+    """
+    from sdata.sclass.dataframegroup import DataFrameGroup
+    group = DataFrameGroup(name=name)
+    with reader:
+        for key in keys:
+            group.add(reader.read(key), key=str(key), overwrite=True)
+    return group
+
+
+@runtime_checkable
+class DataFrameReader(Protocol):
+    """Struktureller Vertrag jeder Quelle."""
+
+    def read(self, selector: Any = None) -> "DataFrame":
+        ...
+
+    def close(self) -> None:
+        ...
+
+
+class BaseDataFrameReader(ABC):
+    """Template-Method: vom Backend lesen, dann den Eingangsvertrag prüfen."""
+
+    def __init__(self,
+                 require_metadata: Tuple[str, ...] = (),
+                 require_columns: Tuple[str, ...] = (),
+                 require_units: Tuple[str, ...] = ()):
+        """Store the require_* contract (Pflichtschlüssel/-spalten/-einheiten)."""
+        self.require_metadata = tuple(require_metadata)
+        self.require_columns = tuple(require_columns)
+        self.require_units = tuple(require_units)
+        self._count = 0
+
+    def read(self, selector: Any = None) -> "DataFrame":
+        sdf = self._read_impl(selector)
+        check_contract(sdf, self.require_metadata, self.require_columns,
+                       self.require_units, role="reader")
+        self._count += 1
+        logger.info("%s read %r", type(self).__name__, sdf.sname)
+        return sdf
+
+    @abstractmethod
+    def _read_impl(self, selector: Any) -> "DataFrame":
+        ...
+
+    def close(self) -> None:
+        """Release resources (default: no-op)."""
+
+    def __enter__(self) -> "BaseDataFrameReader":
+        """Enter the context manager (returns self)."""
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        """Exit the context manager, closing the reader."""
+        self.close()
+
+
+class ParquetReader(BaseDataFrameReader):
+    """Liest **eine** Parquet-URI (fsspec: lokal, ``s3://``, …) als :class:`DataFrame`.
+
+    Die Metadaten reisen im Format mit (``_sdata``-Schema-Key, RFC 0007 §6.1);
+    :meth:`DataFrame.from_parquet_bytes` restauriert ``metadata``/``column_metadata``/
+    ``description``.
+    """
+
+    def __init__(self, uri: str, *, directory: bool = False, **contract: Any):
+        """Bind the source fsspec ``uri`` and the require_* contract.
+
+        :param uri: Quell-URI (Einzeldatei) bzw. Quell-Verzeichnis (``directory=True``).
+        :param directory: Verzeichnis-Modus — ``read(sname)`` liest ``<uri>/<sname>.spq``;
+          :meth:`keys` listet die Mitglieder (Gegenseite zu ``ParquetWriter(directory=True)``).
+        """
+        super().__init__(**contract)
+        self.uri = uri
+        self.directory = directory
+
+    def _read_impl(self, selector: Any) -> "DataFrame":
+        from sdata.sclass.blob import fsspec
+        from sdata.sclass.dataframe import DataFrame
+
+        if fsspec is None:  # pragma: no cover - optionale Abhängigkeit
+            raise ImportError("ParquetReader requires: pip install sdata[blob]")
+        if self.directory:
+            if selector is None:
+                raise ValueError(
+                    "directory ParquetReader needs a key (sname); use keys()/read_group")
+            src = "{}/{}.spq".format(self.uri.rstrip("/"), selector)
+        else:
+            if selector is not None:
+                raise ValueError(
+                    "ParquetReader reads exactly one URI; selector is not supported")
+            src = self.uri
+        with fsspec.open(src, "rb") as fh:
+            return DataFrame.from_parquet_bytes(fh.read())
+
+    def keys(self) -> list:
+        """Mitglied-``sname``s eines Verzeichnisses (globt ``<uri>/*.spq``).
+
+        :return: sortierte Liste der ``sname``s; mit ``read_group(reader, reader.keys())``
+          liest man das ganze Verzeichnis in eine :class:`DataFrameGroup`.
+        :raises ValueError: außerhalb des Verzeichnis-Modus.
+        """
+        if not self.directory:
+            raise ValueError("keys() requires directory mode")
+        from sdata.sclass.blob import fsspec
+
+        if fsspec is None:  # pragma: no cover - optionale Abhängigkeit
+            raise ImportError("ParquetReader requires: pip install sdata[blob]")
+        base = self.uri.rstrip("/")
+        fs, _, _ = fsspec.get_fs_token_paths(base)
+        paths = fs.glob("{}/*.spq".format(base))
+        return sorted(p.rsplit("/", 1)[-1][:-4] for p in paths)
+
+
+class StoreReader(BaseDataFrameReader):
+    """Selektor-Zugriff und Iteration über einen :class:`JSON1SQLiteStore`.
+
+    Invertiert den Flatten-Adapter des :class:`~sdata.iolib.writer.StoreWriter`
+    (RFC 0007 §6.2): die ``_sdata_*``-Spalten machen den Record auffindbar, das
+    verlustfreie ``to_dict`` unter ``"sdata"`` stellt das Objekt wieder her.
+    """
+
+    def __init__(self, target: Any, **contract: Any):
+        """Bind a store: an existing ``JSON1SQLiteStore`` (not owned) or a DB path (owned)."""
+        super().__init__(**contract)
+        from sdata.iolib.json1sqlitestore import JSON1SQLiteStore
+
+        if hasattr(target, "get_id_by_key"):
+            self._store = target
+            self._owns = False
+        else:
+            self._store = JSON1SQLiteStore(target)
+            self._owns = True
+
+    def _read_impl(self, selector: Any) -> "DataFrame":
+        from sdata.sclass.dataframe import DataFrame
+
+        row = self._resolve(selector)
+        if row is None or "sdata" not in row:
+            raise KeyError(f"no DataFrame record for selector {selector!r}")
+        return DataFrame.from_dict(row["sdata"])
+
+    def _resolve(self, selector: Any):
+        """Dokumentiert deterministisch: ``int`` → Record-ID, sonst suuid, dann sname."""
+        if isinstance(selector, int):
+            return self._store.get(selector)
+        sel = str(selector)  # SUUID-Objekte -> str (RFC 0007 F4)
+        rid = (self._store.get_id_by_key("_sdata_suuid", sel)
+               or self._store.get_id_by_key("_sdata_sname", sel))
+        return self._store.get(rid) if rid is not None else None
+
+    def iter(self) -> Iterator["DataFrame"]:
+        """Alle DataFrame-Records des Stores; Records ohne ``"sdata"``-Payload
+        (Fremdobjekte) werden übersprungen."""
+        from sdata.sclass.dataframe import DataFrame
+
+        for row in self._store.fetch_all():
+            if isinstance(row, dict) and "sdata" in row:
+                yield DataFrame.from_dict(row["sdata"])
+
+    def close(self) -> None:
+        """If this reader owns the store, close its connection."""
+        if self._owns:
+            self._store.conn.close()
+
+
+class SqlReader(BaseDataFrameReader):
+    """Rekonstruiert ein :class:`DataFrame` aus der ``sdata_dataframe_meta``-Tabelle.
+
+    * ``fresh=False`` (Default): Daten **und** Metadaten aus dem ``sdata``-JSON der
+      Metazeile — der verlustfreie **Snapshot zum Schreibzeitpunkt** (inkl. Einheiten/
+      Ontologie, die ``to_sql`` selbst verliert).
+    * ``fresh=True``: Metadaten aus dem Snapshot, Datenzeilen per ``read_sql`` aus
+      ``target_table`` — der **aktuelle** Tabellenstand (bei ``if_exists="append"``
+      kumulativ über mehrere Schreibvorgänge; Provenienz pro Zeile: ``suuid``-Spalte,
+      RFC 0007 §6.2). Der Tabellenname läuft durch dieselbe Allowlist wie im Writer.
+    """
+
+    def __init__(self, conn: Any, *, fresh: bool = False, **contract: Any):
+        """Bind a DBAPI/PEP-249 connection (oder ``engine.raw_connection()``)."""
+        super().__init__(**contract)
+        self._conn = conn
+        self.fresh = fresh
+        self.meta_table = _SQL_META_TABLE
+
+    def _read_impl(self, selector: Any) -> "DataFrame":
+        from sdata.sclass.dataframe import DataFrame
+
+        sel = str(selector)
+        row = self._conn.execute(_SQL_META_SELECT, (sel, sel)).fetchone()
+        if row is None:
+            raise KeyError(f"no entry in {self.meta_table} for selector {selector!r}")
+        target_table, sdata_json = row[0], row[1]
+        sdf = DataFrame.from_dict(json.loads(sdata_json))
+        if self.fresh:
+            table = _safe_identifier(target_table)
+            # Identifier ist Allowlist-validiert (RFC 0009 §5.3); Parameter-Binding
+            # für Identifier existiert in SQL nicht.
+            query = "SELECT * FROM " + table  # nosec B608
+            sdf.df = pd.read_sql_query(query, self._conn)
+        return sdf