sdata 1.5.0__py3-none-any.whl

sdata/parameter.py

@@ -0,0 +1,414 @@
+import json
+import logging
+from typing import Dict, Tuple, List, Union, Optional
+import random
+
+logger = logging.getLogger(__name__)
+
+
+class Distribution:
+    def __init__(self, name: str, **params):
+        self.name = name
+        self.params = params
+        self.dtype = float
+
+    def to_dict(self) -> Dict:
+        return {
+            "name": self.name,
+            "params": self.params
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict):
+        return cls(name=data["name"], **data.get("params", {}))
+
+    def generate_value(self, bounds: Union[Tuple[float, float], List[Union[int, str]]]) -> Union[float, int, str]:
+        min_val, max_val = bounds
+        if self.name == "uniform":
+            return random.uniform(min_val, max_val)
+        raise ValueError(f"Unsupported distribution: {self.name}")
+
+
+class Parameter:
+    def __init__(self, name: str, dtype: type, description: Optional[str] = None, fix: bool = False):
+        self.name = name
+        self.dtype = dtype
+        self.description = description or ""
+        self.fix = fix
+
+    def _validate_and_cast_value(self, value):
+        if not isinstance(value, self.dtype):
+            try:
+                return self.dtype(value)
+            except (ValueError, TypeError):
+                import warnings
+                warnings.warn(f"Value {value} could not be cast to {self.dtype}. Keeping the original value.")
+                return value
+        return value
+
+    def to_dict(self) -> Dict:
+        return {
+            "class_name": self.__class__.__name__,
+            "name": self.name,
+            # "dtype": self.dtype.__name__,
+            "description": self.description,
+            "fix": self.fix
+        }
+
+    @classmethod
+    def get_subclasses(cls):
+        class_name = "Parameter"
+        all_subclasses = {}
+        subclasses = {subclass.__name__: subclass for subclass in cls.__subclasses__()}
+        all_subclasses.update(subclasses)
+        for subclassname, subcls in subclasses.items():
+            subsubclasses = {subclass.__name__: subclass for subclass in subcls.__subclasses__()}
+            #            all_subclasses = all_subclasses | subsubclasses
+            all_subclasses.update(subsubclasses)
+
+        return all_subclasses
+
+    @classmethod
+    def from_dict(cls, data: Dict):
+        class_name = data.get("class_name", "Parameter")
+        subclasses = cls.get_subclasses()
+        if class_name in subclasses:
+            return subclasses[class_name].from_dict(data)
+        raise ValueError(f"Unknown class_name '{class_name}' in data.")
+
+
+class Variable(Parameter):
+    def __init__(self, name: str, value: Union[bool, float, int, str], dtype: type,
+                 bounds: Optional[Tuple[float, float]] = None,
+                 distribution: Optional[Distribution] = None,
+                 description: Optional[str] = None, fix: bool = False):
+        description = description or ""
+        super().__init__(name, dtype, description, fix)
+        self.bounds = bounds
+        self.distribution = distribution or Distribution("uniform")
+        self.value = self._validate_and_cast_value(value)
+
+    def sample_value(self) -> Union[bool, float, int, str]:
+        if self.fix:
+            return self.value
+        sampled = self.distribution.generate_value(self.bounds)
+        self.value = self._validate_and_cast_value(sampled)
+        return self.value
+
+    def to_dict(self) -> Dict:
+        data = super().to_dict()
+        data.update({
+            "value": self.value,
+            "bounds": self.bounds,
+            "distribution": self.distribution.to_dict() if self.distribution else None
+        })
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict):
+        distribution = Distribution.from_dict(data["distribution"]) if data.get("distribution") else None
+        return cls(
+            name=data["name"],
+            value=data["value"],
+            bounds=data.get("bounds"),
+            distribution=distribution,
+            description=data.get("description"),
+            fix=data.get("fix", False)
+        )
+
+    def sample_values(self, n: int = 1) -> List[Union[bool, float, int, str]]:
+        """
+        Generate n sampled values based on the defined distribution.
+        """
+        if self.fix is True:
+            return [self.value] * n
+        else:
+            return [self.dtype(self.distribution.generate_value(self.bounds)) for _ in range(n)]
+
+    def __repr__(self):
+        try:
+            value_bounds = f"{self.min:.2g},{self.max:.2g}"
+        except:
+            value_bounds = str(self.bounds)
+        return f"({self.name}|{self.value}|[{value_bounds}]|{self.dtype.__name__})"
+
+    def __str__(self):
+        return self.__repr__()
+
+
+class DiscreteVariable(Parameter):
+    def __init__(self, name: str, value: Union[bool, float, int, str], dtype: type,
+                 discrete_values: List[Union[bool, int, str]],
+                 description: Optional[str] = None, fix: bool = False):
+        description = description or ""
+        super().__init__(name, dtype, description, fix)
+        self.discrete_values = [self._validate_and_cast_value(v) for v in discrete_values]
+        self.value = self._validate_and_cast_value(value)
+
+    def sample_value(self) -> Union[bool, float, int, str]:
+        if self.fix:
+            return self.value
+        self.value = random.choice(self.discrete_values)
+        return self.value
+
+    def to_dict(self) -> Dict:
+        data = super().to_dict()
+        data.update({
+            "value": self.value,
+            "discrete_values": self.discrete_values
+        })
+        return data
+
+    def __repr__(self):
+        try:
+            if self.discrete_values and len(self.discrete_values) > 4:
+                value_bounds = self.discrete_values[:4] + ["..."]
+            elif self.discrete_values:
+                value_bounds = self.discrete_values
+            else:
+                value_bounds = "None"
+        except:
+            value_bounds = "!"
+        return f"({self.name}|{self.value}|[{value_bounds}]|{self.dtype.__name__})"
+
+    def __str__(self):
+        return self.__repr__()
+
+    def sample_values(self, n: int = 1) -> List[Union[bool, float, int, str]]:
+        """
+        Generate n sampled values based on the defined distribution.
+        """
+        if self.fix is True:
+            return [self.value] * n
+        else:
+            return [random.choice(self.discrete_values) for _ in range(n)]
+
+
+# Specialized Variable Classes
+class VariableInt(Variable):
+    dtype = int
+
+    def __init__(self, name: str, value: int, bounds: Optional[Tuple[int, int]] = None,
+                 distribution: Optional[Distribution] = None, description: Optional[str] = None, fix: bool = False):
+        super().__init__(name, value, int, bounds, distribution, description, fix)
+
+    @property
+    def min(self):
+        if isinstance(self.bounds, (list, tuple)):
+            return self.dtype(self.bounds[0])
+        raise ValueError("min is only available for continuous boundss")
+
+    @property
+    def max(self):
+        if isinstance(self.bounds, (list, tuple)):
+            return self.dtype(self.bounds[1])
+        raise ValueError("max is only available for continuous boundss")
+
+
+class VariableFloat(Variable):
+    dtype = float
+
+    def __init__(self, name: str, value: float, bounds: Optional[Tuple[float, float]] = None,
+                 distribution: Optional[Distribution] = None, description: Optional[str] = None, fix: bool = False):
+        if bounds is None:
+            bounds = [self.dtype(value), self.dtype(value)]
+        super().__init__(name, value, float, bounds, distribution, description, fix)
+
+    @property
+    def min(self):
+        if isinstance(self.bounds, (list, tuple)):
+            return self.bounds[0]
+        raise ValueError("min is only available for continuous boundss")
+
+    @property
+    def max(self):
+        if isinstance(self.bounds, (list, tuple)):
+            return self.bounds[1]
+        raise ValueError("ma"
+                         "x is only available for continuous boundss")
+
+
+class VariableStr(Variable):
+    dtype = str
+
+    def __init__(self, name: str, value: str, bounds: Optional[Tuple[str, str]] = None,
+                 distribution: Optional[Distribution] = None, description: Optional[str] = None, fix: bool = False):
+        super().__init__(name, value, str, bounds, distribution, description, fix)
+
+    def sample_values(self, n: int = 1) -> List[Union[bool, float, int, str]]:
+        """
+        Generate n sampled values based on the defined distribution.
+        """
+        return [self.value] * n
+
+
+# Specialized DiscreteVariable Classes
+class DiscreteVariableInt(DiscreteVariable):
+    dtype = int
+
+    def __init__(self, name: str, value: int, discrete_values: List[int],
+                 description: Optional[str] = None, fix: bool = False):
+        super().__init__(name, value, int, discrete_values, description, fix)
+
+
+class DiscreteVariableFloat(DiscreteVariable):
+    dtype = float
+
+    def __init__(self, name: str, value: float, discrete_values: List[float],
+                 description: Optional[str] = None, fix: bool = False):
+        super().__init__(name, value, float, discrete_values, description, fix)
+
+
+class DiscreteVariableStr(DiscreteVariable):
+    dtype = str
+
+    def __init__(self, name: str, value: str, discrete_values: List[str],
+                 description: Optional[str] = None, fix: bool = False):
+        super().__init__(name, value, str, discrete_values, description, fix)
+
+
+class DiscreteVariableBool(DiscreteVariable):
+    dtype = bool
+
+    def __init__(self, name: str, value: bool, discrete_values: Optional[List[bool]] = None,
+                 description: Optional[str] = None, fix: bool = False):
+        discrete_values = [True, False]
+        super().__init__(name, value, bool, discrete_values, description, fix)
+
+
+class ParameterSet:
+    def __init__(self, name=None):
+        """
+        ParameterSet class to hold multiple _parameter in a dictionary.
+        """
+        self.name = name or "N.N."
+        self._parameter = {}
+
+    @property
+    def parameters(self):
+        return self._parameter
+
+    def update(self, parameter_list):
+        """
+
+        """
+        for parameter in parameter_list:
+            self._parameter[parameter.name] = parameter
+
+    def __getitem__(self, name):
+        return self.get(name)
+
+    def __contains__(self, item):
+        return item in self._parameter.keys()
+
+    def pop(self, pname):
+        try:
+            return self._parameter.pop(pname)
+        except KeyError as exp:
+            return None
+
+    def __iter__(self):
+        for x in self._parameter.values():
+            yield x
+
+    def items(self):
+        """
+        Provide a dict-like items view for parameters.
+
+        :return: Items view of parameters.
+        """
+        return self._parameter.items()
+
+    def keys(self):
+        """
+        Provide a dict-like keys view for parameters.
+
+        :return: Keys view of parameters.
+        """
+        return self._parameter.keys()
+
+    def values(self):
+        """
+        Provide a dict-like values view for parameters.
+
+        :return: Values view of parameters.
+        """
+        return self._parameter.values()
+
+    def get(self, name, default=None):
+        if self._parameter.get(name) is not None:
+            return self._parameter.get(name)
+        else:
+            return default
+
+    def set(self, parameter: Parameter):
+        """
+        Add a parameter to the set.
+
+        :param parameter: A Parameter object to add.
+        """
+        self._parameter[parameter.name] = parameter
+
+    def add_parameters(self, parameter_list: List):
+        """
+        Add a parameter to the set.
+
+        :param parameter: A Parameter object to add.
+        """
+        for parameter in parameter_list:
+            self._parameter[parameter.name] = parameter
+
+    def to_dict(self) -> Dict[str, Dict]:
+        """
+        Convert the entire parameter set to a dictionary.
+
+        :return: A dictionary of parameter dictionaries.
+        """
+        d = {"name": self.name}
+        d["parameter"] = {pname: param.to_dict() for pname, param in self._parameter.items()}
+        return d
+
+    @classmethod
+    def from_dict(cls, data: Dict):
+        """
+        Create a ParameterSet instance from a dictionary.
+
+        :param data: A dictionary containing parameter set data.
+        :return: A ParameterSet instance.
+        """
+        instance = cls(name=data.get("name", "N.N."))
+        parameters = data.get("parameter", {})
+        for pname, param_data in parameters.items():
+            parameter = Parameter.from_dict(param_data)  # Assuming Parameter has a from_dict method
+            instance.set(parameter)
+        return instance
+
+    def to_json(self) -> str:
+        """
+        Convert the ParameterSet to a JSON string.
+
+        :return: JSON string representation of the ParameterSet.
+        """
+        import json
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str):
+        """
+        Create a ParameterSet instance from a JSON string.
+
+        :param json_str: JSON string containing parameter set data.
+        :return: A ParameterSet instance.
+        """
+        data = json.loads(json_str)
+        return cls.from_dict(data)
+
+    def print(self):
+        for pname in sorted(self._parameter.keys()):
+            logger.info("%s", self._parameter[pname])
+
+    def __repr__(self):
+        return f"ParameterSet({self.name}|{list(self._parameter.keys())})"
+
+    def __str__(self):
+        return self.__repr__()

sdata/schema.py

@@ -0,0 +1,273 @@
+# -*- coding: utf-8 -*-
+"""Metadaten-Schemata / Templates zum *Vollqualifizieren* von Metadaten.
+
+Ein :class:`MetadataSchema` deklariert die erwarteten Attribute einer Datenklasse
+(Name, dtype, Einheit, required, ontology, …) und kann Metadaten dagegen
+*validieren* sowie *vervollständigen* (Defaults/Einheiten/Ontologie auffüllen).
+
+Reine-Python-Validierung ist immer verfügbar; mit dem optionalen Extra
+``[schema]`` (jsonschema) lässt sich zusätzlich gegen ein generiertes JSON Schema
+prüfen.
+"""
+from dataclasses import dataclass, field
+from typing import Any, List, Optional
+
+import numpy as np
+
+import sdata.dtypes as dtypes
+from sdata import units
+
+try:  # optionales Backend ([schema]=jsonschema)
+    import jsonschema as _jsonschema
+except ImportError:  # pragma: no cover - abhängig vom Environment
+    _jsonschema = None
+
+__all__ = ["AttrSpec", "MetadataSchema", "TableSchema", "ValidationReport"]
+
+# sdata-dtype -> JSON-Schema-Typ
+_JSON_TYPE = {
+    "str": "string", "uri": "string", "bytes": "string", "timestamp": "string",
+    "int": "integer", "float": "number", "bool": "boolean",
+    "list": "array", "json": "object",
+}
+
+
+def _is_empty(value):
+    if value is None or value == "":
+        return True
+    try:
+        return bool(np.isnan(value))
+    except (TypeError, ValueError):
+        return False
+
+
+@dataclass
+class AttrSpec:
+    """Spezifikation eines erwarteten Attributs."""
+    name: str
+    dtype: str = "str"
+    unit: str = "-"
+    required: bool = False
+    ontology: str = ""
+    description: str = ""
+    default: Any = None
+    allowed: Optional[List[Any]] = None
+
+    def to_dict(self):
+        return {"name": self.name, "dtype": self.dtype, "unit": self.unit,
+                "required": self.required, "ontology": self.ontology,
+                "description": self.description, "default": self.default,
+                "allowed": self.allowed}
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(**{k: d[k] for k in d if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class ValidationReport:
+    """Ergebnis einer Schema-Validierung (truthy, wenn ``ok``)."""
+    ok: bool
+    missing: List[str] = field(default_factory=list)
+    type_errors: List[tuple] = field(default_factory=list)
+    unit_errors: List[str] = field(default_factory=list)
+    enum_errors: List[str] = field(default_factory=list)
+    extra: List[str] = field(default_factory=list)
+    messages: List[str] = field(default_factory=list)
+
+    def __bool__(self):
+        return self.ok
+
+    def _repr_html_(self):
+        color = "#2e7d32" if self.ok else "#c62828"
+        status = "OK" if self.ok else "INVALID"
+        rows = ""
+        for label, items in [("missing", self.missing), ("type", self.type_errors),
+                             ("unit", self.unit_errors), ("enum", self.enum_errors),
+                             ("extra", self.extra), ("messages", self.messages)]:
+            if items:
+                rows += "<tr><td><b>{}</b></td><td>{}</td></tr>".format(label, items)
+        return ("<div><b style='color:{}'>ValidationReport: {}</b>"
+                "<table>{}</table></div>").format(color, status, rows)
+
+
+class MetadataSchema:
+    """Sammlung von :class:`AttrSpec` für eine Datenklasse."""
+
+    def __init__(self, name, specs, topology_class=None, sdata_class=None):
+        self.name = name
+        self.specs = list(specs)
+        self.topology_class = topology_class
+        self.sdata_class = sdata_class
+
+    @property
+    def _by_name(self):
+        return {s.name: s for s in self.specs}
+
+    def to_dict(self):
+        return {"name": self.name,
+                "topology_class": self.topology_class,
+                "sdata_class": self.sdata_class,
+                "specs": [s.to_dict() for s in self.specs]}
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(name=d.get("name", "N.N."),
+                   specs=[AttrSpec.from_dict(s) for s in d.get("specs", [])],
+                   topology_class=d.get("topology_class"),
+                   sdata_class=d.get("sdata_class"))
+
+    def validate(self, metadata):
+        """Prüfe ``metadata`` gegen das Schema (wirft nie; liefert ValidationReport)."""
+        report = ValidationReport(ok=True)
+        specs = self._by_name
+        for name, spec in specs.items():
+            attr = metadata.get(name)
+            if spec.required and (attr is None or _is_empty(attr.value)):
+                report.missing.append(name)
+            if attr is None:
+                continue
+            if dtypes.resolve(attr.dtype) != dtypes.resolve(spec.dtype):
+                report.type_errors.append((name, spec.dtype))
+            if spec.unit not in ("-", "") and \
+                    units.normalize_symbol(attr.unit) != units.normalize_symbol(spec.unit):
+                report.unit_errors.append(name)
+            if spec.allowed is not None and attr.value not in spec.allowed:
+                report.enum_errors.append(name)
+        for name in metadata.user_attributes.keys():
+            if name not in specs:
+                report.extra.append(name)
+        report.ok = not (report.missing or report.type_errors
+                         or report.unit_errors or report.enum_errors)
+        return report
+
+    def apply(self, metadata):
+        """Vervollständige ``metadata`` in-place: fehlende Attribute aus Defaults
+        anlegen, vorhandene um Einheit/Ontologie/Beschreibung ergänzen. Gibt
+        ``metadata`` zurück (für nicht-destruktiv: ``schema.apply(md.copy())``)."""
+        for spec in self.specs:
+            attr = metadata.get(spec.name)
+            if attr is None:
+                metadata.set_attr(spec.name, spec.default, dtype=spec.dtype,
+                                  unit=spec.unit, ontology=spec.ontology,
+                                  description=spec.description)
+                continue
+            if attr.unit in ("-", "", None) and spec.unit not in ("-", ""):
+                attr.unit = spec.unit
+            if not attr.ontology and spec.ontology:
+                attr.ontology = spec.ontology
+            if not attr.description and spec.description:
+                attr.description = spec.description
+        return metadata
+
+    def to_json_schema(self):
+        """Generiere ein JSON Schema (Draft 2020-12) für ``metadata.get_udict()``."""
+        properties = {}
+        required = []
+        for spec in self.specs:
+            prop = {"type": _JSON_TYPE.get(dtypes.resolve(spec.dtype), "string")}
+            if spec.description:
+                prop["description"] = spec.description
+            if spec.allowed is not None:
+                prop["enum"] = spec.allowed
+            properties[spec.name] = prop
+            if spec.required:
+                required.append(spec.name)
+        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema",
+                  "type": "object", "properties": properties}
+        if required:
+            schema["required"] = required
+        return schema
+
+    def validate_jsonschema(self, metadata):
+        """Validiere via ``jsonschema`` (falls installiert), sonst native ``validate``."""
+        if _jsonschema is None:
+            return self.validate(metadata)
+        try:
+            _jsonschema.validate(metadata.get_udict(), self.to_json_schema())
+            return ValidationReport(ok=True)
+        except _jsonschema.ValidationError as exp:
+            return ValidationReport(ok=False, messages=[str(exp).splitlines()[0]])
+
+
+class TableSchema:
+    """Spalten-Schema für eine :class:`~sdata.sclass.dataframe.DataFrame`.
+
+    Deklariert je Spalte den erwarteten ``name``/``dtype``/``unit``/``required``
+    (wiederverwendet :class:`AttrSpec`) und kann ein DataFrame dagegen *validieren*
+    sowie dessen ``column_metadata`` aus dem Schema *vervollständigen*.
+
+    Der dtype wird gegen die tatsächlichen ``df.dtypes`` geprüft, die Einheit gegen
+    die ``column_metadata``-Annotation der Spalte.
+    """
+
+    def __init__(self, name, columns):
+        self.name = name
+        self.columns = list(columns)
+
+    @property
+    def _by_name(self):
+        return {s.name: s for s in self.columns}
+
+    def to_dict(self):
+        return {"name": self.name, "columns": [s.to_dict() for s in self.columns]}
+
+    @classmethod
+    def from_dict(cls, d):
+        return cls(name=d.get("name", "N.N."),
+                   columns=[AttrSpec.from_dict(s) for s in d.get("columns", [])])
+
+    def validate(self, dataframe):
+        """Prüfe ein :class:`DataFrame` gegen das Spalten-Schema.
+
+        Wirft nie; liefert einen :class:`ValidationReport` mit fehlenden Spalten,
+        dtype-Abweichungen (gegen ``df.dtypes``), Einheiten-Abweichungen (gegen
+        ``column_metadata``) und zusätzlichen (nicht spezifizierten) Spalten.
+        """
+        report = ValidationReport(ok=True)
+        df = dataframe.df
+        cm = dataframe.column_metadata
+        present = {str(c) for c in df.columns}
+        specs = self._by_name
+        for name, spec in specs.items():
+            if name not in present:
+                report.missing.append(name)
+                continue
+            actual = dtypes.resolve(str(df[name].dtype))
+            if actual is not None and dtypes.resolve(spec.dtype) != actual:
+                report.type_errors.append((name, spec.dtype))
+            if spec.unit not in ("-", ""):
+                attr = cm.get(name)
+                attr_unit = attr.unit if attr is not None else "-"
+                if units.normalize_symbol(attr_unit) != units.normalize_symbol(spec.unit):
+                    report.unit_errors.append(name)
+        for name in present:
+            if name not in specs:
+                report.extra.append(name)
+        report.ok = not (report.missing or report.type_errors or report.unit_errors)
+        return report
+
+    def apply(self, dataframe):
+        """Vervollständige ``column_metadata`` in-place aus dem Schema.
+
+        Für jede im df vorhandene Schema-Spalte: fehlende Annotation anlegen bzw.
+        leere ``unit``/``ontology``/``description`` auffüllen. Gibt ``dataframe`` zurück.
+        """
+        cm = dataframe.column_metadata
+        present = {str(c) for c in dataframe.df.columns}
+        for spec in self.columns:
+            if spec.name not in present:
+                continue
+            attr = cm.get(spec.name)
+            if attr is None:
+                cm.set_attr(spec.name, spec.default, dtype=spec.dtype,
+                            unit=spec.unit, ontology=spec.ontology,
+                            description=spec.description)
+                continue
+            if attr.unit in ("-", "", None) and spec.unit not in ("-", ""):
+                attr.unit = spec.unit
+            if not attr.ontology and spec.ontology:
+                attr.ontology = spec.ontology
+            if not attr.description and spec.description:
+                attr.description = spec.description
+        return dataframe