sdata 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdata/__init__.py +140 -0
- sdata/base.py +684 -0
- sdata/contrib/__init__.py +0 -0
- sdata/contrib/piexif/__init__.py +12 -0
- sdata/contrib/piexif/_common.py +94 -0
- sdata/contrib/piexif/_dump.py +346 -0
- sdata/contrib/piexif/_exceptions.py +2 -0
- sdata/contrib/piexif/_exif.py +638 -0
- sdata/contrib/piexif/_insert.py +60 -0
- sdata/contrib/piexif/_load.py +270 -0
- sdata/contrib/piexif/_remove.py +56 -0
- sdata/contrib/piexif/_transplant.py +45 -0
- sdata/contrib/piexif/_webp.py +240 -0
- sdata/contrib/piexif/helper.py +66 -0
- sdata/contrib/ranger/__init__.py +8 -0
- sdata/contrib/ranger/collections/__init__.py +0 -0
- sdata/contrib/ranger/collections/rangebucketmap.py +346 -0
- sdata/contrib/ranger/collections/rangemap.py +396 -0
- sdata/contrib/ranger/collections/rangeset.py +483 -0
- sdata/contrib/ranger/range/__init__.py +0 -0
- sdata/contrib/ranger/range/cut.py +285 -0
- sdata/contrib/ranger/range/range.py +644 -0
- sdata/contrib/sobol_seq.py +394 -0
- sdata/contrib/sortedcontainers/LICENSE +13 -0
- sdata/contrib/sortedcontainers/__init__.py +74 -0
- sdata/contrib/sortedcontainers/sorteddict.py +789 -0
- sdata/contrib/sortedcontainers/sortedlist.py +2646 -0
- sdata/contrib/sortedcontainers/sortedset.py +733 -0
- sdata/contrib/sqlitedict.py +598 -0
- sdata/contrib/timeflake/__init__.py +40 -0
- sdata/contrib/timeflake/flake.py +69 -0
- sdata/contrib/timeflake/utils.py +47 -0
- sdata/deprecated/__init__.py +10 -0
- sdata/deprecated/data.py +1503 -0
- sdata/did/__init__.py +129 -0
- sdata/did/_http.py +67 -0
- sdata/did/base58btc.py +49 -0
- sdata/did/did_github.py +152 -0
- sdata/did/did_peer4.py +155 -0
- sdata/did/did_web.py +222 -0
- sdata/did/diddoc.py +75 -0
- sdata/did/ed25519.py +225 -0
- sdata/did/eddsa.py +107 -0
- sdata/did/errors.py +40 -0
- sdata/did/issue_vc.py +193 -0
- sdata/did/jose.py +126 -0
- sdata/did/keys.py +104 -0
- sdata/did/utils_didkey.py +161 -0
- sdata/did/verify_vp.py +158 -0
- sdata/doe.py +101 -0
- sdata/dtypes.py +510 -0
- sdata/experiments/__init__.py +331 -0
- sdata/experiments/blob0.py +131 -0
- sdata/experiments/file.py +42 -0
- sdata/experiments/filegroup.py +144 -0
- sdata/experiments/image_old.py +313 -0
- sdata/experiments/ks2.py +15 -0
- sdata/experiments/lapshear.py +8 -0
- sdata/experiments/material.py +7 -0
- sdata/experiments/parameters_deprecated.py +394 -0
- sdata/experiments/workbook.py +226 -0
- sdata/format.py +125 -0
- sdata/imagemeta.py +533 -0
- sdata/interactive.py +118 -0
- sdata/iolib/__init__.py +51 -0
- sdata/iolib/hashtree.py +264 -0
- sdata/iolib/json1sqlitestore.py +554 -0
- sdata/iolib/jsonsqlitestore.py +704 -0
- sdata/iolib/pud.py +242 -0
- sdata/iolib/reader.py +262 -0
- sdata/iolib/writer.py +419 -0
- sdata/metadata.py +882 -0
- sdata/parameter.py +414 -0
- sdata/schema.py +273 -0
- sdata/sclass/__init__.py +111 -0
- sdata/sclass/bfo_classes.py +362 -0
- sdata/sclass/blob.py +362 -0
- sdata/sclass/content.py +87 -0
- sdata/sclass/dataframe.py +1251 -0
- sdata/sclass/dataframegroup.py +187 -0
- sdata/sclass/dependency_graph.py +85 -0
- sdata/sclass/filereference.py +124 -0
- sdata/sclass/image.py +218 -0
- sdata/sclass/lazy_namespace.py +155 -0
- sdata/sclass/process.py +383 -0
- sdata/sclass/process_graph.py +221 -0
- sdata/sclass/zipfile.py +135 -0
- sdata/semantic.py +346 -0
- sdata/suuid.py +136 -0
- sdata/timestamp.py +336 -0
- sdata/tools.py +14 -0
- sdata/units.py +602 -0
- sdata/vocab.py +149 -0
- sdata-1.5.0.dist-info/METADATA +264 -0
- sdata-1.5.0.dist-info/RECORD +98 -0
- sdata-1.5.0.dist-info/WHEEL +5 -0
- sdata-1.5.0.dist-info/licenses/LICENSE-MIT +21 -0
- sdata-1.5.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Geordnete Sammlung benannter sdata-DataFrames (RFC 0011).
|
|
2
|
+
|
|
3
|
+
Anders als früher hält die Gruppe ihre Mitglieder als vollwertige
|
|
4
|
+
:class:`~sdata.sclass.dataframe.DataFrame` (mit reichem ``Metadata``/
|
|
5
|
+
``column_metadata`` — Einheit/Label/Ontologie/dtype, RFC 0006), nicht mehr als
|
|
6
|
+
rohe pandas-Frames mit einem ``{label, unit}``-dict. Die öffentliche API
|
|
7
|
+
(``add_dataframe``/``get_dataframe``/``get_column_metadata``/…) bleibt
|
|
8
|
+
abwärtskompatibel; ``from_dict`` liest **beide** Serialisierungs-Layouts (das alte
|
|
9
|
+
``{parquet, column_metadata}`` und das neue ``{sdata: <DataFrame.to_dict>}``).
|
|
10
|
+
"""
|
|
11
|
+
import base64
|
|
12
|
+
import io
|
|
13
|
+
import logging
|
|
14
|
+
from typing import Any, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from sdata.base import Base
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class DataFrameGroup(Base):
|
|
24
|
+
"""Sammlung benannter :class:`~sdata.sclass.dataframe.DataFrame` (eine flache Ebene).
|
|
25
|
+
|
|
26
|
+
Die Mitglieder liegen in ``self.data['dataframes']`` als
|
|
27
|
+
:class:`~sdata.sclass.dataframe.DataFrame`-Objekte; jedes trägt seine volle
|
|
28
|
+
Spaltensemantik selbst (kein separates ``{label, unit}``-dict mehr).
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
32
|
+
"""Initialize the group (forwards ``**kwargs`` to :class:`Base`)."""
|
|
33
|
+
super().__init__(**kwargs)
|
|
34
|
+
if 'dataframes' not in self.data:
|
|
35
|
+
self.data['dataframes'] = {}
|
|
36
|
+
|
|
37
|
+
# ------------------------------------------------------------------ helpers
|
|
38
|
+
@staticmethod
|
|
39
|
+
def _as_dataframe(df, key=None, column_metadata=None):
|
|
40
|
+
"""Wrap a pandas ``df`` (or pass a :class:`DataFrame` through) and annotate columns."""
|
|
41
|
+
from sdata.sclass.dataframe import DataFrame
|
|
42
|
+
sdf = df if isinstance(df, DataFrame) else DataFrame(df=df, name=key or "df")
|
|
43
|
+
if column_metadata:
|
|
44
|
+
for col, meta in column_metadata.items():
|
|
45
|
+
sdf.set_column(col, label=meta.get('label'), unit=meta.get('unit'))
|
|
46
|
+
return sdf
|
|
47
|
+
|
|
48
|
+
# -------------------------------------------------------------- public API
|
|
49
|
+
def add(self, df, key: Optional[str] = None, overwrite: bool = False):
|
|
50
|
+
"""Add a :class:`DataFrame` (or pandas ``df``); returns the stored member.
|
|
51
|
+
|
|
52
|
+
:param df: a :class:`~sdata.sclass.dataframe.DataFrame` or a pandas ``DataFrame``.
|
|
53
|
+
:param key: storage key (defaults to the member's ``sname``).
|
|
54
|
+
:param overwrite: allow replacing an existing key.
|
|
55
|
+
:raises ValueError: if the key exists and ``overwrite`` is False.
|
|
56
|
+
"""
|
|
57
|
+
sdf = self._as_dataframe(df, key=key)
|
|
58
|
+
key = key or sdf.sname
|
|
59
|
+
if key in self.data['dataframes'] and not overwrite:
|
|
60
|
+
raise ValueError(f"DataFrame with key '{key}' already exists. "
|
|
61
|
+
"Set overwrite=True to replace.")
|
|
62
|
+
self.data['dataframes'][key] = sdf
|
|
63
|
+
logger.debug("Added DataFrame '%s' to %s", key, self.sname)
|
|
64
|
+
return sdf
|
|
65
|
+
|
|
66
|
+
def add_dataframe(
|
|
67
|
+
self,
|
|
68
|
+
key: str,
|
|
69
|
+
df: pd.DataFrame,
|
|
70
|
+
column_metadata: Optional[Dict[str, Dict[str, str]]] = None,
|
|
71
|
+
overwrite: bool = False,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Add a pandas ``df`` under ``key`` (backward-compatible API).
|
|
74
|
+
|
|
75
|
+
:param column_metadata: optional ``{colname: {'label': ..., 'unit': ...}}``;
|
|
76
|
+
if given, must cover **exactly** the df columns and each entry must have
|
|
77
|
+
both ``label`` and ``unit``. The annotations are written through to the
|
|
78
|
+
member's per-column :class:`~sdata.metadata.Metadata`.
|
|
79
|
+
:raises ValueError: on key clash without ``overwrite`` or invalid metadata.
|
|
80
|
+
"""
|
|
81
|
+
if key in self.data['dataframes'] and not overwrite:
|
|
82
|
+
raise ValueError(f"DataFrame with key '{key}' already exists. "
|
|
83
|
+
"Set overwrite=True to replace.")
|
|
84
|
+
if column_metadata is not None:
|
|
85
|
+
if set(column_metadata.keys()) != set(df.columns):
|
|
86
|
+
raise ValueError("column_metadata keys must match the DataFrame's columns exactly.")
|
|
87
|
+
for meta in column_metadata.values():
|
|
88
|
+
if not all(k in meta for k in ['label', 'unit']):
|
|
89
|
+
raise ValueError("Each column_metadata entry must have 'label' and 'unit' keys.")
|
|
90
|
+
sdf = self._as_dataframe(df, key=key, column_metadata=column_metadata)
|
|
91
|
+
self.data['dataframes'][key] = sdf
|
|
92
|
+
logger.debug("Added DataFrame '%s' to %s", key, self.sname)
|
|
93
|
+
|
|
94
|
+
def get(self, key: str):
|
|
95
|
+
"""Return the stored :class:`DataFrame` member (or ``None``)."""
|
|
96
|
+
return self.data['dataframes'].get(key)
|
|
97
|
+
|
|
98
|
+
def get_dataframe(self, key: str) -> Optional[pd.DataFrame]:
|
|
99
|
+
"""Return the member's pandas ``DataFrame`` (backward-compatible), or ``None``."""
|
|
100
|
+
member = self.data['dataframes'].get(key)
|
|
101
|
+
return member.df if member is not None else None
|
|
102
|
+
|
|
103
|
+
def get_column_metadata(self, key: str) -> Optional[Dict[str, Dict[str, str]]]:
|
|
104
|
+
"""Return ``{colname: {'label': str, 'unit': str}}`` for a member (backward-compatible).
|
|
105
|
+
|
|
106
|
+
Derived from the member's per-column :class:`~sdata.metadata.Metadata`.
|
|
107
|
+
"""
|
|
108
|
+
member = self.data['dataframes'].get(key)
|
|
109
|
+
if member is None:
|
|
110
|
+
return None
|
|
111
|
+
out: Dict[str, Dict[str, str]] = {}
|
|
112
|
+
for col in member.df.columns:
|
|
113
|
+
attr = member.get_column(str(col))
|
|
114
|
+
out[str(col)] = {
|
|
115
|
+
'label': attr.label if attr is not None else '',
|
|
116
|
+
'unit': attr.unit if attr is not None else '',
|
|
117
|
+
}
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
def remove_dataframe(self, key: str) -> None:
|
|
121
|
+
"""Remove a member by key.
|
|
122
|
+
|
|
123
|
+
:raises KeyError: if the key does not exist.
|
|
124
|
+
"""
|
|
125
|
+
if key not in self.data['dataframes']:
|
|
126
|
+
raise KeyError(f"DataFrame with key '{key}' not found.")
|
|
127
|
+
del self.data['dataframes'][key]
|
|
128
|
+
logger.debug("Removed DataFrame '%s' from %s", key, self.sname)
|
|
129
|
+
|
|
130
|
+
def list_dataframes(self) -> List[str]:
|
|
131
|
+
"""List all member keys."""
|
|
132
|
+
return list(self.data['dataframes'].keys())
|
|
133
|
+
|
|
134
|
+
def items(self):
|
|
135
|
+
"""Iterate ``(key, DataFrame)`` over the members (order preserved)."""
|
|
136
|
+
return self.data['dataframes'].items()
|
|
137
|
+
|
|
138
|
+
# ------------------------------------------------------------ serialization
|
|
139
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
140
|
+
"""Serialize each member via its own lossless :meth:`DataFrame.to_dict`.
|
|
141
|
+
|
|
142
|
+
Layout: ``data.dataframes[key] = {'sdata': <DataFrame.to_dict()>}`` — the
|
|
143
|
+
member keeps its full metadata/column_metadata (RFC 0011, no lossy
|
|
144
|
+
``{label, unit}`` reduction).
|
|
145
|
+
"""
|
|
146
|
+
data_copy = dict(self.data)
|
|
147
|
+
data_copy['dataframes'] = {
|
|
148
|
+
key: {'sdata': member.to_dict()}
|
|
149
|
+
for key, member in self.data['dataframes'].items()
|
|
150
|
+
}
|
|
151
|
+
result = super().to_dict()
|
|
152
|
+
result['data'] = data_copy
|
|
153
|
+
return result
|
|
154
|
+
|
|
155
|
+
@classmethod
|
|
156
|
+
def from_dict(cls, d: Dict[str, Any]) -> 'DataFrameGroup':
|
|
157
|
+
"""Reconstruct a group, reading **both** the new and the legacy layout.
|
|
158
|
+
|
|
159
|
+
* new: ``{'sdata': <DataFrame.to_dict>}`` → :meth:`DataFrame.from_dict`;
|
|
160
|
+
* legacy: ``{'parquet': <b64>, 'column_metadata': {col: {label, unit}}}``
|
|
161
|
+
→ decode the Parquet payload and lift the flat ``{label, unit}`` dict onto
|
|
162
|
+
the member's per-column metadata.
|
|
163
|
+
"""
|
|
164
|
+
from sdata.sclass.dataframe import DataFrame
|
|
165
|
+
from sdata.format import ensure_compatible
|
|
166
|
+
from sdata.metadata import Metadata
|
|
167
|
+
# cls() statt super().from_dict: Base.from_dict baut über sdata_factory eine
|
|
168
|
+
# dynamische Klasse ohne die Group-Methoden (wie DataFrame.from_dict).
|
|
169
|
+
d = ensure_compatible(d) # RFC 0010: Version prüfen
|
|
170
|
+
instance = cls()
|
|
171
|
+
instance.metadata = Metadata.from_dict(d.get("metadata", {}))
|
|
172
|
+
instance.description = d.get("description", "")
|
|
173
|
+
raw = dict(d.get("data", {}))
|
|
174
|
+
members: Dict[str, Any] = {}
|
|
175
|
+
for key, entry in raw.get('dataframes', {}).items():
|
|
176
|
+
if isinstance(entry, dict) and 'sdata' in entry:
|
|
177
|
+
members[key] = DataFrame.from_dict(entry['sdata'])
|
|
178
|
+
elif isinstance(entry, dict) and 'parquet' in entry: # Legacy-Layout
|
|
179
|
+
df = pd.read_parquet(io.BytesIO(base64.b64decode(entry['parquet'])),
|
|
180
|
+
engine='pyarrow')
|
|
181
|
+
members[key] = cls._as_dataframe(
|
|
182
|
+
df, key=key, column_metadata=entry.get('column_metadata'))
|
|
183
|
+
else: # bereits DataFrame
|
|
184
|
+
members[key] = entry
|
|
185
|
+
raw['dataframes'] = members
|
|
186
|
+
instance.data = raw
|
|
187
|
+
return instance
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from typing import Dict, Set, List, Iterable, TypeVar, Hashable
|
|
2
|
+
|
|
3
|
+
T = TypeVar('T', bound=Hashable)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CircularDependencyError(ValueError):
|
|
7
|
+
"""Exception raised when circular dependencies are detected during topological sort."""
|
|
8
|
+
|
|
9
|
+
def __init__(self, remaining_graph: Dict[T, Set[T]]):
|
|
10
|
+
message = (
|
|
11
|
+
"Circular dependencies exist among these items: "
|
|
12
|
+
f"{{{', '.join(f'{key}: {sorted(value)}' for key, value in sorted(remaining_graph.items()))}}}"
|
|
13
|
+
)
|
|
14
|
+
super().__init__(message)
|
|
15
|
+
self.remaining_graph = remaining_graph
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def toposort(graph: Dict[T, Iterable[T]]) -> Iterable[Set[T]]:
|
|
19
|
+
"""Perform a topological sort on a directed acyclic graph (DAG).
|
|
20
|
+
|
|
21
|
+
Dependencies are expressed as a dictionary where keys are nodes and values are
|
|
22
|
+
iterables of dependent nodes (nodes that the key depends on). The output is a
|
|
23
|
+
generator yielding sets of nodes in topological order. The first set contains
|
|
24
|
+
nodes with no dependencies, and each subsequent set contains nodes that depend
|
|
25
|
+
only on nodes in preceding sets.
|
|
26
|
+
|
|
27
|
+
If the graph contains cycles, a CircularDependencyError is raised.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
graph: A dictionary representing the dependency graph.
|
|
31
|
+
|
|
32
|
+
Yields:
|
|
33
|
+
Sets of nodes in topological order.
|
|
34
|
+
|
|
35
|
+
Raises:
|
|
36
|
+
CircularDependencyError: If a cycle is detected in the graph.
|
|
37
|
+
"""
|
|
38
|
+
if not graph:
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
# Create a copy of the graph, converting dependencies to sets and removing self-dependencies.
|
|
42
|
+
graph = {node: set(dep for dep in deps if dep != node) for node, deps in graph.items()}
|
|
43
|
+
|
|
44
|
+
# Identify nodes that appear only as dependencies (no outgoing edges defined).
|
|
45
|
+
extra_nodes = {dep for deps in graph.values() for dep in deps} - set(graph)
|
|
46
|
+
graph.update({node: set() for node in extra_nodes})
|
|
47
|
+
|
|
48
|
+
while True:
|
|
49
|
+
# Find nodes with no remaining dependencies.
|
|
50
|
+
independent_nodes = {node for node, deps in graph.items() if not deps}
|
|
51
|
+
if not independent_nodes:
|
|
52
|
+
break
|
|
53
|
+
yield independent_nodes
|
|
54
|
+
# Remove processed nodes from the graph.
|
|
55
|
+
graph = {
|
|
56
|
+
node: (deps - independent_nodes)
|
|
57
|
+
for node, deps in graph.items()
|
|
58
|
+
if node not in independent_nodes
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if graph:
|
|
62
|
+
raise CircularDependencyError(graph)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def toposort_flatten(graph: Dict[T, Iterable[T]], sort: bool = True) -> List[T]:
|
|
66
|
+
"""Flatten the topological sort into a single list.
|
|
67
|
+
|
|
68
|
+
This function consumes the generator from toposort() and appends the nodes
|
|
69
|
+
from each level to a list. If sort is True, nodes within each level are sorted
|
|
70
|
+
for deterministic order.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
graph: A dictionary representing the dependency graph.
|
|
74
|
+
sort: Whether to sort nodes within each topological level (default: True).
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
A list of nodes in topological order.
|
|
78
|
+
|
|
79
|
+
Raises:
|
|
80
|
+
CircularDependencyError: If a cycle is detected in the graph.
|
|
81
|
+
"""
|
|
82
|
+
result: List[T] = []
|
|
83
|
+
for level in toposort(graph):
|
|
84
|
+
result.extend(sorted(level) if sort else list(level))
|
|
85
|
+
return result
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import io
|
|
3
|
+
import os
|
|
4
|
+
import base64
|
|
5
|
+
from typing import Any, Dict, Optional, Union, Iterable
|
|
6
|
+
import logging
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from sdata import SUUID
|
|
9
|
+
from sdata.metadata import Metadata, Attribute
|
|
10
|
+
from sdata.sclass.blob import Blob
|
|
11
|
+
from sdata.base import Base
|
|
12
|
+
import hashlib
|
|
13
|
+
import datetime
|
|
14
|
+
from sdata.timestamp import get_utc_timestamp
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FileReference(Blob):
|
|
20
|
+
SDATA_CLS = "sdata.sclass.filereference.FileReference"
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
filetype: Optional[str] = '', # Default to generic binary, e.g., '.pdf', '.png', '.jpg'
|
|
25
|
+
**kwargs: Any
|
|
26
|
+
) -> None:
|
|
27
|
+
"""
|
|
28
|
+
Initialize FileReference as a :class:`~sdata.sclass.blob.Blob` over a file URI.
|
|
29
|
+
|
|
30
|
+
The file path is kept as the Blob ``uri`` content, so the reference also
|
|
31
|
+
provides ``content_bytes``/``open``/``exists``/``verify``/``size`` from Blob.
|
|
32
|
+
|
|
33
|
+
:param filetype: vestigial; the file suffix is derived from ``name``.
|
|
34
|
+
:param kwargs: Keyword arguments passed to Base (e.g. ``name`` = file path).
|
|
35
|
+
"""
|
|
36
|
+
filepath = kwargs.get("name", "noname")
|
|
37
|
+
basename = os.path.basename(filepath)
|
|
38
|
+
kwargs["name"] = basename
|
|
39
|
+
path = Path(filepath)
|
|
40
|
+
# Blob-Grundlage: die Datei als uri-Content -> behält den Pfad (zuvor verloren)
|
|
41
|
+
super().__init__(content_type="uri", value=filepath,
|
|
42
|
+
filetype=path.suffix.lstrip(".").lower() or "binary", **kwargs)
|
|
43
|
+
|
|
44
|
+
self.metadata.add("_sdata_name", basename, description="base name of the file without folder")
|
|
45
|
+
self.metadata.add("_sdata_stem", path.stem, description="file name without suffix")
|
|
46
|
+
self.metadata.add("_sdata_filetype", path.suffix.lower(), description="file suffix")
|
|
47
|
+
self.metadata.add("_sdata_filesize", None, dtype="int", description="file size in bytes")
|
|
48
|
+
self.metadata.add("_sdata_filectime", None, description="UTC file creation date")
|
|
49
|
+
self.metadata.add("_sdata_sha3_256", None, description="SHA-3-256")
|
|
50
|
+
|
|
51
|
+
@staticmethod
|
|
52
|
+
def get_hash(filepath):
|
|
53
|
+
sh = hashlib.sha3_256()
|
|
54
|
+
with open(filepath, "rb") as fh:
|
|
55
|
+
sh.update(fh.read())
|
|
56
|
+
content_hash = sh.hexdigest()
|
|
57
|
+
return content_hash
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def from_file(cls, filepath: str):
|
|
61
|
+
suuid = SUUID.from_file(class_name="FileReference", filepath=filepath)
|
|
62
|
+
fr = cls(name=filepath, suuid=suuid)
|
|
63
|
+
|
|
64
|
+
fr.metadata.add("_sdata_sha3_256", cls.get_hash(filepath))
|
|
65
|
+
|
|
66
|
+
p = Path(filepath)
|
|
67
|
+
ctime = p.stat().st_ctime
|
|
68
|
+
fr.metadata.add("_sdata_filectime", get_utc_timestamp(datetime.datetime.fromtimestamp(ctime)))
|
|
69
|
+
|
|
70
|
+
fr.metadata.add("_sdata_filesize", p.stat().st_size)
|
|
71
|
+
|
|
72
|
+
return fr
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def filetype(self) -> str:
|
|
76
|
+
# robust: _sdata_filetype wird erst nach super().__init__() gesetzt, Blobs
|
|
77
|
+
# _autofill_metadata fragt filetype aber bereits währenddessen ab.
|
|
78
|
+
attr = self.metadata.get("_sdata_filetype")
|
|
79
|
+
return attr.value if attr is not None else ""
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def stem(self) -> str:
|
|
83
|
+
return self.metadata.get("_sdata_stem").value
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def ctime(self) -> str:
|
|
87
|
+
return self.metadata.get("_sdata_filectime").value
|
|
88
|
+
|
|
89
|
+
def to_dataframe(self):
|
|
90
|
+
df = self.metadata.to_dataframe()[["value"]].T
|
|
91
|
+
df.index = [self.sname]
|
|
92
|
+
return df
|
|
93
|
+
|
|
94
|
+
class FileReferences(Base):
|
|
95
|
+
SDATA_CLS = "sdata.sclass.filereference.FileReferences"
|
|
96
|
+
|
|
97
|
+
def __init__(
|
|
98
|
+
self,
|
|
99
|
+
**kwargs: Any
|
|
100
|
+
) -> None:
|
|
101
|
+
"""
|
|
102
|
+
Initialize FileReferences with content type, value, and filetype.
|
|
103
|
+
:param kwargs: Keyword arguments passed to Base.__init__ (e.g., name, description).
|
|
104
|
+
:raises ValueError: If invalid content_type or mismatched value type.
|
|
105
|
+
"""
|
|
106
|
+
super().__init__(**kwargs)
|
|
107
|
+
self.filereferences = {}
|
|
108
|
+
|
|
109
|
+
def add(self, filereference: FileReference) -> None:
|
|
110
|
+
self.filereferences[filereference.sname] = filereference
|
|
111
|
+
|
|
112
|
+
def get_filereferences(self) -> Iterable[FileReference]:
|
|
113
|
+
return self.filereferences.values()
|
|
114
|
+
|
|
115
|
+
def get_filereferences_df(self) -> pd.DataFrame:
|
|
116
|
+
filereferences = []
|
|
117
|
+
for sname, filereference in self.filereferences.items():
|
|
118
|
+
filereferences.append(filereference.to_dataframe())
|
|
119
|
+
return pd.concat(filereferences)
|
|
120
|
+
|
|
121
|
+
if __name__ == '__main__':
|
|
122
|
+
fr = FileReference("/tmp/test.xlsx")
|
|
123
|
+
|
|
124
|
+
print(fr)
|
sdata/sclass/image.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Image — ein :class:`~sdata.sclass.blob.Blob` über Bild-Inhalt.
|
|
3
|
+
|
|
4
|
+
Der Bild-Inhalt liegt als Blob-Content (``uri`` für Dateien, ``bytes`` für
|
|
5
|
+
In-Memory-Daten). sdata-Metadaten werden **format-übergreifend nativ** in die
|
|
6
|
+
Bilddatei eingebettet (PNG/JPEG/JP2/GIF/WebP/TIFF) — über :mod:`sdata.imagemeta`,
|
|
7
|
+
das ohne Pillow auskommt. Formate **ohne** nativen Metadaten-Träger (z. B. BMP)
|
|
8
|
+
erhalten einen verlustfreien ``<filepath>.meta.json``-**Sidecar**; die ``save``/
|
|
9
|
+
``from_file``-API ist für alle Formate identisch. Pillow wird nur lazy zum
|
|
10
|
+
**Dekodieren/Transkodieren** der Pixel genutzt (:attr:`Image.pil`/
|
|
11
|
+
:meth:`Image.to_numpy`/:meth:`Image.save` bei Formatwechsel) und ist optional
|
|
12
|
+
(``pip install pillow``).
|
|
13
|
+
"""
|
|
14
|
+
import io
|
|
15
|
+
import os
|
|
16
|
+
import logging
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
from sdata import imagemeta
|
|
22
|
+
from sdata.suuid import SUUID
|
|
23
|
+
from sdata.metadata import Metadata
|
|
24
|
+
from sdata.sclass.blob import Blob
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
import PIL.Image
|
|
28
|
+
except ImportError: # pragma: no cover - optionale Abhängigkeit (Pillow)
|
|
29
|
+
PIL = None
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Image(Blob):
|
|
35
|
+
"""Image object based on :class:`~sdata.sclass.blob.Blob`."""
|
|
36
|
+
|
|
37
|
+
SDATA_CLS = "sdata.sclass.image.Image"
|
|
38
|
+
|
|
39
|
+
#: Datei-Endung → (Pillow-Format, :mod:`sdata.imagemeta`-Format) für :meth:`save`.
|
|
40
|
+
_SUFFIX_FORMATS = {
|
|
41
|
+
"png": ("PNG", "png"),
|
|
42
|
+
"jpg": ("JPEG", "jpeg"), "jpeg": ("JPEG", "jpeg"), "jpe": ("JPEG", "jpeg"),
|
|
43
|
+
"jp2": ("JPEG2000", "jp2"), "j2k": ("JPEG2000", "jp2"),
|
|
44
|
+
"jpf": ("JPEG2000", "jp2"), "jpx": ("JPEG2000", "jp2"),
|
|
45
|
+
"gif": ("GIF", "gif"),
|
|
46
|
+
"webp": ("WEBP", "webp"),
|
|
47
|
+
"tif": ("TIFF", "tiff"), "tiff": ("TIFF", "tiff"),
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_file(cls, filepath, project=None, ns_name=None, **kwargs):
|
|
52
|
+
"""Create an Image referencing an image file (kept as ``uri`` content).
|
|
53
|
+
|
|
54
|
+
Any sdata metadata is read back and merged: natively embedded
|
|
55
|
+
(PNG/JPEG/JP2/GIF/WebP/TIFF, Pillow-free) and/or from an adjacent
|
|
56
|
+
``<filepath>.meta.json`` sidecar (for formats without a native container).
|
|
57
|
+
|
|
58
|
+
:param filepath: path to the image file.
|
|
59
|
+
:param project: namespace for the deterministic SUUID (alias of ``ns_name``).
|
|
60
|
+
:param ns_name: namespace for the deterministic SUUID.
|
|
61
|
+
:param kwargs: forwarded to :class:`Blob`/:class:`~sdata.base.Base`.
|
|
62
|
+
:return: an :class:`Image` instance.
|
|
63
|
+
"""
|
|
64
|
+
suuid = SUUID.from_file("Image", filepath, ns_name=ns_name or project)
|
|
65
|
+
suffix = Path(filepath).suffix.lstrip(".").lower() or "binary"
|
|
66
|
+
img = cls(content_type="uri", value=filepath, filetype=suffix,
|
|
67
|
+
name=os.path.basename(filepath), suuid=suuid, **kwargs)
|
|
68
|
+
img._load_embedded_metadata()
|
|
69
|
+
img._load_sidecar_metadata(filepath)
|
|
70
|
+
return img
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def from_bytes(cls, name, image_data, project=None, **kwargs):
|
|
74
|
+
"""Create an Image from in-memory image bytes.
|
|
75
|
+
|
|
76
|
+
Any embedded sdata metadata is read back and merged (Pillow-free).
|
|
77
|
+
|
|
78
|
+
:param name: a name for the image (its suffix sets the filetype).
|
|
79
|
+
:param image_data: the raw image bytes.
|
|
80
|
+
:param project: namespace for the deterministic SUUID.
|
|
81
|
+
:return: an :class:`Image` instance.
|
|
82
|
+
"""
|
|
83
|
+
suuid = SUUID.from_str("Image", name, ns_name=project)
|
|
84
|
+
suffix = Path(name).suffix.lstrip(".").lower() or "binary"
|
|
85
|
+
img = cls(content_type="bytes", value=image_data, filetype=suffix,
|
|
86
|
+
name=name, suuid=suuid, **kwargs)
|
|
87
|
+
img._load_embedded_metadata()
|
|
88
|
+
return img
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def pil(self):
|
|
92
|
+
"""The image decoded lazily with Pillow (:class:`PIL.Image.Image`)."""
|
|
93
|
+
if PIL is None:
|
|
94
|
+
raise ImportError("Pillow is required for Image decoding (pip install pillow).")
|
|
95
|
+
return PIL.Image.open(io.BytesIO(self.content_bytes))
|
|
96
|
+
|
|
97
|
+
def to_numpy(self) -> np.ndarray:
|
|
98
|
+
"""The image as a NumPy array (colour channels)."""
|
|
99
|
+
return np.array(self.pil)
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def basename(self) -> str:
|
|
103
|
+
"""The image file base name (== ``name``)."""
|
|
104
|
+
return self.name
|
|
105
|
+
|
|
106
|
+
def embedded_metadata(self):
|
|
107
|
+
"""Return the sdata metadata embedded in the image bytes, or ``None``.
|
|
108
|
+
|
|
109
|
+
Reads the native sdata payload (PNG ``iTXt`` / JPEG ``APP1`` / JP2 ``uuid``
|
|
110
|
+
box / GIF comment / WebP ``sdAT`` chunk) without Pillow.
|
|
111
|
+
|
|
112
|
+
:return: a :class:`~sdata.metadata.Metadata`, or ``None`` if absent/invalid.
|
|
113
|
+
"""
|
|
114
|
+
raw = imagemeta.extract(self.content_bytes)
|
|
115
|
+
if not raw:
|
|
116
|
+
return None
|
|
117
|
+
try:
|
|
118
|
+
return Metadata.from_json(raw)
|
|
119
|
+
except Exception as exp:
|
|
120
|
+
logger.debug(f"embedded sdata metadata not parseable: {exp}")
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
def _load_embedded_metadata(self) -> None:
|
|
124
|
+
"""Merge sdata metadata embedded in the image bytes (any supported format)."""
|
|
125
|
+
try:
|
|
126
|
+
embedded = self.embedded_metadata()
|
|
127
|
+
except Exception as exp: # e.g. unreadable uri content
|
|
128
|
+
logger.debug(f"no embedded sdata metadata: {exp}")
|
|
129
|
+
return
|
|
130
|
+
if embedded is not None:
|
|
131
|
+
self.metadata.update_from_usermetadata(embedded)
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def sidecar_path(filepath) -> str:
|
|
135
|
+
"""Path of the metadata sidecar for ``filepath`` (``<filepath>.meta.json``)."""
|
|
136
|
+
return str(filepath) + ".meta.json"
|
|
137
|
+
|
|
138
|
+
def write_sidecar(self, filepath) -> str:
|
|
139
|
+
"""Write the sdata metadata next to ``filepath`` as a lossless JSON sidecar.
|
|
140
|
+
|
|
141
|
+
The sidecar carries the **same** payload as the embedded form
|
|
142
|
+
(``metadata.to_json()``), so a round-trip is lossless regardless of whether
|
|
143
|
+
the format has a native metadata container.
|
|
144
|
+
|
|
145
|
+
:param filepath: the image path the sidecar belongs to.
|
|
146
|
+
:return: the sidecar path (``<filepath>.meta.json``).
|
|
147
|
+
"""
|
|
148
|
+
sidecar = self.sidecar_path(filepath)
|
|
149
|
+
with open(sidecar, "w", encoding="utf-8") as fh:
|
|
150
|
+
fh.write(self.metadata.to_json())
|
|
151
|
+
logger.info(f"Image metadata sidecar written to {sidecar}")
|
|
152
|
+
return sidecar
|
|
153
|
+
|
|
154
|
+
def _load_sidecar_metadata(self, filepath) -> None:
|
|
155
|
+
"""Merge metadata from an adjacent ``<filepath>.meta.json`` sidecar, if present."""
|
|
156
|
+
sidecar = self.sidecar_path(filepath)
|
|
157
|
+
if not os.path.exists(sidecar):
|
|
158
|
+
return
|
|
159
|
+
try:
|
|
160
|
+
with open(sidecar, "r", encoding="utf-8") as fh:
|
|
161
|
+
md = Metadata.from_json(fh.read())
|
|
162
|
+
self.metadata.update_from_usermetadata(md)
|
|
163
|
+
logger.debug(f"merged sidecar metadata from {sidecar}")
|
|
164
|
+
except Exception as exp:
|
|
165
|
+
logger.debug(f"sidecar metadata not loadable: {exp}")
|
|
166
|
+
|
|
167
|
+
def save(self, filepath, sidecar=None, **kwargs):
|
|
168
|
+
"""Save the image to ``filepath`` with sdata metadata — one API for all formats.
|
|
169
|
+
|
|
170
|
+
The container is chosen from the file suffix. For a format with a native
|
|
171
|
+
metadata container (PNG/JPEG/JP2/GIF/WebP/TIFF) the metadata is **embedded**:
|
|
172
|
+
without re-encoding if the stored bytes already use that container (lossless,
|
|
173
|
+
Pillow-free), otherwise Pillow transcodes first. For **any other** format
|
|
174
|
+
(e.g. BMP) the image is written via Pillow and the metadata travels in a
|
|
175
|
+
lossless ``<filepath>.meta.json`` **sidecar** — so metadata is never lost.
|
|
176
|
+
|
|
177
|
+
:param filepath: destination path (its suffix selects the format).
|
|
178
|
+
:param sidecar: sidecar policy — ``None`` (default) writes a sidecar only when
|
|
179
|
+
the format has no native container; ``True`` always writes one (in addition
|
|
180
|
+
to embedding); ``False`` never writes one.
|
|
181
|
+
:param kwargs: forwarded to ``PIL.Image.save`` when transcoding.
|
|
182
|
+
:raises ImportError: if Pillow is required (transcode / non-native format) but
|
|
183
|
+
not installed.
|
|
184
|
+
:return: the destination ``filepath``.
|
|
185
|
+
"""
|
|
186
|
+
suffix = Path(filepath).suffix.lstrip(".").lower()
|
|
187
|
+
target = self._SUFFIX_FORMATS.get(suffix)
|
|
188
|
+
|
|
189
|
+
if target is None:
|
|
190
|
+
# Kein nativer Metadaten-Träger → Pillow schreiben + verlustfreier Sidecar.
|
|
191
|
+
if PIL is None:
|
|
192
|
+
raise ImportError("Pillow is required for Image.save (pip install pillow).")
|
|
193
|
+
self.pil.save(filepath, **kwargs)
|
|
194
|
+
if sidecar is not False:
|
|
195
|
+
self.write_sidecar(filepath)
|
|
196
|
+
logger.info("Image.save: '%s' has no native metadata container; "
|
|
197
|
+
"metadata written to a sidecar", filepath)
|
|
198
|
+
return filepath
|
|
199
|
+
|
|
200
|
+
pil_format, meta_fmt = target
|
|
201
|
+
src = self.content_bytes
|
|
202
|
+
if imagemeta.detect_format(src) == meta_fmt:
|
|
203
|
+
img_bytes = src # passender Container → kein Transkodieren
|
|
204
|
+
else:
|
|
205
|
+
if PIL is None:
|
|
206
|
+
raise ImportError("Pillow is required to transcode images "
|
|
207
|
+
"(pip install pillow).")
|
|
208
|
+
buffer = io.BytesIO()
|
|
209
|
+
self.pil.save(buffer, pil_format, **kwargs)
|
|
210
|
+
img_bytes = buffer.getvalue()
|
|
211
|
+
|
|
212
|
+
img_bytes = imagemeta.embed(img_bytes, self.metadata.to_json(), meta_fmt)
|
|
213
|
+
with open(filepath, "wb") as fh:
|
|
214
|
+
fh.write(img_bytes)
|
|
215
|
+
if sidecar is True:
|
|
216
|
+
self.write_sidecar(filepath)
|
|
217
|
+
logger.info(f"Image saved to {filepath}")
|
|
218
|
+
return filepath
|