sdata 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdata/__init__.py +140 -0
- sdata/base.py +684 -0
- sdata/contrib/__init__.py +0 -0
- sdata/contrib/piexif/__init__.py +12 -0
- sdata/contrib/piexif/_common.py +94 -0
- sdata/contrib/piexif/_dump.py +346 -0
- sdata/contrib/piexif/_exceptions.py +2 -0
- sdata/contrib/piexif/_exif.py +638 -0
- sdata/contrib/piexif/_insert.py +60 -0
- sdata/contrib/piexif/_load.py +270 -0
- sdata/contrib/piexif/_remove.py +56 -0
- sdata/contrib/piexif/_transplant.py +45 -0
- sdata/contrib/piexif/_webp.py +240 -0
- sdata/contrib/piexif/helper.py +66 -0
- sdata/contrib/ranger/__init__.py +8 -0
- sdata/contrib/ranger/collections/__init__.py +0 -0
- sdata/contrib/ranger/collections/rangebucketmap.py +346 -0
- sdata/contrib/ranger/collections/rangemap.py +396 -0
- sdata/contrib/ranger/collections/rangeset.py +483 -0
- sdata/contrib/ranger/range/__init__.py +0 -0
- sdata/contrib/ranger/range/cut.py +285 -0
- sdata/contrib/ranger/range/range.py +644 -0
- sdata/contrib/sobol_seq.py +394 -0
- sdata/contrib/sortedcontainers/LICENSE +13 -0
- sdata/contrib/sortedcontainers/__init__.py +74 -0
- sdata/contrib/sortedcontainers/sorteddict.py +789 -0
- sdata/contrib/sortedcontainers/sortedlist.py +2646 -0
- sdata/contrib/sortedcontainers/sortedset.py +733 -0
- sdata/contrib/sqlitedict.py +598 -0
- sdata/contrib/timeflake/__init__.py +40 -0
- sdata/contrib/timeflake/flake.py +69 -0
- sdata/contrib/timeflake/utils.py +47 -0
- sdata/deprecated/__init__.py +10 -0
- sdata/deprecated/data.py +1503 -0
- sdata/did/__init__.py +129 -0
- sdata/did/_http.py +67 -0
- sdata/did/base58btc.py +49 -0
- sdata/did/did_github.py +152 -0
- sdata/did/did_peer4.py +155 -0
- sdata/did/did_web.py +222 -0
- sdata/did/diddoc.py +75 -0
- sdata/did/ed25519.py +225 -0
- sdata/did/eddsa.py +107 -0
- sdata/did/errors.py +40 -0
- sdata/did/issue_vc.py +193 -0
- sdata/did/jose.py +126 -0
- sdata/did/keys.py +104 -0
- sdata/did/utils_didkey.py +161 -0
- sdata/did/verify_vp.py +158 -0
- sdata/doe.py +101 -0
- sdata/dtypes.py +510 -0
- sdata/experiments/__init__.py +331 -0
- sdata/experiments/blob0.py +131 -0
- sdata/experiments/file.py +42 -0
- sdata/experiments/filegroup.py +144 -0
- sdata/experiments/image_old.py +313 -0
- sdata/experiments/ks2.py +15 -0
- sdata/experiments/lapshear.py +8 -0
- sdata/experiments/material.py +7 -0
- sdata/experiments/parameters_deprecated.py +394 -0
- sdata/experiments/workbook.py +226 -0
- sdata/format.py +125 -0
- sdata/imagemeta.py +533 -0
- sdata/interactive.py +118 -0
- sdata/iolib/__init__.py +51 -0
- sdata/iolib/hashtree.py +264 -0
- sdata/iolib/json1sqlitestore.py +554 -0
- sdata/iolib/jsonsqlitestore.py +704 -0
- sdata/iolib/pud.py +242 -0
- sdata/iolib/reader.py +262 -0
- sdata/iolib/writer.py +419 -0
- sdata/metadata.py +882 -0
- sdata/parameter.py +414 -0
- sdata/schema.py +273 -0
- sdata/sclass/__init__.py +111 -0
- sdata/sclass/bfo_classes.py +362 -0
- sdata/sclass/blob.py +362 -0
- sdata/sclass/content.py +87 -0
- sdata/sclass/dataframe.py +1251 -0
- sdata/sclass/dataframegroup.py +187 -0
- sdata/sclass/dependency_graph.py +85 -0
- sdata/sclass/filereference.py +124 -0
- sdata/sclass/image.py +218 -0
- sdata/sclass/lazy_namespace.py +155 -0
- sdata/sclass/process.py +383 -0
- sdata/sclass/process_graph.py +221 -0
- sdata/sclass/zipfile.py +135 -0
- sdata/semantic.py +346 -0
- sdata/suuid.py +136 -0
- sdata/timestamp.py +336 -0
- sdata/tools.py +14 -0
- sdata/units.py +602 -0
- sdata/vocab.py +149 -0
- sdata-1.5.0.dist-info/METADATA +264 -0
- sdata-1.5.0.dist-info/RECORD +98 -0
- sdata-1.5.0.dist-info/WHEEL +5 -0
- sdata-1.5.0.dist-info/licenses/LICENSE-MIT +21 -0
- sdata-1.5.0.dist-info/top_level.txt +1 -0
sdata/format.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Format-Versionierung des serialisierten sdata-Objekts (RFC 0010).
|
|
2
|
+
|
|
3
|
+
Ein **monoton steigender Integer** ``_sdata_format_version`` beschreibt die Shape
|
|
4
|
+
des serialisierten Objekt-JSON — getrennt von der Paketversion
|
|
5
|
+
(``_sdata_version`` = ``sdata.__version__``, die bei jedem Release steigt, aber
|
|
6
|
+
kein Kompatibilitätssignal ist).
|
|
7
|
+
|
|
8
|
+
Beim Lesen läuft jedes Payload durch :func:`ensure_compatible` (ein Choke-Point in
|
|
9
|
+
:meth:`Base.from_dict`/:meth:`DataFrame.from_dict`/``_restore_from_attrs``):
|
|
10
|
+
|
|
11
|
+
* fehlt das Feld → **v1** (Objekte vor diesem RFC), still lesen;
|
|
12
|
+
* gleich → normal;
|
|
13
|
+
* älter → :func:`register_migration`-Treppe (v1→v2→…) anwenden;
|
|
14
|
+
* neuer → :class:`FormatVersionWarning` + best-effort (``strict=True`` →
|
|
15
|
+
:class:`IncompatibleFormatError`).
|
|
16
|
+
|
|
17
|
+
Vorbild: das ``PRAGMA user_version`` + ``migrate()``-Muster des
|
|
18
|
+
:class:`~sdata.iolib.json1sqlitestore.JSON1SQLiteStore` (dort auf DB-Schema-Ebene).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
import warnings
|
|
24
|
+
from typing import Any, Callable, Dict, Optional
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"SDATA_FORMAT_VERSION_KEY",
|
|
30
|
+
"CURRENT_FORMAT_VERSION",
|
|
31
|
+
"FormatVersionWarning",
|
|
32
|
+
"IncompatibleFormatError",
|
|
33
|
+
"read_format_version",
|
|
34
|
+
"ensure_compatible",
|
|
35
|
+
"register_migration",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
#: Reservierter Metadaten-Schlüssel für die Formatversion.
|
|
39
|
+
SDATA_FORMAT_VERSION_KEY = "_sdata_format_version"
|
|
40
|
+
|
|
41
|
+
#: Aktuelle Formatversion; steigt **nur** bei einem Shape-Bruch des Objekt-JSON.
|
|
42
|
+
CURRENT_FORMAT_VERSION = 1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class FormatVersionWarning(UserWarning):
|
|
46
|
+
"""Ein gelesenes Objekt trägt eine **neuere** Formatversion als unterstützt."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class IncompatibleFormatError(Exception):
|
|
50
|
+
"""Ein Objekt kann nicht auf die aktuelle Formatversion gebracht werden."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
#: Migrations-Treppe: ``{from_version: callable(payload) -> payload}``. Leer bis zum
|
|
54
|
+
#: ersten echten Shape-Bruch; jede Stufe hebt genau um eine Version an.
|
|
55
|
+
_MIGRATIONS: Dict[int, Callable[[Dict[str, Any]], Dict[str, Any]]] = {}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def register_migration(from_version: int):
|
|
59
|
+
"""Registriere eine Migration ``payload(v{from}) -> payload(v{from+1})``.
|
|
60
|
+
|
|
61
|
+
:param from_version: die Ausgangs-Formatversion, die diese Stufe anhebt.
|
|
62
|
+
"""
|
|
63
|
+
def deco(fn: Callable[[Dict[str, Any]], Dict[str, Any]]):
|
|
64
|
+
_MIGRATIONS[from_version] = fn
|
|
65
|
+
return fn
|
|
66
|
+
return deco
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def read_format_version(payload: Optional[Dict[str, Any]]) -> int:
|
|
70
|
+
"""Lies die Formatversion aus einem Objekt- oder ``attrs``-Payload.
|
|
71
|
+
|
|
72
|
+
Toleriert beide Shapes (``{"metadata": {...}}`` aus ``to_dict`` **und** ein
|
|
73
|
+
flaches Metadaten-Dict) sowie das genestete ``{"value": …}`` je Attribut.
|
|
74
|
+
Fehlt das Feld oder ist es unparsebar, gilt **v1**.
|
|
75
|
+
|
|
76
|
+
:param payload: Objekt-Dict, ``attrs['_sdata']``-Dict oder ``None``.
|
|
77
|
+
:return: die Formatversion als ``int`` (Default 1).
|
|
78
|
+
"""
|
|
79
|
+
if not isinstance(payload, dict):
|
|
80
|
+
return 1
|
|
81
|
+
meta = payload.get("metadata", payload)
|
|
82
|
+
if not isinstance(meta, dict):
|
|
83
|
+
return 1
|
|
84
|
+
node = meta.get(SDATA_FORMAT_VERSION_KEY)
|
|
85
|
+
if isinstance(node, dict): # to_dict nestet {"value": …, "dtype": …}
|
|
86
|
+
node = node.get("value")
|
|
87
|
+
try:
|
|
88
|
+
return int(node) if node is not None else 1
|
|
89
|
+
except (TypeError, ValueError):
|
|
90
|
+
return 1
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def ensure_compatible(payload: Dict[str, Any], *,
|
|
94
|
+
current: int = CURRENT_FORMAT_VERSION,
|
|
95
|
+
strict: bool = False) -> Dict[str, Any]:
|
|
96
|
+
"""Prüfe/migriere ein Payload auf ``current`` — der Lese-Choke-Point (RFC 0010 §6).
|
|
97
|
+
|
|
98
|
+
:param payload: das zu lesende Objekt-/attrs-Dict (wird bei Migration verändert
|
|
99
|
+
zurückgegeben; ohne Migration unverändert durchgereicht).
|
|
100
|
+
:param current: Zielversion (Default :data:`CURRENT_FORMAT_VERSION`).
|
|
101
|
+
:param strict: bei einer **neueren** Version statt zu warnen hart scheitern.
|
|
102
|
+
:raises IncompatibleFormatError: bei fehlender Migrationsstufe oder (mit
|
|
103
|
+
``strict``) bei einer neueren Version.
|
|
104
|
+
:raises FormatVersionWarning: (als Warnung) bei einer neueren Version ohne ``strict``.
|
|
105
|
+
"""
|
|
106
|
+
if not isinstance(payload, dict):
|
|
107
|
+
return payload
|
|
108
|
+
v = read_format_version(payload)
|
|
109
|
+
if v == current:
|
|
110
|
+
return payload
|
|
111
|
+
if v > current:
|
|
112
|
+
msg = f"sdata object format v{v} is newer than supported v{current}"
|
|
113
|
+
if strict:
|
|
114
|
+
raise IncompatibleFormatError(msg)
|
|
115
|
+
warnings.warn(msg, FormatVersionWarning)
|
|
116
|
+
return payload
|
|
117
|
+
while v < current:
|
|
118
|
+
step = _MIGRATIONS.get(v)
|
|
119
|
+
if step is None:
|
|
120
|
+
raise IncompatibleFormatError(
|
|
121
|
+
f"no migration registered for sdata format v{v} -> v{v + 1}")
|
|
122
|
+
payload = step(payload)
|
|
123
|
+
logger.info("migrated sdata payload v%d -> v%d", v, v + 1)
|
|
124
|
+
v += 1
|
|
125
|
+
return payload
|
sdata/imagemeta.py
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Native, format-übergreifende Einbettung von sdata-Metadaten in Bild-Bytes.
|
|
3
|
+
|
|
4
|
+
Reiner Python-Code (**keine** Pillow-Abhängigkeit für die Metadaten-Schicht): ein
|
|
5
|
+
Text-Payload (i. d. R. das sdata-Metadaten-JSON) wird über eine **einheitliche
|
|
6
|
+
API** verlustfrei in den jeweiligen Bildcontainer geschrieben bzw. daraus gelesen.
|
|
7
|
+
|
|
8
|
+
Unterstützte Container und ihr nativer Träger:
|
|
9
|
+
|
|
10
|
+
* **PNG** — ``iTXt``-Chunk mit Schlüsselwort ``sdata`` (UTF-8)
|
|
11
|
+
* **JPEG** — ``APP1``-Segment mit Kennung ``sdata\\0`` (UTF-8)
|
|
12
|
+
* **JP2** — ``uuid``-Box (JPEG 2000, ISO BMFF) mit fester sdata-UUID
|
|
13
|
+
* **GIF** — Comment-Extension mit Präfix ``sdata\\0``
|
|
14
|
+
* **WebP** — eigener RIFF-Chunk ``sdAT`` (von Decodern als unbekannt ignoriert)
|
|
15
|
+
* **TIFF** — privates IFD-Tag (65000); die Original-Bytes bleiben unverändert
|
|
16
|
+
|
|
17
|
+
Das Format wird an den Magic-Bytes erkannt (:func:`detect_format`); :func:`embed`
|
|
18
|
+
und :func:`extract` wählen den passenden Handler. Die Schreibsemantik ist
|
|
19
|
+
*replace* (eine vorhandene sdata-Nutzlast wird ersetzt, nicht dupliziert). Pillow
|
|
20
|
+
(optional) wird nur zum **Transkodieren der Pixel** benötigt, nicht für die
|
|
21
|
+
Metadaten — das Lesen funktioniert daher vollständig Pillow-frei.
|
|
22
|
+
|
|
23
|
+
:Example:
|
|
24
|
+
|
|
25
|
+
>>> from sdata import imagemeta
|
|
26
|
+
>>> png_with_meta = imagemeta.embed(png_bytes, '{"name": "probe"}') # doctest: +SKIP
|
|
27
|
+
>>> imagemeta.extract(png_with_meta) # doctest: +SKIP
|
|
28
|
+
'{"name": "probe"}'
|
|
29
|
+
"""
|
|
30
|
+
import struct
|
|
31
|
+
import zlib
|
|
32
|
+
import logging
|
|
33
|
+
from typing import Iterator, List, Optional, Tuple
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
#: Kennung/Schlüsselwort der sdata-Nutzlast in allen Containern.
|
|
38
|
+
SDATA_ID = "sdata"
|
|
39
|
+
_SID = SDATA_ID.encode("ascii")
|
|
40
|
+
#: JPEG-``APP1``-Kennung (``sdata\\0``) zur Unterscheidung von EXIF/XMP.
|
|
41
|
+
_JPEG_IDENT = _SID + b"\x00"
|
|
42
|
+
#: GIF-Comment-Präfix (``sdata\\0``) zur Erkennung der sdata-Comment-Extension.
|
|
43
|
+
_GIF_PREFIX = _SID + b"\x00"
|
|
44
|
+
#: 16-Byte-UUID für die JP2-``uuid``-Box (genau 16 Byte).
|
|
45
|
+
SDATA_JP2_UUID = b"SDATA-IMG-META\x00\x00"
|
|
46
|
+
#: WebP-FourCC des sdata-Chunks (unbekannt → von WebP-Decodern ignoriert).
|
|
47
|
+
_WEBP_FOURCC = b"sdAT"
|
|
48
|
+
#: Maximale JPEG-``APP1``-Nutzlast (Längenfeld 16 bit, inkl. 2 Längen-Bytes).
|
|
49
|
+
_JPEG_MAX_PAYLOAD = 0xFFFF - 2 - len(_JPEG_IDENT)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ImageMetadataError(Exception):
|
|
53
|
+
"""Basisfehler der Bild-Metadaten-Schicht."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class UnsupportedImageFormatError(ImageMetadataError):
|
|
57
|
+
"""Das Bildformat wird (zum Schreiben) nicht unterstützt."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class PayloadTooLargeError(ImageMetadataError):
|
|
61
|
+
"""Die Nutzlast passt nicht in ein einzelnes Format-Segment (z. B. JPEG ``APP1``)."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ====================================================================== PNG
|
|
65
|
+
_PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _png_chunks(data: bytes) -> Iterator[Tuple[bytes, bytes]]:
|
|
69
|
+
"""Iteriere ``(chunk_type, chunk_data)`` eines PNG (ohne 8-Byte-Signatur)."""
|
|
70
|
+
pos = len(_PNG_MAGIC)
|
|
71
|
+
while pos + 8 <= len(data):
|
|
72
|
+
(length,) = struct.unpack(">I", data[pos:pos + 4])
|
|
73
|
+
ctype = data[pos + 4:pos + 8]
|
|
74
|
+
cdata = data[pos + 8:pos + 8 + length]
|
|
75
|
+
yield ctype, cdata
|
|
76
|
+
pos += 12 + length # length(4) + type(4) + data + crc(4)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _png_make_chunk(ctype: bytes, cdata: bytes) -> bytes:
|
|
80
|
+
"""Baue einen PNG-Chunk (Länge + Typ + Daten + CRC-32 über Typ+Daten)."""
|
|
81
|
+
crc = zlib.crc32(ctype + cdata) & 0xFFFFFFFF
|
|
82
|
+
return struct.pack(">I", len(cdata)) + ctype + cdata + struct.pack(">I", crc)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _png_itxt(payload: str) -> bytes:
|
|
86
|
+
"""``iTXt``-Chunk mit Schlüsselwort ``sdata`` (unkomprimiert, UTF-8)."""
|
|
87
|
+
body = (_SID + b"\x00" # keyword + null
|
|
88
|
+
+ b"\x00\x00" # compression flag + method
|
|
89
|
+
+ b"\x00" # (leerer) language tag + null
|
|
90
|
+
+ b"\x00" # (leeres) translated keyword + null
|
|
91
|
+
+ payload.encode("utf-8"))
|
|
92
|
+
return _png_make_chunk(b"iTXt", body)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _png_embed(data: bytes, payload: str) -> bytes:
|
|
96
|
+
"""Schreibe ``payload`` als ``iTXt``-Chunk vor ``IEND`` (ersetzt vorhandenen)."""
|
|
97
|
+
out = bytearray(_PNG_MAGIC)
|
|
98
|
+
for ctype, cdata in _png_chunks(data):
|
|
99
|
+
if ctype in (b"iTXt", b"tEXt") and cdata.split(b"\x00", 1)[0] == _SID:
|
|
100
|
+
continue # vorhandenen sdata-Text-Chunk verwerfen (replace)
|
|
101
|
+
if ctype == b"IEND":
|
|
102
|
+
out += _png_itxt(payload)
|
|
103
|
+
out += _png_make_chunk(ctype, cdata)
|
|
104
|
+
return bytes(out)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _png_extract(data: bytes) -> Optional[str]:
|
|
108
|
+
"""Lies die sdata-Nutzlast aus ``iTXt``/``tEXt`` mit Schlüsselwort ``sdata``."""
|
|
109
|
+
for ctype, cdata in _png_chunks(data):
|
|
110
|
+
if ctype == b"iTXt":
|
|
111
|
+
keyword, _, rest = cdata.partition(b"\x00")
|
|
112
|
+
if keyword == _SID:
|
|
113
|
+
rest = rest[2:] # compflag + method
|
|
114
|
+
_, _, rest = rest.partition(b"\x00") # language tag
|
|
115
|
+
_, _, text = rest.partition(b"\x00") # translated keyword
|
|
116
|
+
return text.decode("utf-8")
|
|
117
|
+
elif ctype == b"tEXt":
|
|
118
|
+
keyword, _, text = cdata.partition(b"\x00")
|
|
119
|
+
if keyword == _SID:
|
|
120
|
+
return text.decode("latin-1")
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ===================================================================== JPEG
|
|
125
|
+
_JPEG_MAGIC = b"\xff\xd8\xff"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _jpeg_segments(data: bytes) -> Iterator[Tuple[int, int, Optional[int], Optional[bytes]]]:
|
|
129
|
+
"""Iteriere ``(start, end, marker, segment_payload)`` ab SOI bis einschließlich SOS.
|
|
130
|
+
|
|
131
|
+
``segment_payload`` ist ``None`` für eigenständige Marker (RSTn/TEM) bzw. ab
|
|
132
|
+
SOS/EOI (dort umfasst die Spanne ``start..len(data)``, also den Rest).
|
|
133
|
+
"""
|
|
134
|
+
pos = 2 # nach SOI
|
|
135
|
+
n = len(data)
|
|
136
|
+
while pos < n:
|
|
137
|
+
if data[pos] != 0xFF:
|
|
138
|
+
yield pos, n, None, None # kein Marker mehr → Rest
|
|
139
|
+
return
|
|
140
|
+
start = pos
|
|
141
|
+
while pos < n and data[pos] == 0xFF:
|
|
142
|
+
pos += 1 # Füll-Bytes (0xFF) überspringen
|
|
143
|
+
marker = data[pos]
|
|
144
|
+
if marker in (0xDA, 0xD9): # SOS / EOI → Rest verbatim
|
|
145
|
+
yield start, n, marker, None
|
|
146
|
+
return
|
|
147
|
+
if 0xD0 <= marker <= 0xD7 or marker == 0x01: # RSTn / TEM: ohne Länge
|
|
148
|
+
yield start, pos + 1, marker, None
|
|
149
|
+
pos += 1
|
|
150
|
+
continue
|
|
151
|
+
(length,) = struct.unpack(">H", data[pos + 1:pos + 3])
|
|
152
|
+
end = pos + 1 + length
|
|
153
|
+
yield start, end, marker, data[pos + 3:end]
|
|
154
|
+
pos = end
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _jpeg_strip(data: bytes) -> bytes:
|
|
158
|
+
"""Entferne ein vorhandenes sdata-``APP1``-Segment (SOI bleibt erhalten)."""
|
|
159
|
+
out = bytearray(data[:2]) # SOI
|
|
160
|
+
for start, end, marker, seg in _jpeg_segments(data):
|
|
161
|
+
if marker == 0xE1 and seg is not None and seg.startswith(_JPEG_IDENT):
|
|
162
|
+
continue
|
|
163
|
+
out += data[start:end]
|
|
164
|
+
return bytes(out)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _jpeg_embed(data: bytes, payload: str) -> bytes:
|
|
168
|
+
"""Schreibe ``payload`` als ``APP1``-Segment direkt hinter SOI (replace)."""
|
|
169
|
+
body = _JPEG_IDENT + payload.encode("utf-8")
|
|
170
|
+
if len(body) + 2 > 0xFFFF:
|
|
171
|
+
raise PayloadTooLargeError(
|
|
172
|
+
f"payload too large for a single JPEG APP1 segment "
|
|
173
|
+
f"(max {_JPEG_MAX_PAYLOAD} bytes)")
|
|
174
|
+
segment = b"\xff\xe1" + struct.pack(">H", len(body) + 2) + body
|
|
175
|
+
cleaned = _jpeg_strip(data)
|
|
176
|
+
return cleaned[:2] + segment + cleaned[2:]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _jpeg_extract(data: bytes) -> Optional[str]:
|
|
180
|
+
"""Lies die sdata-Nutzlast aus dem ``APP1``-Segment mit Kennung ``sdata\\0``."""
|
|
181
|
+
for _start, _end, marker, seg in _jpeg_segments(data):
|
|
182
|
+
if marker == 0xE1 and seg is not None and seg.startswith(_JPEG_IDENT):
|
|
183
|
+
return seg[len(_JPEG_IDENT):].decode("utf-8")
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# ====================================================================== JP2
|
|
188
|
+
_JP2_MAGIC = b"\x00\x00\x00\x0cjP \r\n\x87\n"
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _jp2_boxes(data: bytes) -> Iterator[Tuple[bytes, int, int, int]]:
|
|
192
|
+
"""Iteriere Top-Level-Boxen als ``(type, box_start, content_start, box_end)``."""
|
|
193
|
+
pos = 0
|
|
194
|
+
n = len(data)
|
|
195
|
+
while pos + 8 <= n:
|
|
196
|
+
(lbox,) = struct.unpack(">I", data[pos:pos + 4])
|
|
197
|
+
tbox = data[pos + 4:pos + 8]
|
|
198
|
+
content_start = pos + 8
|
|
199
|
+
if lbox == 1: # 64-bit XLBox
|
|
200
|
+
(xlbox,) = struct.unpack(">Q", data[pos + 8:pos + 16])
|
|
201
|
+
content_start = pos + 16
|
|
202
|
+
end = pos + xlbox
|
|
203
|
+
elif lbox == 0: # bis Dateiende
|
|
204
|
+
end = n
|
|
205
|
+
else:
|
|
206
|
+
end = pos + lbox
|
|
207
|
+
yield tbox, pos, content_start, end
|
|
208
|
+
if end <= pos: # Schutz vor Endlosschleife
|
|
209
|
+
return
|
|
210
|
+
pos = end
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _jp2_uuid_box(payload: str) -> bytes:
|
|
214
|
+
"""``uuid``-Box mit fester sdata-UUID + UTF-8-Nutzlast."""
|
|
215
|
+
content = SDATA_JP2_UUID + payload.encode("utf-8")
|
|
216
|
+
return struct.pack(">I", len(content) + 8) + b"uuid" + content
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _jp2_embed(data: bytes, payload: str) -> bytes:
|
|
220
|
+
"""Schreibe ``payload`` als ``uuid``-Box vor die ``jp2c``-Codestream-Box (replace)."""
|
|
221
|
+
out = bytearray()
|
|
222
|
+
for tbox, start, cstart, end in _jp2_boxes(data):
|
|
223
|
+
if tbox == b"uuid" and data[cstart:cstart + 16] == SDATA_JP2_UUID:
|
|
224
|
+
continue # vorhandene sdata-Box verwerfen (replace)
|
|
225
|
+
if tbox == b"jp2c":
|
|
226
|
+
out += _jp2_uuid_box(payload)
|
|
227
|
+
out += data[start:end]
|
|
228
|
+
return bytes(out)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _jp2_extract(data: bytes) -> Optional[str]:
|
|
232
|
+
"""Lies die sdata-Nutzlast aus der ``uuid``-Box mit der sdata-UUID."""
|
|
233
|
+
for tbox, _start, cstart, end in _jp2_boxes(data):
|
|
234
|
+
if tbox == b"uuid" and data[cstart:cstart + 16] == SDATA_JP2_UUID:
|
|
235
|
+
return data[cstart + 16:end].decode("utf-8")
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ====================================================================== GIF
|
|
240
|
+
_GIF_MAGICS = (b"GIF87a", b"GIF89a")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _gif_body_start(data: bytes) -> int:
|
|
244
|
+
"""Offset hinter Header + Logical Screen Descriptor + Global Color Table."""
|
|
245
|
+
packed = data[10]
|
|
246
|
+
pos = 13 # 6 (Header) + 7 (Logical Screen Descriptor)
|
|
247
|
+
if packed & 0x80: # Global Color Table vorhanden
|
|
248
|
+
pos += 3 * (2 ** ((packed & 0x07) + 1))
|
|
249
|
+
return pos
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _gif_skip_subblocks(data: bytes, pos: int) -> int:
|
|
253
|
+
"""Position direkt hinter einem Sub-Block-Stream (Terminator ``0x00``)."""
|
|
254
|
+
n = len(data)
|
|
255
|
+
while pos < n:
|
|
256
|
+
size = data[pos]
|
|
257
|
+
pos += 1
|
|
258
|
+
if size == 0:
|
|
259
|
+
break
|
|
260
|
+
pos += size
|
|
261
|
+
return pos
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _gif_read_subblocks(data: bytes, pos: int) -> Tuple[bytes, int]:
|
|
265
|
+
"""Lies einen Sub-Block-Stream zu ``(bytes, end_pos)`` zusammen."""
|
|
266
|
+
out = bytearray()
|
|
267
|
+
n = len(data)
|
|
268
|
+
while pos < n:
|
|
269
|
+
size = data[pos]
|
|
270
|
+
pos += 1
|
|
271
|
+
if size == 0:
|
|
272
|
+
break
|
|
273
|
+
out += data[pos:pos + size]
|
|
274
|
+
pos += size
|
|
275
|
+
return bytes(out), pos
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _gif_comment_spans(data: bytes) -> Iterator[Tuple[int, int, bytes]]:
|
|
279
|
+
"""Iteriere ``(start, end, text)`` jeder Comment-Extension auf Block-Ebene."""
|
|
280
|
+
pos = _gif_body_start(data)
|
|
281
|
+
n = len(data)
|
|
282
|
+
while pos < n:
|
|
283
|
+
intro = data[pos]
|
|
284
|
+
if intro == 0x3B: # Trailer
|
|
285
|
+
return
|
|
286
|
+
if intro == 0x2C: # Image Descriptor
|
|
287
|
+
packed = data[pos + 9]
|
|
288
|
+
pos += 10
|
|
289
|
+
if packed & 0x80: # Local Color Table
|
|
290
|
+
pos += 3 * (2 ** ((packed & 0x07) + 1))
|
|
291
|
+
pos += 1 # LZW minimum code size
|
|
292
|
+
pos = _gif_skip_subblocks(data, pos)
|
|
293
|
+
continue
|
|
294
|
+
if intro == 0x21: # Extension
|
|
295
|
+
label = data[pos + 1]
|
|
296
|
+
sub_start = pos + 2
|
|
297
|
+
if label == 0xFE: # Comment Extension
|
|
298
|
+
text, end = _gif_read_subblocks(data, sub_start)
|
|
299
|
+
yield pos, end, text
|
|
300
|
+
pos = end
|
|
301
|
+
else: # Graphic Control / Application / …
|
|
302
|
+
pos = _gif_skip_subblocks(data, sub_start)
|
|
303
|
+
continue
|
|
304
|
+
return # unbekannter Block → abbrechen
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _gif_subblocks(payload: bytes) -> bytes:
|
|
308
|
+
"""Zerlege ``payload`` in 255-Byte-Sub-Blöcke mit ``0x00``-Terminator."""
|
|
309
|
+
out = bytearray()
|
|
310
|
+
for i in range(0, len(payload), 255):
|
|
311
|
+
chunk = payload[i:i + 255]
|
|
312
|
+
out += bytes([len(chunk)]) + chunk
|
|
313
|
+
out += b"\x00"
|
|
314
|
+
return bytes(out)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _gif_strip(data: bytes) -> bytes:
|
|
318
|
+
"""Entferne vorhandene sdata-Comment-Extensions (replace)."""
|
|
319
|
+
spans = [(s, e) for s, e, t in _gif_comment_spans(data)
|
|
320
|
+
if t.startswith(_GIF_PREFIX)]
|
|
321
|
+
if not spans:
|
|
322
|
+
return data
|
|
323
|
+
out = bytearray()
|
|
324
|
+
prev = 0
|
|
325
|
+
for start, end in spans:
|
|
326
|
+
out += data[prev:start]
|
|
327
|
+
prev = end
|
|
328
|
+
out += data[prev:]
|
|
329
|
+
return bytes(out)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _gif_embed(data: bytes, payload: str) -> bytes:
|
|
333
|
+
"""Schreibe ``payload`` als Comment-Extension hinter den Header (replace)."""
|
|
334
|
+
cleaned = _gif_strip(data)
|
|
335
|
+
at = _gif_body_start(cleaned)
|
|
336
|
+
ext = b"\x21\xfe" + _gif_subblocks(_GIF_PREFIX + payload.encode("utf-8"))
|
|
337
|
+
return cleaned[:at] + ext + cleaned[at:]
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _gif_extract(data: bytes) -> Optional[str]:
|
|
341
|
+
"""Lies die sdata-Nutzlast aus der Comment-Extension mit Präfix ``sdata\\0``."""
|
|
342
|
+
for _start, _end, text in _gif_comment_spans(data):
|
|
343
|
+
if text.startswith(_GIF_PREFIX):
|
|
344
|
+
return text[len(_GIF_PREFIX):].decode("utf-8")
|
|
345
|
+
return None
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# ===================================================================== WebP
|
|
349
|
+
def _webp_chunks(data: bytes) -> Iterator[Tuple[bytes, bytes]]:
|
|
350
|
+
"""Iteriere ``(fourcc, chunk_data)`` eines RIFF/WebP (ab dem ``WEBP``-Tag)."""
|
|
351
|
+
pos = 12 # 'RIFF' + size(4) + 'WEBP'
|
|
352
|
+
n = len(data)
|
|
353
|
+
while pos + 8 <= n:
|
|
354
|
+
fourcc = data[pos:pos + 4]
|
|
355
|
+
(size,) = struct.unpack("<I", data[pos + 4:pos + 8])
|
|
356
|
+
body = data[pos + 8:pos + 8 + size]
|
|
357
|
+
yield fourcc, body
|
|
358
|
+
pos += 8 + size + (size & 1) # ungerade Größe → 1 Pad-Byte
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _webp_embed(data: bytes, payload: str) -> bytes:
|
|
362
|
+
"""Schreibe ``payload`` als eigenen ``sdAT``-Chunk ans Ende (replace)."""
|
|
363
|
+
chunks: List[Tuple[bytes, bytes]] = [
|
|
364
|
+
(fourcc, body) for fourcc, body in _webp_chunks(data)
|
|
365
|
+
if fourcc != _WEBP_FOURCC
|
|
366
|
+
]
|
|
367
|
+
chunks.append((_WEBP_FOURCC, payload.encode("utf-8")))
|
|
368
|
+
out = bytearray()
|
|
369
|
+
for fourcc, body in chunks:
|
|
370
|
+
out += fourcc + struct.pack("<I", len(body)) + body
|
|
371
|
+
if len(body) & 1:
|
|
372
|
+
out += b"\x00"
|
|
373
|
+
return b"RIFF" + struct.pack("<I", len(out) + 4) + b"WEBP" + bytes(out)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _webp_extract(data: bytes) -> Optional[str]:
|
|
377
|
+
"""Lies die sdata-Nutzlast aus dem ``sdAT``-Chunk."""
|
|
378
|
+
for fourcc, body in _webp_chunks(data):
|
|
379
|
+
if fourcc == _WEBP_FOURCC:
|
|
380
|
+
return body.decode("utf-8")
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# ====================================================================== TIFF
|
|
385
|
+
_TIFF_LE = b"II\x2a\x00" # little-endian, classic TIFF (magic 42)
|
|
386
|
+
_TIFF_BE = b"MM\x00\x2a" # big-endian, classic TIFF (magic 42)
|
|
387
|
+
#: privates TIFF-Tag (Bereich 32768–65535) für die sdata-Nutzlast.
|
|
388
|
+
_TIFF_TAG = 65000
|
|
389
|
+
_TIFF_TYPE_UNDEFINED = 7 # 7 = UNDEFINED (rohe Bytes)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _tiff_endian(data: bytes) -> str:
|
|
393
|
+
"""``struct``-Präfix für die Byte-Reihenfolge des TIFF (``<`` für ``II``)."""
|
|
394
|
+
return "<" if data[:2] == b"II" else ">"
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _tiff_entries(data: bytes, e: str, ifd_off: int):
|
|
398
|
+
"""Lies ``(entries, next_ifd_offset)`` einer IFD.
|
|
399
|
+
|
|
400
|
+
``entries`` ist eine Liste ``(tag, type, count, value_field)`` (value_field =
|
|
401
|
+
die 4 rohen Value/Offset-Bytes, unverändert übernommen).
|
|
402
|
+
"""
|
|
403
|
+
(count,) = struct.unpack(e + "H", data[ifd_off:ifd_off + 2])
|
|
404
|
+
entries = []
|
|
405
|
+
pos = ifd_off + 2
|
|
406
|
+
for _ in range(count):
|
|
407
|
+
tag, typ, cnt = struct.unpack(e + "HHI", data[pos:pos + 8])
|
|
408
|
+
entries.append((tag, typ, cnt, data[pos + 8:pos + 12]))
|
|
409
|
+
pos += 12
|
|
410
|
+
(next_off,) = struct.unpack(e + "I", data[pos:pos + 4])
|
|
411
|
+
return entries, next_off
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _tiff_embed(data: bytes, payload: str) -> bytes:
|
|
415
|
+
"""Schreibe ``payload`` ohne Offset-Chirurgie ein (replace-Semantik).
|
|
416
|
+
|
|
417
|
+
Die Original-Bytes bleiben **unverändert** (alle bestehenden Offsets, inkl.
|
|
418
|
+
``StripOffsets``, bleiben gültig). Eine **Kopie** der ersten IFD — ergänzt um ein
|
|
419
|
+
privates Tag mit der Nutzlast — wird ans Dateiende angehängt und der Header auf
|
|
420
|
+
diese neue IFD umgelenkt. Mehrfaches Einbetten ersetzt logisch die Nutzlast;
|
|
421
|
+
verwaiste Vorgänger-Bytes bleiben ungenutzt im File (kein Re-Pack).
|
|
422
|
+
"""
|
|
423
|
+
e = _tiff_endian(data)
|
|
424
|
+
(ifd0_off,) = struct.unpack(e + "I", data[4:8])
|
|
425
|
+
entries, next_off = _tiff_entries(data, e, ifd0_off)
|
|
426
|
+
entries = [en for en in entries if en[0] != _TIFF_TAG] # vorhandenes Tag droppen
|
|
427
|
+
body = payload.encode("utf-8")
|
|
428
|
+
blob = bytearray(data)
|
|
429
|
+
if len(body) <= 4:
|
|
430
|
+
value_field = body + b"\x00" * (4 - len(body)) # inline (≤4 Byte)
|
|
431
|
+
else:
|
|
432
|
+
blob += b"\x00" * (len(blob) & 1) # Value wortausrichten
|
|
433
|
+
payload_off = len(blob)
|
|
434
|
+
blob += body
|
|
435
|
+
value_field = struct.pack(e + "I", payload_off)
|
|
436
|
+
entries.append((_TIFF_TAG, _TIFF_TYPE_UNDEFINED, len(body), value_field))
|
|
437
|
+
entries.sort(key=lambda en: en[0]) # IFD-Einträge aufsteigend
|
|
438
|
+
blob += b"\x00" * (len(blob) & 1) # IFD wortausrichten
|
|
439
|
+
new_ifd_off = len(blob)
|
|
440
|
+
blob += struct.pack(e + "H", len(entries))
|
|
441
|
+
for tag, typ, count, value_field in entries:
|
|
442
|
+
blob += struct.pack(e + "HHI", tag, typ, count) + value_field
|
|
443
|
+
blob += struct.pack(e + "I", next_off)
|
|
444
|
+
struct.pack_into(e + "I", blob, 4, new_ifd_off) # Header → neue IFD
|
|
445
|
+
return bytes(blob)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _tiff_extract(data: bytes) -> Optional[str]:
|
|
449
|
+
"""Lies die sdata-Nutzlast aus dem privaten TIFF-Tag (inline oder per Offset)."""
|
|
450
|
+
e = _tiff_endian(data)
|
|
451
|
+
(ifd0_off,) = struct.unpack(e + "I", data[4:8])
|
|
452
|
+
entries, _next = _tiff_entries(data, e, ifd0_off)
|
|
453
|
+
for tag, _typ, count, value_field in entries:
|
|
454
|
+
if tag == _TIFF_TAG:
|
|
455
|
+
if count <= 4:
|
|
456
|
+
return value_field[:count].decode("utf-8")
|
|
457
|
+
(off,) = struct.unpack(e + "I", value_field)
|
|
458
|
+
return data[off:off + count].decode("utf-8")
|
|
459
|
+
return None
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
# ================================================================== Fassade
|
|
463
|
+
#: Registry: ``fmt -> (embed, extract)``.
|
|
464
|
+
_HANDLERS = {
|
|
465
|
+
"png": (_png_embed, _png_extract),
|
|
466
|
+
"jpeg": (_jpeg_embed, _jpeg_extract),
|
|
467
|
+
"jp2": (_jp2_embed, _jp2_extract),
|
|
468
|
+
"gif": (_gif_embed, _gif_extract),
|
|
469
|
+
"webp": (_webp_embed, _webp_extract),
|
|
470
|
+
"tiff": (_tiff_embed, _tiff_extract),
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def detect_format(data: bytes) -> Optional[str]:
|
|
475
|
+
"""Erkenne das Bildformat an den Magic-Bytes.
|
|
476
|
+
|
|
477
|
+
:param data: die Bild-Bytes.
|
|
478
|
+
:return: ``"png"``/``"jpeg"``/``"jp2"``/``"gif"``/``"webp"`` oder ``None``.
|
|
479
|
+
"""
|
|
480
|
+
if data.startswith(_PNG_MAGIC):
|
|
481
|
+
return "png"
|
|
482
|
+
if data.startswith(_JPEG_MAGIC):
|
|
483
|
+
return "jpeg"
|
|
484
|
+
if data.startswith(_JP2_MAGIC):
|
|
485
|
+
return "jp2"
|
|
486
|
+
if data[:6] in _GIF_MAGICS:
|
|
487
|
+
return "gif"
|
|
488
|
+
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
|
489
|
+
return "webp"
|
|
490
|
+
if data[:4] in (_TIFF_LE, _TIFF_BE):
|
|
491
|
+
return "tiff"
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def supported_formats() -> Tuple[str, ...]:
|
|
496
|
+
"""Die unterstützten Format-Schlüssel (Reihenfolge der Registry)."""
|
|
497
|
+
return tuple(_HANDLERS)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def embed(data: bytes, payload: str, fmt: Optional[str] = None) -> bytes:
|
|
501
|
+
"""Bette ``payload`` (Text) nativ in die Bild-Bytes ein (replace-Semantik).
|
|
502
|
+
|
|
503
|
+
:param data: die Original-Bild-Bytes.
|
|
504
|
+
:param payload: der einzubettende Text (i. d. R. sdata-Metadaten-JSON).
|
|
505
|
+
:param fmt: Format-Schlüssel; ``None`` → automatische Erkennung.
|
|
506
|
+
:return: neue Bild-Bytes mit eingebetteter sdata-Nutzlast.
|
|
507
|
+
:raises UnsupportedImageFormatError: wenn das Format unbekannt/nicht unterstützt ist.
|
|
508
|
+
:raises PayloadTooLargeError: wenn die Nutzlast nicht in ein Segment passt (JPEG).
|
|
509
|
+
"""
|
|
510
|
+
fmt = fmt or detect_format(data)
|
|
511
|
+
if fmt not in _HANDLERS:
|
|
512
|
+
raise UnsupportedImageFormatError(
|
|
513
|
+
f"unsupported image format: {fmt!r} "
|
|
514
|
+
f"(supported: {', '.join(_HANDLERS)})")
|
|
515
|
+
result = _HANDLERS[fmt][0](data, payload)
|
|
516
|
+
logger.debug("embedded %d-byte sdata payload into %s image", len(payload), fmt)
|
|
517
|
+
return result
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def extract(data: bytes, fmt: Optional[str] = None) -> Optional[str]:
|
|
521
|
+
"""Lies eine eingebettete sdata-Nutzlast aus den Bild-Bytes (Pillow-frei).
|
|
522
|
+
|
|
523
|
+
Lenient beim Lesen: unbekannte/nicht unterstützte Formate liefern ``None``
|
|
524
|
+
(kein Fehler), ebenso Bilder ohne eingebettete sdata-Nutzlast.
|
|
525
|
+
|
|
526
|
+
:param data: die Bild-Bytes.
|
|
527
|
+
:param fmt: Format-Schlüssel; ``None`` → automatische Erkennung.
|
|
528
|
+
:return: die eingebettete Nutzlast (Text) oder ``None``.
|
|
529
|
+
"""
|
|
530
|
+
fmt = fmt or detect_format(data)
|
|
531
|
+
if fmt not in _HANDLERS:
|
|
532
|
+
return None
|
|
533
|
+
return _HANDLERS[fmt][1](data)
|