sdata 1.5.0__py3-none-any.whl

sdata/sclass/blob.py

@@ -0,0 +1,362 @@
+import base64
+import logging
+import hashlib
+import os
+import io
+import mimetypes
+import typing
+from typing import Any, List, Dict, Optional, Literal, Optional, Type
+from sdata.base import Base
+from sdata.sclass.content import ContentIntegrityMixin
+logger = logging.getLogger(__name__)
+
+# Import fsspec for handling various URI schemes (local file, S3, Zip, etc.)
+# Note: fsspec must be installed (pip install fsspec) and for S3, also install s3fs (pip install s3fs)
+try:
+    import fsspec
+except ImportError:  # pragma: no cover - optionales Backend (sdata[blob])
+    logger.warning('fsspec not installed')
+    fsspec = None
+
+class Blob(ContentIntegrityMixin, Base):
+    """
+    A derived class from Base that represents a generic binary large object (Blob).
+    Stores the content in self.data['content'] as a dictionary with:
+    - 'type': 'bytes' for in-memory bytes (base64-encoded for serialization) or 'uri' for a filesystem URI (local path, S3 object, Zip path, etc., handled via fsspec).
+    - 'value': The base64-encoded bytes string (for 'bytes') or the URI string (for 'uri').
+    - 'filetype': The file type (e.g., 'pdf', 'png', 'jpg', 'txt', or any custom type). This is always stored and serialized.
+
+    Additionally, integrates hash calculations (SHA1 and MD5) from the provided class for integrity checks.
+    The actual bytes are loaded lazily when accessed via .content_bytes property, ensuring large content is not loaded unless explicitly requested.
+    For serialization in to_dict(), bytes are base64-encoded if type is 'bytes'; URIs are kept as-is.
+    Supports PDFs, images (png, jpg), or any arbitrary file types.
+    Uses fsspec to handle various URI schemes:
+    - Local file: 'file:///path/to/file.pdf' or simply '/path/to/file.pdf'
+    - S3: 's3://bucket/key.pdf'
+    - Zip: 'zip://innerfile.txt::/path/to/outer.zip'
+    """
+
+    ContentType = Literal['bytes', 'uri']
+
+    DEFAULT_METADATA = {
+    # "uuid": {
+    #     "name": "uuid",
+    #     "value": "123e4567-e89b-12d3-a456-426614174000",
+    #     "unit": None,
+    #     "dtype": "str",
+    #     "description": "A universally unique identification number (Universally Unique Identifier) that makes the asset globally unique. It is used to avoid duplicates and enable references.",
+    #     "label": "UUID",
+    #     "required": True,
+    #     "ontology": "saf:uuid"
+    # },
+    "checksum": {
+        "name": "checksum",
+        "value": None, #"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+        "unit": None,
+        "dtype": "str",
+        "description": "A hash value (based on the SHA-256 algorithm) that verifies the integrity of the asset. It ensures that the resource has not been modified (e.g., during downloads or storage).",
+        "label": "Checksum (SHA-256)",
+        "required": True,
+        "ontology": "schema:sha256"
+    },
+    "mime_type": {
+        "name": "mime_type",
+        "value": None, #"application/pdf",
+        "unit": None,
+        "dtype": "str",
+        "description": "The MIME type of the resource (e.g., 'application/pdf' for PDFs or 'text/html' for web pages), which indicates how the asset should be interpreted and processed.",
+        "label": "MIME Type",
+        "required": True,
+        "ontology": "dcat:mediaType"
+    },
+    "source_uri": {
+        "name": "source_uri",
+        "value": None, #"https://example.com/original/file.pdf",
+        "unit": None,
+        "dtype": "str",
+        "description": "The original URI (Uniform Resource Identifier) from which the asset originates (e.g., a URL to a website or file).",
+        "label": "Source URI",
+        "required": False,
+        "ontology": "dcterms:source"
+    },
+    "creation_date": {
+        "name": "creation_date",
+        "value": None, #"2025-10-16T12:00:00Z",
+        "unit": None,
+        "dtype": "datetime",
+        "description": "The creation date of the asset, to track the history.",
+        "label": "Creation Date",
+        "required": True,
+        "ontology": "dcterms:created"
+    },
+    "modified_date": {
+        "name": "modified_date",
+        "value": None, #"2025-10-16T14:30:00Z",
+        "unit": None,
+        "dtype": "datetime",
+        "description": "The date of the last modification, important for version control.",
+        "label": "Modified Date",
+        "required": False,
+        "ontology": "dcterms:modified"
+    },
+    "publisher": {
+        "name": "publisher",
+        "value": None, #"ISO",
+        "unit": None,
+        "dtype": "str",
+        "description": "The publisher or owner of the resource (e.g., an organization like ISO or a company).",
+        "label": "Publisher",
+        "required": False,
+        "ontology": "dcterms:publisher"
+    },
+    "license": {
+        "name": "license",
+        "value": None, #"Creative Commons BY-SA 4.0",
+        "unit": None,
+        "dtype": "str",
+        "description": "Information about the license (e.g., Creative Commons, proprietary), which regulates the usage rights.",
+        "label": "License",
+        "required": False,
+        "ontology": "dcterms:license"
+    }
+}
+
+    def __init__(
+            self,
+            content_type: Optional[ContentType] = 'bytes',
+            value: Optional[Any] = None,  # bytes for 'bytes', str URI for 'uri'
+            filetype: Optional[str] = 'binary',  # Default to generic binary, e.g., 'pdf', 'png', 'jpg'
+            **kwargs: Any
+    ) -> None:
+        """
+        Initialize Blob with content type, value, and filetype.
+        No content loading occurs here; loading is lazy via content_bytes property.
+
+        :param content_type: 'bytes' for in-memory bytes or 'uri' for filesystem URI (local, S3, Zip, etc.).
+        :param value: The content value - bytes object for 'bytes', URI string for 'uri' (e.g., '/local/path', 's3://bucket/key').
+        :param filetype: The type of the file (e.g., 'pdf', 'png', 'jpg', 'txt').
+        :param kwargs: Keyword arguments passed to Base.__init__ (e.g., name, description).
+        :raises ValueError: If invalid content_type or mismatched value type.
+        """
+        super().__init__(**kwargs)
+        self._content_cache = None  # nicht-serialisierter Lazy-Cache (nicht in self.data)
+        self.metadata.update_from_dict(self.DEFAULT_METADATA)
+
+        if content_type not in typing.get_args(self.ContentType):
+            raise ValueError(f"Invalid content_type: {content_type}")
+
+        self.data['content'] = {
+            'type': content_type,
+            'filetype': filetype,
+        }
+        self._set_value(value)
+        self._autofill_metadata()
+
+        logger.debug(f"Created Blob '{self.sname}' with content_type '{content_type}' and filetype '{filetype}'")
+
+    def _autofill_metadata(self) -> None:
+        """Fill derivable metadata: ``mime_type`` from ``filetype`` and
+        ``creation_date`` from the reserved ``_sdata_ctime`` (no second timestamp).
+        Only sets values that can be derived; leaves the rest untouched.
+        """
+        mime = mimetypes.guess_type(f"x.{self.filetype}")[0]
+        if mime:
+            self.metadata.set_attr("mime_type", mime)
+        ctime = self.metadata.get(self.SDATA_CTIME)
+        if ctime is not None and ctime.value:
+            self.metadata.set_attr("creation_date", ctime.value)
+
+    def _set_value(self, value: Any) -> None:
+        """
+        Set the 'value' in self.data['content'] based on content_type.
+        For 'bytes', store as base64-encoded string; for 'uri', store as str.
+        Handle value=None explicitly to avoid errors during deserialization.
+        """
+        content = self.data['content']
+        ctype = content['type']
+
+        if value is None:
+            content['value'] = None
+            return
+
+        if ctype == 'bytes':
+            if not isinstance(value, bytes):
+                raise ValueError("For 'bytes' type, value must be a bytes object.")
+            content['value'] = base64.b64encode(value).decode('utf-8')
+        elif ctype == 'uri':
+            if not isinstance(value, str):
+                raise ValueError("For 'uri' type, value must be a string URI.")
+            content['value'] = value
+        else:
+            raise ValueError(f"Unknown content_type: {ctype}")
+
+    def set_content(
+            self,
+            content_type: ContentType,
+            value: Any,
+            filetype: Optional[str] = None
+    ) -> None:
+        """
+        Update the content type, value, and optionally filetype.
+        Clears any cached content to maintain lazy loading.
+
+        :param content_type: New content_type ('bytes' or 'uri').
+        :param value: New value (bytes or URI str).
+        :param filetype: Optional new filetype (e.g., 'pdf', 'png', 'jpg').
+        """
+        self.data['content']['type'] = content_type
+        if filetype is not None:
+            self.data['content']['filetype'] = filetype
+        self._set_value(value)
+        self._content_cache = None  # Clear cache for lazy reloading
+        self._autofill_metadata()
+        logger.debug(
+            f"Updated Blob '{self.sname}' to content_type '{content_type}' and filetype '{self.data['content']['filetype']}'")
+
+    @property
+    def content_bytes(self) -> bytes:
+        """
+        Lazily load and retrieve the content as bytes (only when this property is accessed).
+        If type is 'uri', use fsspec to open and read; if 'bytes', decode from base64.
+        Caches the result for subsequent accesses.
+
+        :return: The content as bytes.
+        :raises ValueError: If loading fails or no value set.
+        :raises Exception: If fsspec encounters an error (e.g., invalid URI, missing dependencies like s3fs for S3).
+        """
+        if getattr(self, '_content_cache', None) is not None:
+            return self._content_cache
+
+        content = self.data.get('content')
+        if content is None:
+            raise ValueError("No content set in Blob.")
+
+        ctype = content.get('type')
+        val = content.get('value')
+        if val is None:
+            raise ValueError("No value set in content.")
+
+        if ctype == 'bytes':
+            loaded_bytes = base64.b64decode(val)
+        elif ctype == 'uri':
+            if fsspec is None:
+                raise ImportError("fsspec is required for URI loading.")
+            try:
+                with fsspec.open(val, 'rb') as f:
+                    loaded_bytes = f.read()
+            except Exception as e:
+                raise ValueError(f"Failed to load from URI '{val}': {str(e)}")
+        else:
+            raise ValueError(f"Unknown content_type: {ctype}")
+
+        self._content_cache = loaded_bytes  # Cache the loaded bytes (instance, not serialized)
+        return loaded_bytes
+
+    @property
+    def filetype(self) -> str:
+        """
+        Retrieve the filetype (no content loading required).
+
+        :return: The filetype string (e.g., 'pdf', 'png', 'jpg').
+        """
+        return self.data['content'].get('filetype', 'binary')
+
+    def exists(self) -> bool:
+        """
+        Test whether the blob content exists.
+        For 'uri', checks if the path/URI is accessible via fsspec; for 'bytes', always True if value set.
+        """
+        content = self.data.get('content')
+        if content is None:
+            return False
+
+        ctype = content.get('type')
+        val = content.get('value')
+        if val is None:
+            return False
+
+        if ctype == 'bytes':
+            return True  # In-memory, assumes exists if set
+        elif ctype == 'uri':
+            if fsspec is None:
+                logger.warning('fsspec not installed')
+                return False
+            try:
+                fs, _, paths = fsspec.core.get_fs_token_paths(val)
+                return fs.exists(paths[0])
+            except Exception as e:
+                logger.warning(f"Failed to check existence for URI '{val}': {str(e)}")
+                return False
+        else:
+            return False
+
+    def write(self, uri: str, **kwargs: Any) -> str:
+        """Write the content to a destination ``uri`` via fsspec (local/S3/zip/…).
+
+        :param uri: destination URI/path (e.g. ``/out/file.pdf``, ``s3://bucket/key``).
+        :param kwargs: forwarded to ``fsspec.open``.
+        :return: the destination ``uri``.
+        :raises ImportError: if fsspec is not installed (``pip install sdata[blob]``).
+        """
+        if fsspec is None:
+            raise ImportError("fsspec is required for write() (pip install sdata[blob]).")
+        data = self.content_bytes
+        with fsspec.open(uri, "wb", **kwargs) as f:
+            f.write(data)
+        logger.info(f"Blob content written to {uri}")
+        return uri
+
+    def open(self, mode: str = "rb"):
+        """Return a file-like handle to the content; use as a context manager.
+
+        For ``uri`` content a **streaming** fsspec handle is returned (no full
+        in-memory load); for ``bytes`` content an :class:`io.BytesIO` over the
+        decoded bytes.
+
+        :param mode: open mode (default ``"rb"``).
+        :raises ValueError: if no content/value is set or the content_type is unknown.
+        :raises ImportError: if fsspec is required (uri) but not installed.
+        """
+        content = self.data.get('content')
+        if content is None or content.get('value') is None:
+            raise ValueError("No content set in Blob.")
+        ctype = content['type']
+        if ctype == 'uri':
+            if fsspec is None:
+                raise ImportError("fsspec is required for uri open() (pip install sdata[blob]).")
+            return fsspec.open(content['value'], mode)
+        elif ctype == 'bytes':
+            return io.BytesIO(self.content_bytes)
+        else:
+            raise ValueError(f"Unknown content_type: {ctype}")
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Extend Base.to_dict to include the content dict as-is (with base64 for bytes and filetype).
+        Does not include or load the actual content bytes.
+        """
+        data_copy = self.data.copy()
+        result = super().to_dict()
+        result['data'] = data_copy
+        return result
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> 'Blob':
+        """
+        Create a Blob instance from a dictionary.
+        Restores content dict including filetype; no content loading occurs here (lazy via content_bytes).
+        """
+        instance = super().from_dict(d)
+        if 'content' not in instance.data:
+            instance.data['content'] = {'type': 'bytes', 'filetype': 'binary'}
+        # Validate content structure
+        content = instance.data['content']
+        if 'type' not in content or 'filetype' not in content:
+            raise ValueError("Invalid content structure in dict.")
+        if content['type'] == 'bytes' and 'value' in content:
+            # Ensure value is base64 string; no need to decode here
+            pass
+        elif content['type'] == 'uri' and 'value' in content:
+            if not isinstance(content['value'], str):
+                raise ValueError("URI value must be a string.")
+        return instance

sdata/sclass/content.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""Wiederverwendbarer Integritäts-/Hash-Layer (RFC 0004, Option B).
+
+:class:`ContentIntegrityMixin` stellt Prüfsummen (``sha1``/``md5``/``sha256``),
+``size`` sowie ``update_checksum``/``verify`` über einen ``content_bytes``-Hook und
+das ``metadata``-Objekt (aus :class:`~sdata.base.Base`) bereit. Genutzt von
+:class:`~sdata.sclass.blob.Blob` und :class:`~sdata.sclass.dataframe.DataFrame`.
+"""
+import io
+import hashlib
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ContentIntegrityMixin:
+    """Hash/``verify``/``size`` über ``self.content_bytes`` + ``self.metadata``.
+
+    Subklassen liefern eine ``content_bytes``-Property (``bytes``) und besitzen
+    (über :class:`~sdata.base.Base`) ein ``metadata``-Objekt.
+    """
+
+    def _update_hash(self, hash_obj: Any, buffer_size: int = 65536) -> None:
+        """Speise den Hash-Objekt-Stream aus ``content_bytes`` (chunked)."""
+        bytes_io = io.BytesIO(self.content_bytes)
+        while True:
+            data = bytes_io.read(buffer_size)
+            if not data:
+                break
+            hash_obj.update(data)
+
+    def _content_hexdigest(self, algo) -> Optional[str]:
+        """Hex-Digest des Inhalts mit ``algo`` (z. B. ``hashlib.sha256``); ``None`` bei Fehler."""
+        try:
+            hash_obj = algo()
+            self._update_hash(hash_obj)
+            return hash_obj.hexdigest()
+        except Exception as exp:
+            logger.error(f"Failed to compute {algo().name}: {exp}")
+            return None
+
+    @property
+    def sha1(self) -> Optional[str]:
+        """SHA-1 hex digest of the content (lazy; ``None`` on error)."""
+        return self._content_hexdigest(hashlib.sha1)
+
+    @property
+    def md5(self) -> Optional[str]:
+        """MD5 hex digest of the content (lazy; ``None`` on error)."""
+        return self._content_hexdigest(hashlib.md5)
+
+    @property
+    def sha256(self) -> Optional[str]:
+        """SHA-256 hex digest of the content (lazy; ``None`` on error)."""
+        return self._content_hexdigest(hashlib.sha256)
+
+    @property
+    def size(self) -> Optional[int]:
+        """Size of the content in bytes (lazy; ``None`` on error)."""
+        try:
+            return len(self.content_bytes)
+        except Exception as exp:
+            logger.error(f"Failed to determine size: {exp}")
+            return None
+
+    def update_checksum(self) -> Optional[str]:
+        """Store the SHA-256 of the content in the ``checksum`` metadata (``schema:sha256``).
+
+        :return: the stored SHA-256 hex digest (``None`` if the content is unavailable).
+        """
+        digest = self.sha256
+        self.metadata.set_attr("checksum", digest)
+        return digest
+
+    def verify(self) -> bool:
+        """Verify the content against the stored ``checksum`` (SHA-256) metadata.
+
+        :return: ``True`` iff a checksum is stored and matches the current content;
+          ``False`` on mismatch or when no checksum has been stored yet.
+        """
+        attr = self.metadata.get("checksum")
+        stored = attr.value if attr is not None else None
+        if not stored:
+            logger.warning("verify: no checksum stored (call update_checksum first)")
+            return False
+        return stored == self.sha256