httk-core 2.0.0__py3-none-any.whl

httk/core/datastream/compression.py

@@ -0,0 +1,183 @@
+import bz2
+import gzip
+import io
+import lzma
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import IO, cast
+
+
+@dataclass(frozen=True)
+class CompressionCodec:
+    """
+    A decompression codec for a single container format.
+
+    A codec is an orthogonal layer below the datastream backends: it turns a compressed
+    binary stream into an uncompressed binary stream, independently of where the compressed
+    bytes come from (a filename, an open file, raw bytes, or a remote response).
+    """
+
+    name: str
+    """Canonical, lower-case codec name (e.g. ``"gzip"``); also how an explicit hint selects it."""
+
+    extensions: tuple[str, ...]
+    """Recognized filename suffixes including the leading dot (e.g. ``(".gz",)``)."""
+
+    magics: tuple[bytes, ...]
+    """Leading magic-byte signatures; an empty tuple means the format cannot be sniffed."""
+
+    open_stream: Callable[[io.IOBase], io.IOBase]
+    """Wrap a compressed binary stream and return a readable, decompressed binary stream."""
+
+
+_registry: dict[str, CompressionCodec] = {}
+_MODES = frozenset({"auto", "detect", "extension", "none"})
+
+
+def register_compression(codec: CompressionCodec) -> None:
+    """Register (or replace) a codec under its :attr:`~CompressionCodec.name` (case-insensitive)."""
+    _registry[codec.name.lower()] = codec
+
+
+def known_compressions() -> list[str]:
+    """Return the registered codec names, in registration order."""
+    return list(_registry)
+
+
+def codec_for_name(name: str) -> CompressionCodec | None:
+    """Return the codec whose extension matches the trailing suffix of ``name``, else ``None``."""
+    lowered = name.lower()
+    for codec in _registry.values():
+        for ext in codec.extensions:
+            if lowered.endswith(ext.lower()):
+                return codec
+    return None
+
+
+def split_compression_suffix(name: str) -> tuple[str, CompressionCodec | None]:
+    """
+    Split a trailing compression extension off ``name``.
+
+    ``"data.json.gz"`` becomes ``("data.json", <gzip codec>)``; a name with no recognized
+    compression extension is returned unchanged with ``None``.
+    """
+    codec = codec_for_name(name)
+    if codec is None:
+        return name, None
+    lowered = name.lower()
+    for ext in codec.extensions:
+        if lowered.endswith(ext.lower()):
+            return name[: -len(ext)], codec
+    return name, codec
+
+
+def _max_magic_len() -> int:
+    return max((len(magic) for codec in _registry.values() for magic in codec.magics), default=0)
+
+
+def _match_magic(prefix: bytes) -> CompressionCodec | None:
+    for codec in _registry.values():
+        for magic in codec.magics:
+            if magic and prefix.startswith(magic):
+                return codec
+    return None
+
+
+def sniff_codec(stream: io.IOBase) -> tuple[io.IOBase, CompressionCodec | None]:
+    """
+    Detect a codec from the leading magic bytes of ``stream`` without consuming data.
+
+    A seekable stream is read and rewound; an unseekable stream is peeked (directly when it
+    supports ``peek``, otherwise via a wrapping :class:`io.BufferedReader`). The returned
+    stream must be used in place of the input, since it may be the wrapper.
+    """
+    max_len = _max_magic_len()
+    if max_len == 0:
+        return stream, None
+    if stream.seekable():
+        reader = cast(io.BufferedIOBase, stream)
+        pos = reader.tell()
+        prefix = reader.read(max_len)
+        reader.seek(pos)
+        return stream, _match_magic(prefix)
+    peek = getattr(stream, "peek", None)
+    if callable(peek):
+        return stream, _match_magic(cast(bytes, peek(max_len))[:max_len])
+    buffered = io.BufferedReader(cast(io.RawIOBase, stream))
+    return buffered, _match_magic(buffered.peek(max_len)[:max_len])
+
+
+def open_compressed(stream: io.IOBase, *, compression: str = "auto", name: str | None = None) -> io.IOBase:
+    """
+    Return a decompressed view of ``stream`` according to the ``compression`` hint.
+
+    Values are ``"none"`` (passthrough), ``"extension"`` (decide from ``name`` only),
+    ``"detect"`` (always sniff magic bytes), ``"auto"`` (extension if recognized, else sniff),
+    or a registered codec name (force that codec; an unknown name raises :class:`ValueError`).
+    When no codec applies the stream is returned unchanged.
+    """
+    token = compression.lower()
+    if token == "none":
+        return stream
+    if token == "extension":
+        codec = codec_for_name(name) if name is not None else None
+        return codec.open_stream(stream) if codec is not None else stream
+    if token == "auto":
+        codec = codec_for_name(name) if name is not None else None
+        if codec is not None:
+            return codec.open_stream(stream)
+        stream, codec = sniff_codec(stream)
+        return codec.open_stream(stream) if codec is not None else stream
+    if token == "detect":
+        stream, codec = sniff_codec(stream)
+        return codec.open_stream(stream) if codec is not None else stream
+    codec = _registry.get(token)
+    if codec is None:
+        raise ValueError(f"unknown compression codec {compression!r}; known codecs: {known_compressions()}")
+    return codec.open_stream(stream)
+
+
+def validate_compression(compression: str) -> None:
+    """Raise :class:`ValueError` unless ``compression`` is a known mode or registered codec name."""
+    token = compression.lower()
+    if token in _MODES or token in _registry:
+        return
+    raise ValueError(
+        f"unknown compression {compression!r}; expected one of {sorted(_MODES)} "
+        f"or a registered codec name ({known_compressions()})"
+    )
+
+
+def reject_text_native_compression(compression: str | None) -> None:
+    """
+    Validate a ``compression`` hint for a text-native source (an open text stream or a string).
+
+    Such sources carry no compressed bytes to decode, so only the no-op modes ``"auto"``,
+    ``"extension"``, and ``"none"`` are accepted; a codec name or ``"detect"`` raises
+    :class:`ValueError`. ``None`` (no hint given) is accepted.
+    """
+    if compression is None or compression.lower() in ("auto", "extension", "none"):
+        return
+    raise ValueError(f"compression={compression!r} does not apply to a text-native source")
+
+
+def _open_gzip(stream: io.IOBase) -> io.IOBase:
+    return gzip.GzipFile(fileobj=cast(IO[bytes], stream))
+
+
+def _open_bzip2(stream: io.IOBase) -> io.IOBase:
+    return bz2.BZ2File(cast(IO[bytes], stream))
+
+
+def _open_xz(stream: io.IOBase) -> io.IOBase:
+    return lzma.LZMAFile(cast(IO[bytes], stream))
+
+
+def _open_lzma(stream: io.IOBase) -> io.IOBase:
+    return lzma.LZMAFile(cast(IO[bytes], stream), format=lzma.FORMAT_ALONE)
+
+
+register_compression(CompressionCodec("gzip", (".gz",), (b"\x1f\x8b",), _open_gzip))
+register_compression(CompressionCodec("bzip2", (".bz2",), (b"BZh",), _open_bzip2))
+register_compression(CompressionCodec("xz", (".xz",), (b"\xfd7zXZ\x00",), _open_xz))
+register_compression(CompressionCodec("lzma", (".lzma",), (), _open_lzma))

httk/core/datastream/textstream_api.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+
+
+class TextstreamAPI(ABC):
+    """
+    Abstract base class for a bare minimum API for streamable text data.
+
+    Supports:
+    * read, close, name, and closed with the meanings defined by io.IOBase
+
+    Since it is a *minimal* streaming data API it deliberately omits:
+    seek, tell, etc.; there should be no assumption that the underlying
+    data source is seekable. However, many backend implementations
+    may chose to support them.
+    """
+
+    @abstractmethod
+    def read(self, size: int = -1) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    def close(self) -> None:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def name(self) -> str | None:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def closed(self) -> bool:
+        raise NotImplementedError

httk/core/datastream/textstream_backend.py

@@ -0,0 +1,12 @@
+from typing import Any, ClassVar
+
+from ..views import Backend
+from .textstream_api import TextstreamAPI
+
+
+class TextstreamBackend(Backend["TextstreamBackend"], TextstreamAPI):
+    """
+    Abstract base class for all backends of streaming text data.
+    """
+
+    backend_classes: ClassVar[list[type[Backend[Any]]]]

httk/core/datastream/textstream_common.py

@@ -0,0 +1,37 @@
+import io
+import os
+from abc import ABC, abstractmethod
+
+
+class TextstreamCommon(ABC):
+    """
+    Common superclass for many of the implementations of backends for streaming text data.
+    """
+
+    _f: io.TextIOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    @abstractmethod
+    def _ensure_f(self) -> io.TextIOBase:
+        pass
+
+    def unwrap(self) -> io.TextIOBase:
+        return self._ensure_f()
+
+    def read(self, size: int = -1) -> str:
+        return self._ensure_f().read(size)
+
+    def close(self) -> None:
+        if self._f is not None and not self._f.closed:
+            self._f.close()
+        # A text wrapper closes the stream it wraps but not a decompression source below it.
+        if self._underlying is not None and self._underlying is not self._f and not self._underlying.closed:
+            self._underlying.close()
+        self._closed = True
+
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        return self._ensure_f().seek(offset, whence)
+
+    def tell(self) -> int:
+        return self._ensure_f().tell()

httk/core/datastream/textstream_file.py

@@ -0,0 +1,45 @@
+import io
+from typing import Any
+
+from .compression import reject_text_native_compression
+from .textstream_backend import TextstreamBackend
+from .textstream_common import TextstreamCommon
+
+
+class TextstreamFile(TextstreamCommon, TextstreamBackend):
+    """
+    Backend for file-based (io.TextIOBase-conforming) streaming text data
+    """
+
+    _f: io.TextIOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, obj: io.TextIOBase, **hints: Any) -> Any:
+        if not isinstance(obj, io.TextIOBase):
+            return None
+        if hints and hints.get("kind", "file") != "file":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, obj: io.TextIOBase, **hints: Any) -> None:
+        reject_text_native_compression(hints.get("compression"))
+        self._f = obj
+        # Compression cannot layer under an already-decoded text stream, so nothing to chain-close.
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.TextIOBase:
+        if self._f is None or self._f.closed:
+            raise ValueError("I/O operation on closed stream")
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        self._ensure_f()
+        return getattr(self._f, "name", None)
+
+    @property
+    def closed(self) -> bool:
+        return self._f is None or self._f.closed

httk/core/datastream/textstream_file_view.py

@@ -0,0 +1,201 @@
+import io
+from typing import Any, Iterator, NoReturn, Self
+
+from ..views import unwrap
+from .textstream_api import TextstreamAPI
+from .textstream_backend import TextstreamBackend
+from .textstream_like import TextstreamLike
+from .textstream_view import TextstreamView
+
+
+class TextstreamFileView(TextstreamView, io.TextIOBase, TextstreamAPI):
+    """
+    A view presenting an underlying data streaming backend via the full io.TextIOBase API, which is a superset of TextstreamAPI.
+    """
+
+    _backend: TextstreamBackend
+    _readline_buffer: str
+
+    def __new__(cls, obj: TextstreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        instance = super().__new__(cls)
+        instance._backend = backend
+        instance._readline_buffer = ""
+        return instance
+
+    def __init__(self, obj: TextstreamLike, **hints: Any) -> None:
+        pass
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)
+
+    @property
+    def name(self) -> str | None:
+        return self._backend.name
+
+    @property
+    def closed(self) -> bool:
+        return self._backend.closed
+
+    def close(self) -> None:
+        self._backend.close()
+
+    def readable(self) -> bool:
+        return True
+
+    def writable(self) -> bool:
+        return False
+
+    def seekable(self) -> bool:
+        return hasattr(self._backend, "seek") and hasattr(self._backend, "tell")
+
+    def flush(self) -> None:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        flush = getattr(self._backend, "flush", None)
+        if flush:
+            flush()
+
+    def read(self, size: int | None = -1) -> str:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+
+        if size is None or size < 0:
+            if self._readline_buffer:
+                prefix = self._readline_buffer
+                self._readline_buffer = ""
+                return prefix + self._backend.read()
+            return self._backend.read()
+
+        if size == 0:
+            return ""
+
+        if self._readline_buffer:
+            if len(self._readline_buffer) >= size:
+                out = self._readline_buffer[:size]
+                self._readline_buffer = self._readline_buffer[size:]
+                return out
+            prefix = self._readline_buffer
+            self._readline_buffer = ""
+            return prefix + self._backend.read(size - len(prefix))
+
+        return self._backend.read(size)
+
+    def readline(self, size: int | None = -1) -> str:  # type: ignore[override] ## https://github.com/python/mypy/issues/9643
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if size == 0:
+            return ""
+        if size is None:
+            size = -1
+
+        parts: list[str] = []
+        total = 0
+
+        while True:
+            if self._readline_buffer:
+                chunk = self._readline_buffer
+                self._readline_buffer = ""
+            else:
+                to_read = 8192
+                if size >= 0:
+                    remaining = size - total
+                    if remaining <= 0:
+                        break
+                    to_read = min(to_read, remaining)
+                chunk = self._backend.read(to_read)
+
+            if chunk == "":
+                break
+
+            newline_pos = chunk.find("\n")
+            if newline_pos != -1:
+                newline_pos += 1
+                take = chunk[:newline_pos]
+                rest = chunk[newline_pos:]
+
+                if size >= 0 and total + len(take) > size:
+                    cutoff = size - total
+                    parts.append(take[:cutoff])
+                    self._readline_buffer = take[cutoff:] + rest
+                    break
+
+                parts.append(take)
+                self._readline_buffer = rest
+                break
+
+            if size >= 0 and total + len(chunk) > size:
+                cutoff = size - total
+                parts.append(chunk[:cutoff])
+                self._readline_buffer = chunk[cutoff:]
+                break
+
+            parts.append(chunk)
+            total += len(chunk)
+
+        return "".join(parts)
+
+    def readlines(self, hint: int = -1) -> list[str]:  # type: ignore[override] ## https://github.com/python/mypy/issues/9643
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+
+        lines: list[str] = []
+        total = 0
+
+        while True:
+            line = self.readline()
+            if line == "":
+                break
+            lines.append(line)
+            total += len(line)
+            if hint >= 0 and total >= hint:
+                break
+
+        return lines
+
+    def __iter__(self) -> Iterator[str]:  # type: ignore[override] ## https://github.com/python/mypy/issues/9643
+        return self
+
+    def __next__(self) -> str:  # type: ignore[override] ## https://github.com/python/mypy/issues/9643
+        line = self.readline()
+        if line == "":
+            raise StopIteration
+        return line
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        seek = getattr(self._backend, "seek", None)
+        if not seek:
+            raise io.UnsupportedOperation("underlying stream is not seekable")
+
+        pos = seek(offset, whence)
+        self._readline_buffer = ""
+        return pos
+
+    def tell(self) -> int:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        tell = getattr(self._backend, "tell")
+        if not tell:
+            raise io.UnsupportedOperation("underlying stream does not support tell()")
+
+        pos = tell()
+        return pos - len(self._readline_buffer)
+
+    def detach(self) -> NoReturn:
+        raise io.UnsupportedOperation("detach")
+
+    @property
+    def encoding(self) -> str | None:  # type: ignore[override] ## these should not be writable
+        return getattr(self._backend, "encoding", None)
+
+    @property
+    def errors(self) -> str | None:  # type: ignore[override] ## these should not be writable
+        return getattr(self._backend, "errors", None)
+
+    @property
+    def newlines(self) -> str | tuple[str, ...] | None:  # type: ignore[override] ## these should not be writable
+        return getattr(self._backend, "newlines", None)

httk/core/datastream/textstream_filename.py

@@ -0,0 +1,55 @@
+import io
+from pathlib import Path
+from typing import Any, cast
+
+from .compression import open_compressed, validate_compression
+from .textstream_backend import TextstreamBackend
+from .textstream_common import TextstreamCommon
+
+
+class TextstreamFilename(TextstreamCommon, TextstreamBackend):
+    """
+    Backend for streaming text via operations on a file specfied by a filename
+    """
+
+    _filename: str
+    _compression: str
+    _encoding: str | None
+    _f: io.TextIOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, filename: str | Path, **hints: Any) -> Any:
+        if not isinstance(filename, str | Path):
+            return None
+        if hints and hints.get("kind", "filename") != "filename":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, filename: str | Path, **hints: Any) -> None:
+        self._filename = str(filename)
+        self._compression = hints.get("compression", "extension")
+        validate_compression(self._compression)
+        self._encoding = hints.get("encoding")
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.TextIOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            raw = open(self._filename, "rb")
+            decompressed = open_compressed(raw, compression=self._compression, name=self._filename)
+            self._underlying = raw if decompressed is not raw else None
+            self._f = io.TextIOWrapper(cast(io.BufferedReader, decompressed), encoding=self._encoding or "utf-8")
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return self._filename
+
+    @property
+    def closed(self) -> bool:
+        return self._closed

httk/core/datastream/textstream_filename_view.py

@@ -0,0 +1,35 @@
+from typing import Any, Self
+
+from ..views import unwrap
+from .textstream_backend import TextstreamBackend
+from .textstream_like import TextstreamLike
+from .textstream_view import TextstreamView
+
+
+class TextstreamFilenameView(TextstreamView, str):
+    """
+    A view presenting an underlying data streaming backend via a filename.
+    This view is probably mostly useful for providing filenames to functions that will open them.
+    Note: this view is not lazy (this is impossible for views inherting str, since str is immutable)
+
+    Raises TypeError if created with a streaming data source that do not come with a name, in the future
+    we may consider creating a temporary file to place the data in.
+    """
+
+    _backend: TextstreamBackend
+
+    def __new__(cls, obj: TextstreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        if backend.name is None:
+            raise TypeError("This backend cannot be represented as a filename (no underlying filename)")
+        instance = super().__new__(cls, backend.name)
+        instance._backend = backend
+        return instance
+
+    def __init__(self, obj: TextstreamLike, **hints: Any) -> None:
+        super().__init__()
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)

httk/core/datastream/textstream_like.py

@@ -0,0 +1,15 @@
+import io
+import pathlib
+import urllib.request
+
+from . import textstream_backend, textstream_view
+
+type TextstreamLike = (
+    textstream_backend.TextstreamBackend
+    | textstream_view.TextstreamView
+    | io.TextIOBase
+    | io.StringIO
+    | str
+    | pathlib.Path
+    | urllib.request.Request
+)

httk/core/datastream/textstream_request.py

@@ -0,0 +1,72 @@
+import io
+import urllib.parse
+import urllib.request
+from typing import Any, cast
+
+from .compression import open_compressed, validate_compression
+from .textstream_backend import TextstreamBackend
+from .textstream_common import TextstreamCommon
+
+
+class TextstreamRequest(TextstreamCommon, TextstreamBackend):
+    """
+    Backend for streaming text fetched via a urllib.request.Request.
+    """
+
+    _request: urllib.request.Request
+    _timeout: float | None
+    _encoding: str | None
+    _compression: str
+    _f: io.TextIOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, request: urllib.request.Request, **hints: Any) -> Any:
+        if not isinstance(request, urllib.request.Request):
+            return None
+        if hints and hints.get("kind", "request") != "request":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, request: urllib.request.Request, **hints: Any) -> None:
+        self._request = request
+        self._timeout = hints.get("timeout")
+        self._encoding = hints.get("encoding")
+        self._compression = hints.get("compression", "auto")
+        validate_compression(self._compression)
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.TextIOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            if self._timeout is None:
+                resp = urllib.request.urlopen(self._request)
+            else:
+                resp = urllib.request.urlopen(self._request, timeout=self._timeout)
+            encoding = self._encoding or resp.headers.get_content_charset() or "utf-8"
+            raw = cast(io.IOBase, resp)
+            name = urllib.parse.urlsplit(self._request.full_url).path
+            decompressed = open_compressed(raw, compression=self._compression, name=name)
+            self._underlying = raw if decompressed is not raw else None
+            self._f = io.TextIOWrapper(cast(io.BufferedReader, decompressed), encoding=encoding)
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return None
+
+    @property
+    def url(self) -> str:
+        return self._request.full_url
+
+    @property
+    def request(self) -> urllib.request.Request:
+        return self._request
+
+    @property
+    def closed(self) -> bool:
+        return self._closed