httk-core 2.0.0__py3-none-any.whl

httk/core/datastream/bytestream_bytes_view.py

@@ -0,0 +1,31 @@
+from typing import Any, Self
+
+from ..views import unwrap
+from .bytestream_backend import BytestreamBackend
+from .bytestream_like import BytestreamLike
+from .bytestream_view import BytestreamView
+
+
+class BytestreamBytesView(BytestreamView, bytes):
+    """
+    A view presenting underlying streaming byte data as bytes.
+    This view can be used both to pass bytes in place of streaming data, and for reading streaming data into bytes.
+    Note: this view is not lazy (this is impossible for views inheriting bytes, since bytes is immutable), hence all
+    streaming data is read immediately upon creating this view.
+    """
+
+    _backend: BytestreamBackend
+
+    def __new__(cls, obj: BytestreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        instance = super().__new__(cls, backend.read())
+        instance._backend = backend
+        return instance
+
+    def __init__(self, obj: BytestreamLike, **hints: Any) -> None:
+        super().__init__()
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)

httk/core/datastream/bytestream_common.py

@@ -0,0 +1,38 @@
+import io
+import os
+from abc import ABC, abstractmethod
+from typing import cast
+
+
+class BytestreamCommon(ABC):
+    """
+    Common superclass for many of the implementations of backends for streaming byte data.
+    """
+
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    @abstractmethod
+    def _ensure_f(self) -> io.IOBase:
+        pass
+
+    def unwrap(self) -> io.IOBase:
+        return self._ensure_f()
+
+    def read(self, size: int = -1) -> bytes:
+        return cast(bytes, self._ensure_f().read(size))
+
+    def close(self) -> None:
+        if self._f is not None and not self._f.closed:
+            self._f.close()
+        # A decompression wrapper does not close the source stream it reads from, so close it too.
+        if self._underlying is not None and self._underlying is not self._f and not self._underlying.closed:
+            self._underlying.close()
+        self._closed = True
+
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        return cast(int, self._ensure_f().seek(offset, whence))
+
+    def tell(self) -> int:
+        return cast(int, self._ensure_f().tell())

httk/core/datastream/bytestream_file.py

@@ -0,0 +1,53 @@
+import io
+from typing import Any
+
+from .bytestream_backend import BytestreamBackend
+from .bytestream_common import BytestreamCommon
+from .compression import open_compressed, validate_compression
+
+
+class BytestreamFile(BytestreamCommon, BytestreamBackend):
+    """
+    Backend for file-based (io.IOBase-conforming) streaming byte data.
+    """
+
+    _source: io.IOBase
+    _compression: str
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, obj: io.IOBase, **hints: Any) -> Any:
+        if not isinstance(obj, io.IOBase):
+            return None
+        if hints and hints.get("kind", "file") != "file":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, obj: io.IOBase, **hints: Any) -> None:
+        self._source = obj
+        self._compression = hints.get("compression", "auto")
+        validate_compression(self._compression)
+        self._f = None
+        # We adopt (and hence close) the caller's stream; keep it here so close() reaches it
+        # even when a decompression wrapper is layered on top and when no read ever happens.
+        self._underlying = obj
+        self._closed = False
+
+    def _ensure_f(self) -> io.IOBase:
+        if self._closed or self._source.closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            name = getattr(self._source, "name", None)
+            self._f = open_compressed(self._source, compression=self._compression, name=name)
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        self._ensure_f()
+        return getattr(self._source, "name", None)
+
+    @property
+    def closed(self) -> bool:
+        return self._closed or self._source.closed

httk/core/datastream/bytestream_file_view.py

@@ -0,0 +1,189 @@
+import io
+from typing import Any, Iterator, NoReturn, Self
+
+from ..views import unwrap
+from .bytestream_api import BytestreamAPI
+from .bytestream_backend import BytestreamBackend
+from .bytestream_like import BytestreamLike
+from .bytestream_view import BytestreamView
+
+
+class BytestreamFileView(BytestreamView, io.IOBase, BytestreamAPI):
+    """
+    A view presenting an underlying data streaming backend via an io.IOBase-like API.
+    """
+
+    _backend: BytestreamBackend
+    _readline_buffer: bytes
+
+    def __new__(cls, obj: BytestreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        instance = super().__new__(cls)
+        instance._backend = backend
+        instance._readline_buffer = b""
+        return instance
+
+    def __init__(self, obj: BytestreamLike, **hints: Any) -> None:
+        pass
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)
+
+    @property
+    def name(self) -> str | None:
+        return self._backend.name
+
+    @property
+    def closed(self) -> bool:
+        return self._backend.closed
+
+    def close(self) -> None:
+        self._backend.close()
+
+    def readable(self) -> bool:
+        return True
+
+    def writable(self) -> bool:
+        return False
+
+    def seekable(self) -> bool:
+        return hasattr(self._backend, "seek") and hasattr(self._backend, "tell")
+
+    def flush(self) -> None:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        flush = getattr(self._backend, "flush", None)
+        if flush:
+            flush()
+
+    def read(self, size: int | None = -1) -> bytes:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+
+        if size is None or size < 0:
+            if self._readline_buffer:
+                prefix = self._readline_buffer
+                self._readline_buffer = b""
+                return prefix + self._backend.read()
+            return self._backend.read()
+
+        if size == 0:
+            return b""
+
+        if self._readline_buffer:
+            if len(self._readline_buffer) >= size:
+                out = self._readline_buffer[:size]
+                self._readline_buffer = self._readline_buffer[size:]
+                return out
+            prefix = self._readline_buffer
+            self._readline_buffer = b""
+            return prefix + self._backend.read(size - len(prefix))
+
+        return self._backend.read(size)
+
+    def readline(self, size: int | None = -1) -> bytes:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        if size == 0:
+            return b""
+        if size is None:
+            size = -1
+
+        parts: list[bytes] = []
+        total = 0
+
+        while True:
+            if self._readline_buffer:
+                chunk = self._readline_buffer
+                self._readline_buffer = b""
+            else:
+                to_read = 8192
+                if size >= 0:
+                    remaining = size - total
+                    if remaining <= 0:
+                        break
+                    to_read = min(to_read, remaining)
+                chunk = self._backend.read(to_read)
+
+            if chunk == b"":
+                break
+
+            newline_pos = chunk.find(b"\n")
+            if newline_pos != -1:
+                newline_pos += 1
+                take = chunk[:newline_pos]
+                rest = chunk[newline_pos:]
+
+                if size >= 0 and total + len(take) > size:
+                    cutoff = size - total
+                    parts.append(take[:cutoff])
+                    self._readline_buffer = take[cutoff:] + rest
+                    break
+
+                parts.append(take)
+                self._readline_buffer = rest
+                break
+
+            if size >= 0 and total + len(chunk) > size:
+                cutoff = size - total
+                parts.append(chunk[:cutoff])
+                self._readline_buffer = chunk[cutoff:]
+                break
+
+            parts.append(chunk)
+            total += len(chunk)
+
+        return b"".join(parts)
+
+    def readlines(self, hint: int = -1) -> list[bytes]:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+
+        lines: list[bytes] = []
+        total = 0
+
+        while True:
+            line = self.readline()
+            if line == b"":
+                break
+            lines.append(line)
+            total += len(line)
+            if hint >= 0 and total >= hint:
+                break
+
+        return lines
+
+    def __iter__(self) -> Iterator[bytes]:
+        return self
+
+    def __next__(self) -> bytes:
+        line = self.readline()
+        if line == b"":
+            raise StopIteration
+        return line
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        seek = getattr(self._backend, "seek", None)
+        if not seek:
+            raise io.UnsupportedOperation("underlying stream is not seekable")
+
+        pos = seek(offset, whence)
+        self._readline_buffer = b""
+        return pos
+
+    def tell(self) -> int:
+        if self.closed:
+            raise ValueError("I/O operation on closed file.")
+        tell = getattr(self._backend, "tell", None)
+        if not tell:
+            raise io.UnsupportedOperation("underlying stream does not support tell()")
+
+        pos = tell()
+        return pos - len(self._readline_buffer)
+
+    def detach(self) -> NoReturn:
+        raise io.UnsupportedOperation("detach")

httk/core/datastream/bytestream_filename.py

@@ -0,0 +1,53 @@
+import io
+from pathlib import Path
+from typing import Any
+
+from .bytestream_backend import BytestreamBackend
+from .bytestream_common import BytestreamCommon
+from .compression import open_compressed, validate_compression
+
+
+class BytestreamFilename(BytestreamCommon, BytestreamBackend):
+    """
+    Backend for streaming byte data via operations on a file specified by a filename.
+    """
+
+    _filename: str
+    _compression: str
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, filename: str | Path, **hints: Any) -> Any:
+        if not isinstance(filename, str | Path):
+            return None
+        if hints and hints.get("kind", "filename") != "filename":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, filename: str | Path, **hints: Any) -> None:
+        self._filename = str(filename)
+        self._compression = hints.get("compression", "extension")
+        validate_compression(self._compression)
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.IOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            raw = open(self._filename, "rb")
+            opened = open_compressed(raw, compression=self._compression, name=self._filename)
+            self._underlying = raw if opened is not raw else None
+            self._f = opened
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return self._filename
+
+    @property
+    def closed(self) -> bool:
+        return self._closed

httk/core/datastream/bytestream_filename_view.py

@@ -0,0 +1,34 @@
+from typing import Any, Self
+
+from ..views import unwrap
+from .bytestream_backend import BytestreamBackend
+from .bytestream_like import BytestreamLike
+from .bytestream_view import BytestreamView
+
+
+class BytestreamFilenameView(BytestreamView, str):
+    """
+    A view presenting an underlying data streaming backend via a filename.
+    This view is mostly useful for providing filenames to functions that will open them.
+    Note: this view is not lazy (this is impossible for views inheriting str, since str is immutable).
+
+    Raises TypeError if created with a streaming data source that does not come with a name.
+    """
+
+    _backend: BytestreamBackend
+
+    def __new__(cls, obj: BytestreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        if backend.name is None:
+            raise TypeError("This backend cannot be represented as a filename (no underlying filename)")
+        instance = super().__new__(cls, backend.name)
+        instance._backend = backend
+        return instance
+
+    def __init__(self, obj: BytestreamLike, **hints: Any) -> None:
+        super().__init__()
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)

httk/core/datastream/bytestream_like.py

@@ -0,0 +1,17 @@
+import io
+import pathlib
+import urllib.request
+
+from . import bytestream_backend, bytestream_view
+
+type BytestreamLike = (
+    bytestream_backend.BytestreamBackend
+    | bytestream_view.BytestreamView
+    | io.IOBase
+    | io.BytesIO
+    | bytes
+    | bytearray
+    | str
+    | pathlib.Path
+    | urllib.request.Request
+)

httk/core/datastream/bytestream_request.py

@@ -0,0 +1,69 @@
+import io
+import urllib.parse
+import urllib.request
+from typing import Any, cast
+
+from .bytestream_backend import BytestreamBackend
+from .bytestream_common import BytestreamCommon
+from .compression import open_compressed, validate_compression
+
+
+class BytestreamRequest(BytestreamCommon, BytestreamBackend):
+    """
+    Backend for streaming byte data fetched via a urllib.request.Request.
+    """
+
+    _request: urllib.request.Request
+    _timeout: float | None
+    _compression: str
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, request: urllib.request.Request, **hints: Any) -> Any:
+        if not isinstance(request, urllib.request.Request):
+            return None
+        if hints and hints.get("kind", "request") != "request":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, request: urllib.request.Request, **hints: Any) -> None:
+        self._request = request
+        self._timeout = hints.get("timeout")
+        self._compression = hints.get("compression", "auto")
+        validate_compression(self._compression)
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.IOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            if self._timeout is None:
+                resp = urllib.request.urlopen(self._request)
+            else:
+                resp = urllib.request.urlopen(self._request, timeout=self._timeout)
+            raw = cast(io.IOBase, resp)
+            name = urllib.parse.urlsplit(self._request.full_url).path
+            opened = open_compressed(raw, compression=self._compression, name=name)
+            self._underlying = raw if opened is not raw else None
+            self._f = opened
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return None
+
+    @property
+    def url(self) -> str:
+        return self._request.full_url
+
+    @property
+    def request(self) -> urllib.request.Request:
+        return self._request
+
+    @property
+    def closed(self) -> bool:
+        return self._closed

httk/core/datastream/bytestream_request_view.py

@@ -0,0 +1,44 @@
+import urllib.request
+from typing import Any, Self
+
+from ..views import unwrap
+from .bytestream_backend import BytestreamBackend
+from .bytestream_like import BytestreamLike
+from .bytestream_view import BytestreamView
+
+
+class BytestreamRequestView(BytestreamView, urllib.request.Request):
+    """
+    A view presenting an underlying data streaming backend via a urllib.request.Request.
+    This view is mostly useful for providing a Request to functions that will open it.
+    Note: this view is not lazy (it does not fetch); it only mirrors the underlying request/URL.
+
+    Raises TypeError if created with a streaming data source that does not come with a URL.
+    """
+
+    _backend: BytestreamBackend
+
+    def __new__(cls, obj: BytestreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        request = getattr(backend, "request", None)
+        if request is not None:
+            full_url, data, headers = request.full_url, request.data, request.headers
+        else:
+            url = getattr(backend, "url", None)
+            if url is None:
+                raise TypeError("This backend cannot be represented as a Request (no underlying URL)")
+            full_url, data, headers = url, None, {}
+        instance = super().__new__(cls)
+        # Request is mutable, so its state is initialized here in __new__ (keeping __init__ a no-op),
+        # so that rewrapping an existing view via cls(view) does not re-initialize it.
+        urllib.request.Request.__init__(instance, full_url, data=data, headers=headers)
+        instance._backend = backend
+        return instance
+
+    def __init__(self, obj: BytestreamLike, **hints: Any) -> None:
+        pass
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)

httk/core/datastream/bytestream_url.py

@@ -0,0 +1,74 @@
+import io
+import urllib.parse
+import urllib.request
+from typing import Any, cast
+
+from .bytestream_backend import BytestreamBackend
+from .bytestream_common import BytestreamCommon
+from .compression import open_compressed, validate_compression
+
+_URL_SCHEMES = ("http", "https", "ftp", "file")
+
+
+class BytestreamURL(BytestreamCommon, BytestreamBackend):
+    """
+    Backend for streaming byte data fetched from a URL string.
+    A bare string is interpreted as a URL when its scheme is one of http, https, ftp, or file,
+    or when an explicit kind="url" hint is given.
+    """
+
+    _url: str
+    _timeout: float | None
+    _compression: str
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # mypy does not allow to type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, url: str, **hints: Any) -> Any:
+        if not isinstance(url, str):
+            return None
+        kind = hints.get("kind")
+        if kind == "url":
+            if not urllib.parse.urlsplit(url).scheme:
+                return None
+            return super().__new__(cls)
+        if kind is None and urllib.parse.urlsplit(url).scheme in _URL_SCHEMES:
+            return super().__new__(cls)
+        return None
+
+    def __init__(self, url: str, **hints: Any) -> None:
+        self._url = url
+        self._timeout = hints.get("timeout")
+        self._compression = hints.get("compression", "auto")
+        validate_compression(self._compression)
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.IOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            if self._timeout is None:
+                resp = urllib.request.urlopen(self._url)
+            else:
+                resp = urllib.request.urlopen(self._url, timeout=self._timeout)
+            raw = cast(io.IOBase, resp)
+            name = urllib.parse.urlsplit(self._url).path
+            opened = open_compressed(raw, compression=self._compression, name=name)
+            self._underlying = raw if opened is not raw else None
+            self._f = opened
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return None
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def closed(self) -> bool:
+        return self._closed

httk/core/datastream/bytestream_url_view.py

@@ -0,0 +1,35 @@
+from typing import Any, Self
+
+from ..views import unwrap
+from .bytestream_backend import BytestreamBackend
+from .bytestream_like import BytestreamLike
+from .bytestream_view import BytestreamView
+
+
+class BytestreamURLView(BytestreamView, str):
+    """
+    A view presenting an underlying data streaming backend via a URL string.
+    This view is mostly useful for providing a URL to functions that will open it.
+    Note: this view is not lazy (this is impossible for views inheriting str, since str is immutable).
+
+    Raises TypeError if created with a streaming data source that does not come with a URL.
+    """
+
+    _backend: BytestreamBackend
+
+    def __new__(cls, obj: BytestreamLike, **hints: Any) -> Self:
+        if isinstance(obj, cls):
+            return obj
+        backend = cls._prepare_backend(obj, hints)
+        url = getattr(backend, "url", None)
+        if url is None:
+            raise TypeError("This backend cannot be represented as a URL (no underlying URL)")
+        instance = super().__new__(cls, url)
+        instance._backend = backend
+        return instance
+
+    def __init__(self, obj: BytestreamLike, **hints: Any) -> None:
+        super().__init__()
+
+    def unwrap(self) -> Any:
+        return unwrap(self._backend)

httk/core/datastream/bytestream_view.py

@@ -0,0 +1,16 @@
+from typing import ClassVar, Self
+
+from ..views import View
+from .bytestream_backend import BytestreamBackend
+
+
+class BytestreamView(View[BytestreamBackend]):
+    """
+    Abstract base class for all views of streaming byte data.
+    """
+
+    _backend_base_cls: ClassVar[type[BytestreamBackend]] = BytestreamBackend  # type: ignore[type-abstract]
+    _view_base_cls: ClassVar[type[Self]]
+
+
+BytestreamView._view_base_cls = BytestreamView