pyfect-dev 0.1.0__py3-none-any.whl

pyfect/encoding/tar/__init__.py

@@ -0,0 +1,396 @@
+"""Tar archives as streams (Effect's ``encoding/Tar``).
+
+``decode`` streams entry bodies from the archive input and ``encode`` streams them
+back into archive bytes; neither holds a whole body, which is what these channels
+add to the blocking ``tarfile``. The decoder reads v7, USTAR, PAX extended headers,
+and GNU long names and base-256 numbers. The encoder writes USTAR, adding PAX
+extended headers for fields that do not fit or for custom metadata. Neither touches
+the file system or compresses.
+
+Paths and link targets are returned as written in the archive, so they can be
+absolute or contain ``..``, and a symlink entry can redirect a later entry outside
+the target directory. Validate both before writing entries to disk.
+"""
+# pyright: reportPrivateUsage=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnknownMemberType=false
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any, Literal, Never, TypeIs
+
+from pyfect._js import text_encode
+from pyfect._primitives import _Succeed, _Suspend
+from pyfect._public import publish
+from pyfect.channel import Channel, from_transform
+from pyfect.channel._base import AnyPull, end_with
+from pyfect.encoding._archive import decode_channel
+from pyfect.encoding.tar._decoder import Decoder
+from pyfect.encoding.tar._encoder import SLAB_SIZE, Encoder
+from pyfect.encoding.tar._header import EPOCH, TYPEFLAGS
+from pyfect.encoding.tar._model import Entry, EntryType, FarDate, Header, TarError, TarErrorKind
+from pyfect.exit import Success, void
+from pyfect.exit import fail as fail_exit
+from pyfect.pull import catch_done
+from pyfect.scope import CloseableScope
+from pyfect.stream import Stream, empty, succeed
+from pyfect.stream._base import require_stream
+
+__all__ = [
+    "Entry",
+    "EntryType",
+    "FarDate",
+    "Header",
+    "TarError",
+    "TarErrorKind",
+    "decode",
+    "directory",
+    "encode",
+    "file",
+    "file_stream",
+    "hardlink",
+    "is_tar_error",
+    "symlink",
+]
+
+
+def is_tar_error(value: object) -> TypeIs[TarError]:
+    return isinstance(value, TarError)
+
+
+class _Content(Stream[bytes]):
+    """A body held in memory, which ``encode`` writes without running it. It is the
+    ``body`` that says so, so replacing an entry's body leaves that path."""
+
+    __slots__ = ("content",)
+
+    def __init__(self, content: bytes) -> None:
+        super().__init__(succeed(content)._channel)
+        self.content = content
+
+
+def _header(
+    path: str,
+    kind: Literal["File", "Directory", "Symlink", "Hardlink"],
+    size: int,
+    link_target: str | None,
+    *,
+    mode: int | None,
+    uid: int | None,
+    gid: int | None,
+    uname: str | None,
+    gname: str | None,
+    mtime: datetime | FarDate | None,
+    pax: Mapping[str, str] | None,
+) -> Header:
+    default_mode = 0o755 if kind == "Directory" else 0o777 if kind == "Symlink" else 0o644
+    return Header(
+        path=path,
+        type=kind,
+        typeflag=TYPEFLAGS[kind],
+        size=size,
+        mode=default_mode if mode is None else mode,
+        uid=0 if uid is None else uid,
+        gid=0 if gid is None else gid,
+        uname="" if uname is None else uname,
+        gname="" if gname is None else gname,
+        mtime=EPOCH if mtime is None else mtime,
+        link_target=link_target or None,
+        pax={} if pax is None else pax,
+    )
+
+
+def file(
+    path: str,
+    content: str | bytes,
+    *,
+    mode: int | None = None,
+    uid: int | None = None,
+    gid: int | None = None,
+    uname: str | None = None,
+    gname: str | None = None,
+    mtime: datetime | FarDate | None = None,
+    pax: Mapping[str, str] | None = None,
+) -> Entry:
+    """A regular file from in-memory content; a ``str`` is encoded as UTF-8. Unset
+    fields default to mode ``0o644``, uid and gid 0, empty names, an mtime at the
+    epoch and no extra PAX records, which makes archives reproducible."""
+    data = text_encode(content) if isinstance(content, str) else content
+    header = _header(
+        path,
+        "File",
+        len(data),
+        None,
+        mode=mode,
+        uid=uid,
+        gid=gid,
+        uname=uname,
+        gname=gname,
+        mtime=mtime,
+        pax=pax,
+    )
+    return Entry(header, _Content(data) if data else empty)
+
+
+def file_stream[E = Never, R = Never](
+    path: str,
+    size: int,
+    body: Stream[bytes, E, R],
+    *,
+    mode: int | None = None,
+    uid: int | None = None,
+    gid: int | None = None,
+    uname: str | None = None,
+    gname: str | None = None,
+    mtime: datetime | FarDate | None = None,
+    pax: Mapping[str, str] | None = None,
+) -> Entry[E, R]:
+    """A regular file streamed from ``body``, which runs when the entry is encoded.
+    ``size`` is its exact byte length: ``encode`` fails with ``"SizeMismatch"``
+    otherwise."""
+    require_stream(body, "body")
+    header = _header(
+        path,
+        "File",
+        size,
+        None,
+        mode=mode,
+        uid=uid,
+        gid=gid,
+        uname=uname,
+        gname=gname,
+        mtime=mtime,
+        pax=pax,
+    )
+    return Entry(header, body)
+
+
+def directory(
+    path: str,
+    *,
+    mode: int | None = None,
+    uid: int | None = None,
+    gid: int | None = None,
+    uname: str | None = None,
+    gname: str | None = None,
+    mtime: datetime | FarDate | None = None,
+    pax: Mapping[str, str] | None = None,
+) -> Entry:
+    """A directory (mode ``0o755`` by default). A trailing ``/`` is added to a
+    non-empty ``path``; an empty one stays empty, so that ``encode`` rejects it."""
+    named = path if path == "" or path.endswith("/") else f"{path}/"
+    header = _header(
+        named,
+        "Directory",
+        0,
+        None,
+        mode=mode,
+        uid=uid,
+        gid=gid,
+        uname=uname,
+        gname=gname,
+        mtime=mtime,
+        pax=pax,
+    )
+    return Entry(header, empty)
+
+
+def symlink(
+    path: str,
+    target: str,
+    *,
+    mode: int | None = None,
+    uid: int | None = None,
+    gid: int | None = None,
+    uname: str | None = None,
+    gname: str | None = None,
+    mtime: datetime | FarDate | None = None,
+    pax: Mapping[str, str] | None = None,
+) -> Entry:
+    """A symbolic link to ``target`` (mode ``0o777`` by default)."""
+    header = _header(
+        path,
+        "Symlink",
+        0,
+        target,
+        mode=mode,
+        uid=uid,
+        gid=gid,
+        uname=uname,
+        gname=gname,
+        mtime=mtime,
+        pax=pax,
+    )
+    return Entry(header, empty)
+
+
+def hardlink(
+    path: str,
+    target: str,
+    *,
+    mode: int | None = None,
+    uid: int | None = None,
+    gid: int | None = None,
+    uname: str | None = None,
+    gname: str | None = None,
+    mtime: datetime | FarDate | None = None,
+    pax: Mapping[str, str] | None = None,
+) -> Entry:
+    """A hard link to ``target``, the archive path of an entry written earlier."""
+    header = _header(
+        path,
+        "Hardlink",
+        0,
+        target,
+        mode=mode,
+        uid=uid,
+        gid=gid,
+        uname=uname,
+        gname=gname,
+        mtime=mtime,
+        pax=pax,
+    )
+    return Entry(header, empty)
+
+
+def decode[IE = Never, D = None]() -> Channel[
+    tuple[Entry[TarError | IE], ...], TarError | IE, D, tuple[bytes, ...], IE, D
+]:
+    """A channel decoding archive bytes into entries, one entry per output chunk,
+    each body streaming ``header.size`` bytes straight from the input. Pulling the
+    next entry skips what is left of the current body. A PAX global header's records
+    apply to every entry after it, except ``path``, ``linkpath`` and ``size``.
+
+    Read each body before pulling the next entry and while the entry stream runs,
+    for example in ``map_effect``: a later read fails with ``"BodyConsumed"``, so
+    operators that pull ahead cannot sit between ``decode`` and the reads. A body
+    reads its bytes once: running it again continues where it stopped.
+
+    Directories, links, devices and FIFOs have no body and a ``size`` of 0. Names
+    that are not UTF-8 get replacement characters. PAX headers and GNU long names
+    past 1 MiB fail with ``"InvalidHeader"``, and GNU sparse entries with
+    ``"Unsupported"``. The channel ends only when its input ends: input after the
+    end-of-archive marker is read and discarded, so that the upstream's done value
+    can be passed on.
+
+    An mtime ``datetime`` cannot hold (outside years 1 to 9999) fails with
+    ``"InvalidHeader"``."""
+    return decode_channel(Decoder, Entry, TarError)
+
+
+def encode[E = Never, R = Never, IE = Never, D = None]() -> Channel[  # noqa: PLR0915
+    tuple[bytes, ...], TarError | IE | E, D, tuple[Entry[E, R], ...], IE, D, R
+]:
+    """A channel encoding entries into archive bytes: a USTAR header per entry, a
+    PAX header only for what USTAR cannot hold (a long or non-ASCII path, link target
+    or name, a size of 8 GiB or more, a uid or gid above ``2097151``, an mtime outside
+    USTAR's range, or custom ``pax`` records), and two zero blocks at the end.
+
+    Output is written in chunks of about 64 KiB, and large body chunks pass through
+    uncopied, so bodies stream without being held in memory.
+
+    A body with any byte count but ``header.size`` fails with ``"SizeMismatch"``.
+    ``header.pax`` holds custom metadata: the keys of header fields and
+    ``GNU.sparse.*`` fail with ``"InvalidHeader"``, as does a ``File`` path ending in
+    ``/``, a non-zero size on an entry other than ``File`` and ``Other``, or a device
+    number above ``2097151``. ``mtime`` is rounded down to whole seconds, and a naive
+    ``datetime`` fails with ``"InvalidHeader"``. ``typeflag`` is only written for
+    ``Other`` entries.
+
+    Entries that differ in their bodies' errors or services do not infer one ``E`` and
+    ``R``: annotate the stream, as ``Stream[Entry[E1 | E2, R1 | R2]]``."""
+
+    def transform(upstream: AnyPull, scope: CloseableScope) -> AnyPull:  # noqa: PLR0915
+        encoder = Encoder()
+        entries: tuple[Entry[Any, Any], ...] = ()
+        index = 0
+        done: tuple[Any] | None = None
+        body: AnyPull | None = None
+        body_scope: CloseableScope | None = None
+
+        def on_body(chunks: tuple[bytes, ...] | None) -> Any:
+            nonlocal body, body_scope
+            try:
+                if chunks is not None:
+                    for chunk in chunks:
+                        encoder.body(chunk)
+                    return pull
+            except TarError as error:
+                return fail_exit(error)
+            body = None
+            assert body_scope is not None
+            closing = body_scope.close(void)
+            body_scope = None
+            return closing.flat_map(on_body_closed)
+
+        def on_body_closed(_: None) -> Any:
+            try:
+                encoder.end()
+            except TarError as error:
+                return fail_exit(error)
+            return pull
+
+        def begin(entry: Entry[Any, Any]) -> Any:
+            nonlocal body, body_scope
+            # Directories, links and in-memory files have nothing to stream
+            source = entry.body
+            data = (
+                b"" if source is empty else source.content if isinstance(source, _Content) else None
+            )
+            try:
+                encoder.begin(entry.header)
+                if data is not None:
+                    encoder.body(data)
+                    encoder.end()
+            except TarError as error:
+                return fail_exit(error)
+            if data is not None:
+                return pull
+            body_scope = scope.fork_unsafe()
+
+            def opened(opened_pull: AnyPull) -> Any:
+                nonlocal body
+                body = opened_pull
+                return pull
+
+            return source._channel.to_pull_scoped(body_scope).flat_map(opened)
+
+        def upstream_done(value: object) -> AnyPull:
+            nonlocal done
+            done = (value,)
+            encoder.finish()
+            return _Succeed(None)
+
+        def on_upstream(next_entries: tuple[Entry[Any, Any], ...] | None) -> Any:
+            nonlocal entries, index
+            if next_entries is not None:
+                entries = next_entries
+                index = 0
+            return pull
+
+        def step() -> Any:
+            nonlocal index
+            if encoder.buffered >= SLAB_SIZE:
+                return Success(encoder.flush())
+            if body is not None:
+                return catch_done(body, lambda _: _Succeed(None)).flat_map(on_body)
+            if index < len(entries):
+                entry = entries[index]
+                index += 1
+                return begin(entry)
+            if encoder.buffered > 0:
+                return Success(encoder.flush())
+            if done is not None:
+                return end_with(done[0])
+            return catch_done(upstream, upstream_done).flat_map(on_upstream)
+
+        pull: AnyPull = _Suspend(step)
+        return _Succeed(pull)
+
+    channel: Any = from_transform(
+        lambda upstream, scope: _Suspend(lambda: transform(upstream, scope))
+    )
+    return channel
+
+
+publish(globals())

pyfect/encoding/tar/_decoder.py

@@ -0,0 +1,220 @@
+"""Tar archive bytes to entries, pulled."""
+
+from __future__ import annotations
+
+from pyfect.encoding._archive import NEED_INPUT, ByteQueue, NeedInput
+from pyfect.encoding.tar._header import (
+    BLOCK,
+    PERMISSION_BITS,
+    RawHeader,
+    apply_pax,
+    entry_type,
+    invalid_header,
+    is_header_only,
+    padding,
+    parse_header,
+    parse_pax,
+    read_string,
+    to_mtime,
+    unsupported,
+)
+from pyfect.encoding.tar._model import Header, TarError
+
+_MAX_META_SIZE = 1024 * 1024  # PAX and GNU long name data
+_ZERO_BLOCK = bytes(BLOCK)
+
+
+class Decoder:
+    """``next`` skips unread body bytes; ``read`` consumes the current body. Both
+    keep their progress across ``NEED_INPUT``."""
+
+    def __init__(self) -> None:
+        self._queue = ByteQueue()
+        self._ended = False
+        self._finished = False
+        self._closed = False
+        # Incremented whenever the current body is invalidated
+        self.generation = 0
+        self._remaining = 0
+        self._pad = 0
+        # Bytes to discard before the next header
+        self._skip = 0
+        self._zero_block = False
+        # Archive offset of the last header block read, for error messages
+        self._at = 0
+        # An extended header whose data is still to be read
+        self._meta: RawHeader | None = None
+        self._pax: dict[str, str] | None = None
+        # Records of the PAX global headers read so far
+        self._global: dict[str, str] | None = None
+        self._long_name = ""
+        self._long_link = ""
+
+    def push(self, chunk: bytes) -> None:
+        if not self._finished:
+            self._queue.push(chunk)
+
+    def end(self) -> None:
+        self._ended = True
+
+    def close(self) -> None:
+        self._closed = True
+        self.generation += 1
+        self._queue.clear()
+
+    def next(self) -> Header | NeedInput | None:
+        """The next entry header, or ``None`` once the archive and the input have
+        ended. Input after the end of the archive is discarded."""
+        try:
+            return self._advance()
+        except TarError as error:
+            # Header errors are raised where the archive offset is not known
+            if error.kind in ("InvalidHeader", "Unsupported"):
+                raise TarError(error.kind, f"{error.message} (header at byte {self._at})") from None
+            raise
+
+    def _advance(self) -> Header | NeedInput | None:  # noqa: PLR0911, PLR0912
+        self.generation += 1
+        self._skip += self._remaining + self._pad
+        self._remaining = 0
+        self._pad = 0
+        queue = self._queue
+        while True:
+            if self._finished:
+                return self._finish()
+            while self._skip > 0:
+                if queue.length == 0:
+                    return self._need_input()
+                self._skip -= len(queue.take_some(self._skip))
+            if self._meta is not None:
+                size = self._meta.size
+                typeflag = self._meta.typeflag
+                total = size + padding(size)
+                if queue.length < total:
+                    return self._need_input()
+                data = queue.take(total)[:size]
+                self._meta = None
+                if typeflag == "x":
+                    self._pax = parse_pax(data)
+                elif typeflag == "g":
+                    self._add_global(parse_pax(data))
+                elif typeflag == "L":
+                    self._long_name = read_string(data, 0, size)
+                else:
+                    self._long_link = read_string(data, 0, size)
+                continue
+            if queue.length < BLOCK:
+                if queue.length == 0 and self._ended:
+                    return self._finish()
+                return self._need_input()
+            self._at = queue.consumed
+            block = queue.take(BLOCK)
+            if block == _ZERO_BLOCK:
+                if self._zero_block:
+                    return self._finish()
+                self._zero_block = True
+                continue
+            if self._zero_block:
+                raise invalid_header("non-zero block after an end-of-archive block")
+            raw = parse_header(block)
+            if raw.typeflag in ("x", "g", "L", "K"):
+                if raw.size > _MAX_META_SIZE:
+                    raise invalid_header(f"extended header of {raw.size} bytes exceeds the limit")
+                self._meta = raw
+                continue
+            if raw.typeflag == "S":
+                raise unsupported("GNU sparse files are not supported")
+            header = self._to_header(raw)
+            self._remaining = header.size
+            self._pad = padding(header.size)
+            return header
+
+    def read(self, generation: int) -> tuple[bytes, ...] | NeedInput | None:
+        """The buffered body bytes of the entry from ``generation``, or ``None``
+        once the whole body has been read."""
+        if generation != self.generation:
+            raise TarError(
+                "BodyConsumed",
+                "tar entry body read after the archive stream ended"
+                if self._closed
+                else "tar entry body read after moving to the next entry",
+            )
+        if self._remaining == 0:
+            return None
+        if self._queue.length == 0:
+            return self._need_input()
+        chunks: list[bytes] = []
+        while self._remaining > 0 and self._queue.length > 0:
+            chunk = self._queue.take_some(self._remaining)
+            self._remaining -= len(chunk)
+            chunks.append(chunk)
+        return tuple(chunks)
+
+    def _need_input(self) -> NeedInput:
+        if self._ended:
+            raise _truncated(self._queue.consumed + self._queue.length)
+        return NEED_INPUT
+
+    def _finish(self) -> NeedInput | None:
+        # An extended header must be followed by the entry it describes
+        if not self._finished and (
+            self._pax is not None or self._long_name != "" or self._long_link != ""
+        ):
+            raise _truncated(self._queue.consumed)
+        self._finished = True
+        self._queue.clear()
+        return None if self._ended else NEED_INPUT
+
+    def _add_global(self, records: dict[str, str]) -> None:
+        """Global records apply to every entry that follows, except the ones that
+        only describe a single entry, and an empty value removes a record."""
+        for key, value in records.items():
+            if key in ("path", "linkpath", "size"):
+                continue
+            if value != "":
+                if self._global is None:
+                    self._global = {}
+                self._global[key] = value
+            elif self._global is not None:
+                self._global.pop(key, None)
+        if not self._global:
+            self._global = None
+
+    def _to_header(self, raw: RawHeader) -> Header:
+        # The records of the entry's own extended header override global ones
+        if self._global is None:
+            records = self._pax
+        else:
+            records = self._global if self._pax is None else {**self._global, **self._pax}
+        rest = {} if records is None else apply_pax(raw, records)
+        # Empty GNU long names are ignored, as in Go
+        if self._long_name != "":
+            raw.path = self._long_name
+        if self._long_link != "":
+            raw.link_target = self._long_link
+        self._pax = None
+        self._long_name = ""
+        self._long_link = ""
+        mtime = to_mtime(raw.mtime_millis)
+        kind = entry_type(raw.typeflag, raw.path)
+        device = kind in ("CharDevice", "BlockDevice")
+        return Header(
+            path=raw.path,
+            type=kind,
+            typeflag=raw.typeflag,
+            size=0 if is_header_only(kind) else raw.size,
+            mode=raw.mode & PERMISSION_BITS,
+            uid=raw.uid,
+            gid=raw.gid,
+            uname=raw.uname,
+            gname=raw.gname,
+            mtime=mtime,
+            link_target=None if raw.link_target == "" else raw.link_target,
+            dev_major=raw.dev_major if device else None,
+            dev_minor=raw.dev_minor if device else None,
+            pax=rest,
+        )
+
+
+def _truncated(at: int) -> TarError:
+    return TarError("Truncated", f"unexpected end of tar archive at byte {at}")

pyfect/encoding/tar/_encoder.py

@@ -0,0 +1,83 @@
+"""Entries to tar archive bytes."""
+
+from __future__ import annotations
+
+from pyfect._json_text import quote as quoted
+from pyfect.encoding.tar._header import BLOCK, encode_header, padding
+from pyfect.encoding.tar._model import Header, TarError
+
+SLAB_SIZE = 64 * 1024
+
+_COPY_LIMIT = 16 * 1024
+"""Bodies below this size are copied next to their header instead of being handed
+out as chunks of their own."""
+
+
+def _size_mismatch(header: Header, written: int) -> TarError:
+    return TarError(
+        "SizeMismatch",
+        f"body of {quoted(header.path)} has {'at least' if written > header.size else 'only'} "
+        f"{written} bytes, but its size is {header.size}",
+    )
+
+
+class Encoder:
+    """Packs headers, padding and small body chunks into runs of up to 64 KiB,
+    and passes large chunks through without copying."""
+
+    __slots__ = ("_chunks", "_header", "_run", "_written", "buffered")
+
+    def __init__(self) -> None:
+        self._run = bytearray()
+        self._chunks: list[bytes] = []
+        self.buffered = 0
+        # The entry whose body is being written, and the body bytes written so far
+        self._header: Header | None = None
+        self._written = 0
+
+    def begin(self, header: Header) -> None:
+        self._add(encode_header(header))
+        self._header = header
+        self._written = 0
+
+    def body(self, chunk: bytes) -> None:
+        header = self._header
+        assert header is not None
+        self._written += len(chunk)
+        if self._written > header.size:
+            raise _size_mismatch(header, self._written)
+        if len(chunk) < _COPY_LIMIT:
+            self._add(chunk)
+            return
+        self._cut()
+        self._chunks.append(chunk)
+        self.buffered += len(chunk)
+
+    def end(self) -> None:
+        header = self._header
+        assert header is not None
+        self._header = None
+        if self._written != header.size:
+            raise _size_mismatch(header, self._written)
+        self._add(bytes(padding(header.size)))
+
+    def finish(self) -> None:
+        self._add(bytes(BLOCK * 2))
+
+    def flush(self) -> tuple[bytes, ...]:
+        self._cut()
+        chunks = tuple(self._chunks)
+        self._chunks = []
+        self.buffered = 0
+        return chunks
+
+    def _cut(self) -> None:
+        if self._run:
+            self._chunks.append(bytes(self._run))
+            self._run.clear()
+
+    def _add(self, data: bytes) -> None:
+        if len(self._run) + len(data) > SLAB_SIZE:
+            self._cut()
+        self._run += data
+        self.buffered += len(data)