pyfect-dev 0.1.0__py3-none-any.whl

pyfect/encoding/zip/_writer.py

@@ -0,0 +1,311 @@
+"""The archive bytes ``encode`` writes."""
+
+from __future__ import annotations
+
+import struct
+import zlib
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import timedelta
+
+from pyfect._js import text_encode
+from pyfect._json_text import quote as quoted
+from pyfect.encoding.zip._format import (
+    CENTRAL,
+    DESCRIPTOR,
+    END,
+    END64,
+    EPOCH,
+    EXTRA_TIMESTAMP,
+    EXTRA_ZIP64,
+    FLAG_DESCRIPTOR,
+    FLAG_UTF8,
+    LOCAL,
+    LOCATOR64,
+    MAX16,
+    MAX32,
+    invalid_header,
+    size_mismatch,
+    to_dos,
+)
+from pyfect.encoding.zip._model import Header, ZipError
+
+SLAB_SIZE = 64 * 1024
+
+_VERSION = 20
+_VERSION_ZIP64 = 45
+_CREATOR_UNIX = 3
+_S_IFDIR = 0o040000
+_S_IFREG = 0o100000
+_DOS_DIRECTORY = 0x10
+_MAX_TIMESTAMP = 0x7FFFFFFF
+
+_DEFLATE_HEADROOM = 16 * 1024 * 1024
+"""Room for deflate to grow data that does not compress, when choosing ZIP64."""
+
+_COPY_LIMIT = 16 * 1024
+"""Chunks below this size are copied together instead of being handed out alone."""
+
+
+@dataclass(slots=True)
+class _Record:
+    name: bytes
+    flags: int
+    method: int
+    time: int
+    date: int
+    seconds: int | None
+    external: int
+    offset: int
+    # Whether the local header has ZIP64 sizes, which then also follow the data
+    zip64: bool = False
+    crc: int = 0
+    compressed_size: int = 0
+    size: int = 0
+
+
+def _timestamp(record: _Record) -> bytes:
+    if record.seconds is None:
+        return b""
+    return struct.pack("<HHBI", EXTRA_TIMESTAMP, 5, 1, record.seconds)
+
+
+def _local_header(record: _Record) -> bytes:
+    zip64 = record.zip64
+    extra = _timestamp(record)
+    if zip64:
+        extra = struct.pack("<HHQQ", EXTRA_ZIP64, 16, record.size, record.compressed_size) + extra
+    return (
+        struct.pack(
+            "<IHHHHHIIIHH",
+            LOCAL,
+            _VERSION_ZIP64 if zip64 else _VERSION,
+            record.flags,
+            record.method,
+            record.time,
+            record.date,
+            record.crc,
+            MAX32 if zip64 else record.compressed_size,
+            MAX32 if zip64 else record.size,
+            len(record.name),
+            len(extra),
+        )
+        + record.name
+        + extra
+    )
+
+
+def _data_descriptor(record: _Record) -> bytes:
+    return struct.pack(
+        "<IIQQ" if record.zip64 else "<IIII",
+        DESCRIPTOR,
+        record.crc,
+        record.compressed_size,
+        record.size,
+    )
+
+
+def _central_header(record: _Record) -> bytes:
+    # ZIP64 holds only the fields that overflow, in this order
+    wide = [
+        value for value in (record.size, record.compressed_size, record.offset) if value >= MAX32
+    ]
+    extra = _timestamp(record)
+    if wide:
+        extra = struct.pack(f"<HH{len(wide)}Q", EXTRA_ZIP64, 8 * len(wide), *wide) + extra
+    version = _VERSION if not wide and not record.zip64 else _VERSION_ZIP64
+    return (
+        struct.pack(
+            "<IHHHHHHIIIHHHHHII",
+            CENTRAL,
+            (_CREATOR_UNIX << 8) | version,
+            version,
+            record.flags,
+            record.method,
+            record.time,
+            record.date,
+            record.crc,
+            min(record.compressed_size, MAX32),
+            min(record.size, MAX32),
+            len(record.name),
+            len(extra),
+            0,
+            0,
+            0,
+            record.external,
+            min(record.offset, MAX32),
+        )
+        + record.name
+        + extra
+    )
+
+
+class Writer:
+    __slots__ = ("_chunks", "_current", "_offset", "_records", "buffered")
+
+    def __init__(self) -> None:
+        self._chunks: list[bytes] = []
+        self.buffered = 0
+        self._offset = 0
+        self._records: list[_Record] = []
+        # An entry whose data is being written, its record and whether a data
+        # descriptor follows
+        self._current: tuple[Header, _Record, bool] | None = None
+
+    def flush(self) -> tuple[bytes, ...]:
+        """Hands out the buffered output, joining runs of small chunks; needs
+        ``buffered > 0``."""
+        out: list[bytes] = []
+        run: list[bytes] = []
+        for chunk in self._chunks:
+            if len(chunk) >= _COPY_LIMIT:
+                if run:
+                    out.append(b"".join(run))
+                    run = []
+                out.append(chunk)
+            else:
+                run.append(chunk)
+        if run:
+            out.append(b"".join(run))
+        self._chunks = []
+        self.buffered = 0
+        return tuple(out)
+
+    def _push(self, data: bytes) -> None:
+        if data:
+            self._chunks.append(data)
+            self._offset += len(data)
+            self.buffered += len(data)
+
+    def _record(self, header: Header, flags: int) -> _Record:
+        name = text_encode(header.path)
+        if len(name) > MAX16:
+            raise invalid_header(f"path {quoted(header.path)} is longer than 65535 bytes")
+        time, date = to_dos(header.mtime)
+        seconds = (header.mtime - EPOCH) // timedelta(seconds=1)
+        directory = header.type == "Directory"
+        mode = (0o755 if directory else 0o644) if header.mode is None else header.mode
+        return _Record(
+            name=name,
+            flags=flags if header.path.isascii() else flags | FLAG_UTF8,
+            method=8 if header.type == "File" and header.method == "Deflate" else 0,
+            time=time,
+            date=date,
+            seconds=seconds if 0 <= seconds <= _MAX_TIMESTAMP else None,
+            external=(
+                ((_S_IFDIR if directory else _S_IFREG) | mode) * 0x10000
+                + (_DOS_DIRECTORY if directory else 0)
+            )
+            & MAX32,
+            offset=self._offset,
+        )
+
+    def entry(self, header: Header, content: bytes, data: bytes) -> None:
+        """Writes an entry whose content is known, with its sizes in the local header."""
+        size = 0 if header.type == "Directory" else header.size
+        if size is not None and size != len(content):
+            raise size_mismatch(header.path, len(content), size)
+        record = self._record(header, 0)
+        record.crc = zlib.crc32(content)
+        record.size = len(content)
+        record.compressed_size = len(data)
+        record.zip64 = record.size >= MAX32 or record.compressed_size >= MAX32
+        self._push(_local_header(record))
+        self._push(data)
+        self._records.append(record)
+
+    def begin(self, header: Header) -> None:
+        """Starts an entry whose sizes and CRC-32 follow its data."""
+        directory = header.type == "Directory"
+        stored = directory or header.method == "Store"
+        size = 0 if directory else header.size
+        if stored and size is None:
+            raise invalid_header(f"stored file {quoted(header.path)} is streamed without a size")
+        # An empty stored entry needs no descriptor: readers would take its zero
+        # sizes for missing ones
+        descriptor = not (stored and size == 0)
+        record = self._record(header, FLAG_DESCRIPTOR if descriptor else 0)
+        # Streaming readers find the end of stored data from its size
+        if stored and size is not None:
+            record.size = size
+            record.compressed_size = size
+        # The local header fixes the width of the sizes after the data, so declare
+        # ZIP64 whenever they could reach 4 GiB
+        record.zip64 = size is None or size >= (MAX32 if stored else MAX32 - _DEFLATE_HEADROOM)
+        self._push(_local_header(record))
+        record.size = 0
+        record.compressed_size = 0
+        self._current = (header, record, descriptor)
+
+    def update(self, chunks: Sequence[bytes]) -> None:
+        """Accounts for uncompressed body bytes."""
+        assert self._current is not None
+        record = self._current[1]
+        for chunk in chunks:
+            record.crc = zlib.crc32(chunk, record.crc)
+            record.size += len(chunk)
+
+    def data(self, chunks: Sequence[bytes]) -> None:
+        """Writes stored or compressed body bytes."""
+        assert self._current is not None
+        header, record, _ = self._current
+        for chunk in chunks:
+            self._push(chunk)
+            record.compressed_size += len(chunk)
+        size = 0 if header.type == "Directory" else header.size
+        if size is not None and record.size > size:
+            raise size_mismatch(header.path, record.size, size)
+        if not record.zip64 and record.compressed_size >= MAX32:
+            raise ZipError(
+                "SizeMismatch",
+                f"compressed data of {quoted(header.path)} reached 4 GiB, "
+                "but its local header has no ZIP64 sizes",
+            )
+
+    def end(self) -> None:
+        assert self._current is not None
+        header, record, descriptor = self._current
+        self._current = None
+        if header.size is not None and record.size != header.size:
+            raise size_mismatch(header.path, record.size, header.size)
+        if descriptor:
+            self._push(_data_descriptor(record))
+        self._records.append(record)
+
+    def finish(self) -> None:
+        start = self._offset
+        for record in self._records:
+            self._push(_central_header(record))
+        size = self._offset - start
+        count = len(self._records)
+        if count >= MAX16 or size >= MAX32 or start >= MAX32:
+            end64_offset = self._offset
+            self._push(
+                struct.pack(
+                    "<IQHHIIQQQQ",
+                    END64,
+                    44,
+                    (_CREATOR_UNIX << 8) | _VERSION_ZIP64,
+                    _VERSION_ZIP64,
+                    0,
+                    0,
+                    count,
+                    count,
+                    size,
+                    start,
+                )
+            )
+            self._push(struct.pack("<IIQI", LOCATOR64, 0, end64_offset, 1))
+        self._push(
+            struct.pack(
+                "<IHHHHIIH",
+                END,
+                0,
+                0,
+                min(count, MAX16),
+                min(count, MAX16),
+                min(size, MAX32),
+                min(start, MAX32),
+                0,
+            )
+        )

pyfect/error_reporter.py

@@ -0,0 +1,240 @@
+"""Error reporting: hand failures to reporters such as Sentry or a log
+pipeline (Effect's ``ErrorReporter``).
+
+``eff.with_error_reporting()`` sends the cause of each failure of ``eff``
+to every reporter in ``CurrentErrorReporters``; ``report(cause)`` sends one
+cause. With no reporters (the default) nothing happens. A reporter made
+with ``make`` turns a cause into one ``Report`` per typed error or defect,
+skipping interruptions, causes and errors it has seen already, and errors
+that ask to be ignored.
+
+An error controls its report through attributes (``TaggedError`` declares
+them; any other object may set them):
+
+- ``report_ignored``: ``True`` to skip it.
+- ``report_severity``: the report's ``Severity`` (``INFO`` otherwise).
+- ``report_attributes``: extra key/value metadata.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import weakref
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any, Never, Protocol, cast, overload
+
+from pyfect._core import Effect, _Gen, _Succeed, _WithFiber
+from pyfect._fiber import Fiber
+from pyfect.cause import Cause, Fail, Interrupt
+from pyfect.clock import Clock
+from pyfect.context import Reference
+from pyfect.layer import Layer, effect
+from pyfect.logger import LogLevel, Severity
+from pyfect.scope import Scope
+
+__all__ = [
+    "CurrentErrorReporters",
+    "ErrorReporter",
+    "Report",
+    "get_attributes",
+    "get_severity",
+    "is_ignored",
+    "layer",
+    "make",
+    "report",
+    "report_unsafe",
+]
+
+
+class ErrorReporter(Protocol):
+    """Receives the causes of reported failures. ``timestamp`` is the
+    fiber's ``Clock`` time in nanoseconds."""
+
+    def report(self, *, cause: Cause[object], fiber: Fiber[Any, Any], timestamp: int) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+    """One typed error or defect of a reported cause, as ``make`` hands it
+    to its function. ``error`` is the reason as an exception (a typed error
+    that is not one is wrapped in ``UnhandledError``)."""
+
+    cause: Cause[object]
+    error: BaseException
+    attributes: Mapping[str, object]
+    severity: Severity
+    fiber: Fiber[Any, Any]
+    timestamp: int
+
+
+class CurrentErrorReporters(Reference[tuple[ErrorReporter, ...]]):
+    """The reporters failures are sent to, each once, in order. None by
+    default."""
+
+    @classmethod
+    def default(cls) -> tuple[ErrorReporter, ...]:
+        return ()
+
+
+_NO_ATTRIBUTES: Mapping[str, object] = MappingProxyType({})
+
+
+def is_ignored(error: object) -> bool:
+    """Whether ``error`` asks not to be reported (``report_ignored`` is
+    ``True``)."""
+    return getattr(error, "report_ignored", False) is True
+
+
+def get_severity(error: object) -> Severity:
+    """``error``'s ``report_severity`` when it is a ``LogLevel`` (``ALL`` and
+    ``NONE`` pass through, as in Effect), else ``LogLevel.INFO``."""
+    severity = getattr(error, "report_severity", None)
+    return cast("Severity", severity) if isinstance(severity, LogLevel) else LogLevel.INFO
+
+
+def get_attributes(error: object) -> Mapping[str, object]:
+    """``error``'s ``report_attributes`` mapping, or an empty one."""
+    attributes = getattr(error, "report_attributes", None)
+    return attributes if isinstance(attributes, Mapping) else _NO_ATTRIBUTES  # pyright: ignore[reportUnknownVariableType]
+
+
+class _SeenOnce:
+    """Remembers objects by identity while they live (Effect's ``WeakSet``).
+    An object Python cannot weakly reference but that has a ``__dict__``
+    (a built-in exception such as ``ValueError``) is marked in it instead.
+    Values with neither (``str``, ``int``, ``tuple``, ``dict``...) are never
+    remembered, as Effect never remembers its primitives."""
+
+    __slots__ = ("_mark", "_seen")
+
+    def __init__(self) -> None:
+        self._seen: dict[int, weakref.ref[object]] = {}
+        self._mark = object()
+
+    def first_time(self, value: object) -> bool:
+        key = id(value)
+        known = self._seen.get(key)
+        if known is not None and known() is value:
+            return False
+        try:
+            self._seen[key] = weakref.ref(value, lambda ref: self._forget(key, ref))
+        except TypeError:
+            return self._first_mark(value)
+        return True
+
+    def _first_mark(self, value: object) -> bool:
+        try:
+            marks: list[object] = vars(value).setdefault(_MARKS, [])
+        except TypeError:
+            return True
+        if any(mark is self._mark for mark in marks):
+            return False
+        marks.append(self._mark)
+        return True
+
+    def _forget(self, key: int, ref: weakref.ref[object]) -> None:
+        if self._seen.get(key) is ref:
+            del self._seen[key]
+
+
+_MARKS = "__pyfect_error_reporters__"
+
+
+class _Reporter:
+    __slots__ = ("_report", "_seen")
+
+    def __init__(self, f: Callable[[Report], object]) -> None:
+        self._report = f
+        self._seen = _SeenOnce()
+
+    def report(self, *, cause: Cause[object], fiber: Fiber[Any, Any], timestamp: int) -> None:
+        if not self._seen.first_time(cause):
+            return
+        for reason in cause.reasons:
+            if isinstance(reason, Interrupt):
+                continue
+            original = reason.error if isinstance(reason, Fail) else reason.defect
+            if not self._seen.first_time(original) or is_ignored(original):
+                continue
+            (error,) = Cause((reason,)).pretty_errors()
+            self._report(
+                Report(
+                    cause=cause,
+                    error=error,
+                    attributes=get_attributes(original),
+                    severity=get_severity(original),
+                    fiber=fiber,
+                    timestamp=timestamp,
+                )
+            )
+
+
+def make(f: Callable[[Report], object]) -> ErrorReporter:
+    """A reporter calling ``f`` once per typed error or defect of each
+    cause. Interruptions, ignored errors, and causes or errors this
+    reporter has already seen (by identity) are skipped."""
+    return _Reporter(f)
+
+
+type _Reporters[E, R] = Sequence[ErrorReporter | Effect[ErrorReporter, E, R]]
+
+
+@overload
+def layer[E = Never](
+    reporters: Sequence[ErrorReporter | Effect[ErrorReporter, E, Scope]],
+    *,
+    merge_with_existing: bool = False,
+) -> Layer[Never, E]: ...
+@overload
+def layer[E = Never, R = Never](
+    reporters: Sequence[ErrorReporter | Effect[ErrorReporter, E, Scope | R]],
+    *,
+    merge_with_existing: bool = False,
+) -> Layer[Never, E, R]: ...
+def layer(
+    reporters: _Reporters[Any, Any], *, merge_with_existing: bool = False
+) -> Layer[Any, Any, Any]:
+    """A layer that replaces the current reporters with ``reporters`` (or
+    adds them, with ``merge_with_existing``). An item may be an effect
+    making a reporter; it runs when the layer builds."""
+    given = list(reporters)
+
+    def build():
+        current: list[ErrorReporter] = (
+            list((yield from CurrentErrorReporters())) if merge_with_existing else []
+        )
+        for item in given:
+            made: ErrorReporter = (yield from item) if isinstance(item, Effect) else item
+            if all(made is not existing for existing in current):
+                current.append(made)
+        return tuple(current)
+
+    return effect(CurrentErrorReporters, _Gen(build))
+
+
+_VOID: Effect[None] = _Succeed(None)
+
+
+def report_unsafe(
+    fiber: Fiber[Any, Any], cause: Cause[object], *, defects_only: bool = False
+) -> None:
+    """Send ``cause`` to ``fiber``'s reporters now (with ``defects_only``,
+    only if it has a defect)."""
+    reporters = fiber.get_ref(CurrentErrorReporters)
+    if not reporters or (defects_only and not cause.defects):
+        return
+    timestamp = fiber.get_ref(Clock).now_nanos_unsafe()
+    for reporter in reporters:
+        reporter.report(cause=cause, fiber=fiber, timestamp=timestamp)
+
+
+def report(cause: Cause[object]) -> Effect[None]:
+    """Send ``cause`` to the current reporters."""
+
+    def body(fiber: Fiber[Any, Any]) -> Effect[None]:
+        report_unsafe(fiber, cause)
+        return _VOID
+
+    return _WithFiber(body)