pi-durable-python 1.3.0__py3-none-any.whl

pi_durable/env/__init__.py

@@ -0,0 +1,408 @@
+"""Portable execution-environment capability.
+
+This module publishes the ``FileSystem`` and ``Shell`` capabilities, their result and error
+shapes, and the decoding, line-scanning, watching, and listing helpers. The POSIX backend lives in
+:mod:`pi_durable.env.posix`; the JSONL backend depends only on the filesystem capability.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import (
+    Any,
+    ClassVar,
+    Generic,
+    Literal,
+    Protocol,
+    TypeAlias,
+    TypeVar,
+    runtime_checkable,
+)
+
+from ..context import Context
+from .decode import StreamDecoder, range_decoder, starts_with_bom
+from .line_scan import LineScan, LineScanner
+
+TValue = TypeVar("TValue")
+TError = TypeVar("TError")
+
+
+@dataclass(frozen=True, slots=True)
+class Ok(Generic[TValue]):
+    """A successful result."""
+
+    value: TValue
+    ok: ClassVar[Literal[True]] = True
+
+
+@dataclass(frozen=True, slots=True)
+class Err(Generic[TError]):
+    """A failed result carrying the expected error."""
+
+    error: TError
+    ok: ClassVar[Literal[False]] = False
+
+
+Result: TypeAlias = Ok[TValue] | Err[TError]
+"""The outcome of a fallible operation; expected failures are returned instead of thrown."""
+
+
+def ok(value: TValue) -> Result[TValue, TError]:
+    return Ok(value)
+
+
+def err(error: TError) -> Result[TValue, TError]:
+    return Err(error)
+
+
+def get_or_throw(result: Result[TValue, Any]) -> TValue:
+    if isinstance(result, Err):
+        error = result.error
+        if isinstance(error, BaseException):
+            raise error
+        raise RuntimeError(to_error(error))
+    return result.value
+
+
+def get_or_undefined(result: Result[TValue, Any]) -> TValue | None:
+    return None if isinstance(result, Err) else result.value
+
+
+def to_error(error: object) -> Exception:
+    """Normalize any thrown value to an exception without throwing while formatting it."""
+
+    if isinstance(error, Exception):
+        return error
+    if isinstance(error, str):
+        return RuntimeError(error)
+    try:
+        return RuntimeError(json.dumps(error))
+    except Exception:
+        return RuntimeError(str(error))
+
+
+FileKind: TypeAlias = Literal["file", "directory", "symlink"]
+
+FileErrorCode: TypeAlias = Literal[
+    "aborted",
+    "not_found",
+    "permission_denied",
+    "not_directory",
+    "is_directory",
+    "invalid",
+    "not_supported",
+    "unknown",
+]
+
+
+class FileError(Exception):
+    """A filesystem operation failed. ``code`` classifies the expected failure."""
+
+    def __init__(
+        self,
+        code: FileErrorCode,
+        message: str,
+        path: str | None = None,
+        cause: BaseException | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.name = "FileError"
+        self.code = code
+        self.path = path
+        if cause is not None:
+            self.__cause__ = cause
+
+
+ExecutionErrorCode: TypeAlias = Literal[
+    "aborted",
+    "timeout",
+    "shell_unavailable",
+    "spawn_error",
+    "callback_error",
+    "unknown",
+]
+
+
+class ExecutionError(Exception):
+    """A shell execution failed. ``code`` classifies the expected failure."""
+
+    def __init__(
+        self,
+        code: ExecutionErrorCode,
+        message: str,
+        cause: BaseException | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.name = "ExecutionError"
+        self.code = code
+        #: Spill file of a command that timed out or was aborted after its output crossed the thresholds.
+        self.spill_path: str | None = None
+        if cause is not None:
+            self.__cause__ = cause
+
+
+class FileInfo(dict[str, Any]):
+    """Metadata of one file, directory, or symbolic link.
+
+    A plain ``dict`` subclass exposing the upstream fields: ``name``, ``path``, ``kind``,
+    ``size``, and ``mtimeMs``.
+    """
+
+
+class TextLine(dict[str, Any]):
+    """One line of text and whether it ended with a terminator."""
+
+
+class TextLineReader(Protocol):
+    async def read_line(self, context: Context) -> Result[TextLine | None, FileError]: ...
+
+    async def close(self, context: Context) -> None: ...
+
+
+class BinaryReader(Protocol):
+    """Positional reads from one opened regular file; all calls see the same file if renamed."""
+
+    async def info(self, context: Context) -> Result[FileInfo, FileError]: ...
+
+    async def read(self, offset: int, length: int, context: Context) -> Result[bytes, FileError]: ...
+
+    async def scan_lines(self, options: dict[str, Any], context: Context) -> Result[LineScan, FileError]: ...
+
+    async def close(self, context: Context) -> None: ...
+
+
+class WatchTarget(dict[str, Any]):
+    """A file or directory to watch."""
+
+
+class WatchPaths(dict[str, Any]):
+    paths: list[str]
+
+
+class WatchOverflow(dict[str, Any]):
+    overflow: Literal[True]
+
+
+class WatchError(dict[str, Any]):
+    error: FileError
+
+
+WatchChange: TypeAlias = WatchPaths | WatchOverflow | WatchError
+
+
+class FileWatcher(Protocol):
+    mode: Literal["native", "polling"]
+
+    async def close(self, context: Context) -> None: ...
+
+
+class DirPage(dict[str, Any]):
+    entries: list[FileInfo]
+    done: bool
+
+
+class DirReader(Protocol):
+    async def next(self, max_entries: int, context: Context) -> Result[DirPage, FileError]: ...
+
+    async def close(self, context: Context) -> None: ...
+
+
+class ListFilesOptions(dict[str, Any]):
+    """Options for the optional ``FileSystem.list_files``.
+
+    The POSIX backend reads the ``.gitignore`` chain and ``.git/info/exclude``; global Git
+    excludes are not read.
+    """
+
+
+class FileListEntry(dict[str, Any]):
+    """One entry of a :class:`FileListing`."""
+
+
+class FileListing(dict[str, Any]):
+    """Files, directories, and symbolic links below a directory."""
+
+
+class ShellSpillOptions(dict[str, Any]):
+    """Spill the complete output to a temporary file once it exceeds either threshold."""
+
+
+class ShellExecResult(dict[str, Any]):
+    """The exit code and, when it spilled, the file holding the complete raw output."""
+
+
+class ShellExecOptions(dict[str, Any]):
+    """Options for one shell command."""
+
+
+class ShellOutputWindow(dict[str, Any]):
+    """The tail of the combined output a caller keeps, and how often it samples it."""
+
+
+@dataclass(frozen=True, slots=True)
+class ShellOutputSkip:
+    """Output omitted immediately before a chunk."""
+
+    bytes: int
+    newlines: int
+    ends_with_newline: bool
+
+
+@dataclass(frozen=True, slots=True)
+class ShellOutputInfo:
+    """Which stream a chunk came from, and what was omitted immediately before it."""
+
+    stream: Literal["stdout", "stderr"]
+    skipped: ShellOutputSkip | None = None
+
+
+class Shell(Protocol):
+    """Run commands through the environment's shell or as an argv array."""
+
+    async def exec(
+        self,
+        command: str | Sequence[str],
+        options: ShellExecOptions | None,
+        context: Context,
+    ) -> Result[ShellExecResult, ExecutionError]: ...
+
+    async def cleanup(self, context: Context) -> None: ...
+
+
+@runtime_checkable
+class FileSystem(Protocol):
+    """Portable filesystem capability. Operations return failures rather than throwing."""
+
+    id: str
+    cwd: str
+
+    async def absolute_path(self, path: str, context: Context) -> Result[str, FileError]: ...
+
+    async def join_path(self, parts: Sequence[str], context: Context) -> Result[str, FileError]: ...
+
+    async def read_text_file(self, path: str, context: Context) -> Result[str, FileError]: ...
+
+    async def open_text_line_reader(
+        self, path: str, context: Context
+    ) -> Result[TextLineReader, FileError]: ...
+
+    async def read_text_lines(
+        self,
+        path: str,
+        options: dict[str, Any] | None,
+        context: Context,
+    ) -> Result[list[str], FileError]: ...
+
+    async def read_binary_file(self, path: str, context: Context) -> Result[bytes, FileError]: ...
+
+    async def open_binary_reader(
+        self, path: str, options: dict[str, Any] | None, context: Context
+    ) -> Result[BinaryReader, FileError]: ...
+
+    async def write_file(
+        self, path: str, content: str | bytes, context: Context
+    ) -> Result[None, FileError]: ...
+
+    async def append_file(
+        self, path: str, content: str | bytes, context: Context
+    ) -> Result[None, FileError]: ...
+
+    async def truncate_file(self, path: str, size: int, context: Context) -> Result[None, FileError]: ...
+
+    async def flush_file(self, path: str, context: Context) -> Result[None, FileError]: ...
+
+    async def rename_file(
+        self, source_path: str, destination_path: str, context: Context
+    ) -> Result[None, FileError]: ...
+
+    async def file_info(self, path: str, context: Context) -> Result[FileInfo, FileError]: ...
+
+    async def list_dir(self, path: str, context: Context) -> Result[list[FileInfo], FileError]: ...
+
+    async def list_files(
+        self,
+        path: str,
+        options: ListFilesOptions | None,
+        context: Context,
+    ) -> Result[FileListing, FileError]: ...
+
+    async def open_dir_reader(self, path: str, context: Context) -> Result[DirReader, FileError]: ...
+
+    async def watch(
+        self,
+        targets: Sequence[WatchTarget],
+        on_change: Callable[[WatchChange], None],
+        context: Context,
+    ) -> Result[FileWatcher, FileError]: ...
+
+    async def canonical_path(self, path: str, context: Context) -> Result[str, FileError]: ...
+
+    async def exists(self, path: str, context: Context) -> Result[bool, FileError]: ...
+
+    async def create_dir(
+        self, path: str, options: dict[str, Any] | None, context: Context
+    ) -> Result[None, FileError]: ...
+
+    async def remove(
+        self, path: str, options: dict[str, Any] | None, context: Context
+    ) -> Result[None, FileError]: ...
+
+    async def create_temp_dir(self, prefix: str | None, context: Context) -> Result[str, FileError]: ...
+
+    async def create_temp_file(
+        self, options: dict[str, Any] | None, context: Context
+    ) -> Result[str, FileError]: ...
+
+    async def cleanup(self, context: Context) -> None: ...
+
+
+class ExecutionEnv(FileSystem, Shell, Protocol):
+    """A portable filesystem plus shell execution."""
+
+
+__all__ = [
+    "BinaryReader",
+    "DirPage",
+    "DirReader",
+    "Err",
+    "ExecutionEnv",
+    "ExecutionError",
+    "ExecutionErrorCode",
+    "FileError",
+    "FileErrorCode",
+    "FileInfo",
+    "FileKind",
+    "FileListEntry",
+    "FileListing",
+    "FileSystem",
+    "FileWatcher",
+    "LineScan",
+    "LineScanner",
+    "ListFilesOptions",
+    "Ok",
+    "Result",
+    "Shell",
+    "ShellExecOptions",
+    "ShellExecResult",
+    "ShellOutputInfo",
+    "ShellOutputSkip",
+    "ShellOutputWindow",
+    "ShellSpillOptions",
+    "StreamDecoder",
+    "TextLine",
+    "TextLineReader",
+    "WatchChange",
+    "WatchError",
+    "WatchOverflow",
+    "WatchPaths",
+    "WatchTarget",
+    "err",
+    "get_or_throw",
+    "get_or_undefined",
+    "ok",
+    "range_decoder",
+    "starts_with_bom",
+    "to_error",
+]

pi_durable/env/decode.py

@@ -0,0 +1,48 @@
+"""Streaming UTF-8 decoding that matches decoding a whole byte range at once.
+
+Python's incremental ``utf-8`` decoder keeps a byte-order mark, so the port mirrors the upstream
+WHATWG ``TextDecoder`` configuration: the range decoder never strips a mark, and
+:class:`StreamDecoder` drops one only when it starts the stream.
+"""
+
+from __future__ import annotations
+
+import codecs
+from typing import Final
+
+_BOM: Final = b"\xef\xbb\xbf"
+
+
+def range_decoder() -> codecs.IncrementalDecoder:
+    """A decoder for one byte range; the caller strips a leading mark when it starts the file."""
+
+    return codecs.getincrementaldecoder("utf-8")("replace")
+
+
+def starts_with_bom(first_bytes: bytes) -> bool:
+    """Whether decoding the whole input drops its first three bytes as a byte-order mark."""
+
+    return first_bytes[:3] == _BOM
+
+
+class StreamDecoder:
+    """Decode one stream chunk by chunk exactly like decoding all of it at once."""
+
+    __slots__ = ("_decoder", "_started")
+
+    def __init__(self) -> None:
+        self._decoder = range_decoder()
+        self._started = False
+
+    def decode(self, data: bytes | None = None) -> str:
+        """Text for ``data``, holding back an incomplete character; no ``data`` flushes the stream."""
+
+        text = self._decoder.decode(b"", True) if data is None else self._decoder.decode(data, False)
+        if self._started or text == "":
+            return text
+        self._started = True
+        # U+FEFF encodes only as EF BB BF, so a leading U+FEFF is exactly a mark at the stream start.
+        return text[1:] if text.startswith("\ufeff") else text
+
+
+__all__ = ["StreamDecoder", "range_decoder", "starts_with_bom"]

pi_durable/env/line_scan.py

@@ -0,0 +1,190 @@
+"""Find where selected lines of a file are, in bounded memory.
+
+Port of the v1.3.0 ``env/line-scan`` helper. A caller feeds a file's bytes in order; the finished
+scan gives byte offsets and decoded sizes for the selected line range.
+"""
+
+from __future__ import annotations
+
+import codecs
+import math
+from typing import Any
+
+from ..truncate import utf8_byte_length
+from .decode import range_decoder, starts_with_bom
+
+_NEWLINE = 0x0A
+_MAX_SAFE_INTEGER = 9_007_199_254_740_991
+
+
+class LineScan(dict[str, Any]):
+    """Where lines of a file are, as :meth:`BinaryReader.scan_lines` found them."""
+
+
+class LineScanner:
+    """Compute a :class:`LineScan` from a file's bytes fed in order.
+
+    ``start_line`` and ``end_line`` must be non-negative integers with ``end_line > start_line``;
+    an absent ``end_line`` means to the end of the file.
+    """
+
+    __slots__ = (
+        "_bom",
+        "_end",
+        "_end_line",
+        "_first_line",
+        "_first_line_bytes",
+        "_first_line_end",
+        "_head",
+        "_last_line_start",
+        "_line_start",
+        "_newlines",
+        "_position",
+        "_selected_bytes",
+        "_selection",
+        "_start",
+        "_start_line",
+    )
+
+    def __init__(self, start_line: int, end_line: int | None = None) -> None:
+        if (
+            isinstance(start_line, bool)
+            or not isinstance(start_line, int)
+            or start_line < 0
+            or start_line > _MAX_SAFE_INTEGER
+        ):
+            raise ValueError("Invalid line range")
+        if end_line is not None and (
+            isinstance(end_line, bool)
+            or not isinstance(end_line, int)
+            or end_line <= start_line
+            or end_line > _MAX_SAFE_INTEGER
+        ):
+            raise ValueError("Invalid line range")
+        self._start_line = start_line
+        self._end_line = math.inf if end_line is None else end_line
+        self._position = 0
+        self._newlines = 0
+        self._line_start = 0
+        self._start: int | None = None
+        self._end: int | None = None
+        self._first_line_end: int | None = None
+        self._last_line_start: int | None = None
+        self._selected_bytes = 0
+        self._first_line_bytes = 0
+        self._selection: codecs.IncrementalDecoder | None = None
+        self._first_line: codecs.IncrementalDecoder | None = None
+        self._head: list[int] | None = []
+        self._bom = False
+        if start_line == 0:
+            self._begin(0)
+
+    def push(self, chunk: bytes) -> None:
+        """Feed the next bytes of the file."""
+
+        if self._head is not None:
+            take = min(3 - len(self._head), len(chunk))
+            self._head.extend(chunk[:take])
+            if len(self._head) < 3:
+                return
+            self._release_head()
+            chunk = chunk[take:]
+        self._process(chunk)
+
+    def _release_head(self) -> None:
+        head = bytes(self._head or ())
+        self._head = None
+        self._bom = starts_with_bom(head)
+        self._process(head)
+
+    def _process(self, chunk: bytes) -> None:
+        base = self._position
+        consumed = 0
+        index = chunk.find(b"\n")
+        while index != -1:
+            # The newline ends line ``self._newlines``; it belongs to the selection between lines only.
+            self._feed(chunk, base, consumed, index)
+            line = self._newlines
+            position = base + index
+            if line == self._start_line:
+                self._end_first_line(position)
+            if line == self._end_line - 1:
+                self._end_selection(position)
+            self._feed(chunk, base, index, index + 1)
+            consumed = index + 1
+            self._newlines += 1
+            self._line_start = position + 1
+            if self._newlines == self._start_line:
+                self._begin(self._line_start)
+            if self._newlines == self._end_line - 1:
+                self._last_line_start = self._line_start
+            index = chunk.find(b"\n", consumed)
+        self._feed(chunk, base, consumed, len(chunk))
+        self._position += len(chunk)
+
+    def finish(self) -> LineScan:
+        """Finish the scan; no more bytes may follow."""
+
+        if self._head is not None:
+            self._release_head()
+        size = self._position
+        if self._start is None:
+            return LineScan(
+                newlines=self._newlines,
+                start=size,
+                end=size,
+                firstLineEnd=size,
+                lastLineStart=size,
+                selectedBytes=0,
+                firstLineBytes=0,
+            )
+        if self._first_line_end is None:
+            self._end_first_line(size)
+        if self._end is None:
+            self._end_selection(size)
+        return LineScan(
+            newlines=self._newlines,
+            start=self._start,
+            end=self._end if self._end is not None else size,
+            firstLineEnd=self._first_line_end if self._first_line_end is not None else size,
+            # A selection that reaches past the last line ends with the last line.
+            lastLineStart=self._last_line_start if self._last_line_start is not None else self._line_start,
+            selectedBytes=self._selected_bytes,
+            firstLineBytes=self._first_line_bytes,
+        )
+
+    def _begin(self, start: int) -> None:
+        self._start = start
+        if self._start_line == self._end_line - 1:
+            self._last_line_start = start
+        self._selection = range_decoder()
+        self._first_line = range_decoder()
+
+    def _end_first_line(self, position: int) -> None:
+        self._first_line_end = position
+        if self._first_line is not None:
+            self._first_line_bytes += utf8_byte_length(self._first_line.decode(b"", True))
+        self._first_line = None
+
+    def _end_selection(self, position: int) -> None:
+        self._end = position
+        if self._selection is not None:
+            self._selected_bytes += utf8_byte_length(self._selection.decode(b"", True))
+        self._selection = None
+
+    def _feed(self, chunk: bytes, base: int, start: int, end: int) -> None:
+        """Feed ``chunk[start:end]``, which starts at file offset ``base``, to open decoders."""
+
+        # Decoding the whole file drops a leading byte-order mark.
+        if self._bom and base + start < 3:
+            start = min(end, 3 - base)
+        if end <= start:
+            return
+        data = chunk[start:end]
+        if self._selection is not None:
+            self._selected_bytes += utf8_byte_length(self._selection.decode(data, False))
+        if self._first_line is not None:
+            self._first_line_bytes += utf8_byte_length(self._first_line.decode(data, False))
+
+
+__all__ = ["LineScanner"]