pathapi 0.1.0__py3-none-any.whl

pathapi/_tree.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import typing as _ty
+
+from .path import Path, Pathname
+
+__all__ = ["TreePath"]
+
+
+class TreePath(Path):
+    """A `Path` whose names are POSIX segments below the root of a tree that a
+    backend serves (a remote directory, the members of an archive).
+
+    Joined and normalized like `PurePosixPath`: empty and "." segments
+    collapse and an absolute argument restarts the join. Relative and rooted
+    spellings name the same node, because the tree has no working directory.
+    A subclass supplies the constructor, `_rebuild()` and the backend it
+    carries; this class holds the names only."""
+
+    __slots__ = ("_segments",)
+
+    @classmethod
+    def _family(cls) -> type:
+        """The class directly under `TreePath` that `cls` derives from: its
+        instances can donate a backend to a join."""
+        return next(c for c in cls.__mro__ if TreePath in c.__bases__)
+
+    @classmethod
+    def _join(cls, segments: "_ty.Iterable[object]") -> "tuple[str, TreePath | None]":
+        """`(text, donor)`: the joined text of `segments` and the first of them
+        that belongs to this class's family, whose backend the result keeps.
+        `NotImplementedError` for another `Path` (so `/` answers
+        `NotImplemented`), `TypeError` for what is not a path at all."""
+        path = ""
+        donor = None
+        family = cls._family()
+        for segment in segments:
+            if isinstance(segment, family):
+                text = segment.as_posix()
+                if donor is None:
+                    donor = segment
+            elif isinstance(segment, Path):
+                raise NotImplementedError()
+            elif isinstance(segment, Pathname):
+                text = "/".join(segment.segments)
+            elif isinstance(segment, str):
+                text = segment
+            else:
+                raise TypeError(
+                    "argument should be a str or a Pathname, "
+                    f"not {type(segment).__name__!r}"
+                )
+            if text.startswith("/") or not path:
+                path = text
+            elif text:
+                path = f"{path}/{text}"
+        return path, donor
+
+    @staticmethod
+    def _parse(path: str) -> tuple:
+        """Segments of a joined path string: `("", "")` for the root,
+        `("", name, ...)` for another rooted path, `(name, ...)` for a
+        relative one and `()` for the empty path."""
+        names = [name for name in path.split("/") if name and name != "."]
+        if path.startswith("/"):
+            return ("", *names) if names else ("", "")
+        return tuple(names)
+
+    def _rebuild(self, text: str) -> _ty.Self:
+        """A path of this class and backend for the joined text `text`."""
+        raise NotImplementedError
+
+    def __str__(self) -> str:
+        return self.as_posix()
+
+    def __copy__(self):
+        return self._rebuild(self.as_posix())
+
+    def __deepcopy__(self, memo):
+        # A backend is a connection or a handle, not part of the value.
+        inst = memo[id(self)] = self.__copy__()
+        return inst
+
+    @property
+    def segments(self):
+        return self._segments
+
+    @property
+    def parts(self):
+        return self.segments, self._backend
+
+    @property
+    def parent(self):
+        segments = self.segments
+        if not segments or segments == ("", ""):
+            return self
+        if len(segments) == 2 and segments[0] == "":
+            return self._rebuild("/")
+        return self._rebuild("/".join(segments[:-1]))
+
+    def _cwd(self):
+        return self._rebuild("/")
+
+    def with_segments(self, *segments: str):
+        return self._rebuild("/".join(segments))
+
+    def relative_to(self, other, *, walk_up: bool = False):
+        """`self` below `other`, as a relative path on the same backend.
+
+        `other` is a path of this class on the same tree or a `str`. Names are
+        compared as written, with no `..` applied. `ValueError` when `self`
+        is not below `other`, when the two differ in being rooted, or when
+        the trees differ. With `walk_up`, a path beside `other` gets a `..`
+        per level climbed; a `..` in `other` cannot be walked."""
+        if isinstance(other, self._family()):
+            if not self._same_filesystem(other):
+                raise ValueError(
+                    f"{str(self)!r} and {str(other)!r} are not in the same tree"
+                )
+        else:
+            other = self.with_segments(other)
+        rooted = self.segments[:1] == ("",)
+        other_rooted = other.segments[:1] == ("",)
+        names = [name for name in self.segments if name]
+        other_names = [name for name in other.segments if name]
+        for step in range(len(other_names) + 1):
+            keep = len(other_names) - step
+            if rooted == other_rooted and names[:keep] == other_names[:keep]:
+                return self.with_segments(*[".."] * step, *names[keep:])
+            if not walk_up:
+                raise ValueError(
+                    f"{str(self)!r} is not in the subpath of {str(other)!r}"
+                )
+            if keep and other_names[keep - 1] == "..":
+                raise ValueError(f"'..' segment in {str(other)!r} cannot be walked")
+        raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors")

pathapi/archive/__init__.py

@@ -0,0 +1,44 @@
+"""Zip and tar archives as trees of paths, on the standard library alone
+(`zipfile`, `tarfile`).
+
+`ArchivePath`, `ZipPath` and `TarPath` are `Path`s for the members of an
+archive file, which is any `Path`; they need no URI.
+"""
+
+from __future__ import annotations
+
+import sys as _sys
+import types as _types
+
+from . import _backend
+from ._pack import detect_format, make_archive, unpack_archive
+from ._path import ArchivePath, TarPath, ZipPath
+from ._snapshot import MAX_LINKS, LinkTarget
+
+__all__ = [
+    "ArchivePath",
+    "LinkTarget",
+    "MAX_LINKS",
+    "MEMBER_SPOOL_BYTES",
+    "TarPath",
+    "ZipPath",
+    "detect_format",
+    "make_archive",
+    "unpack_archive",
+]
+
+
+class _Module(_types.ModuleType):
+    """The module, with `MEMBER_SPOOL_BYTES` (the size past which a member read
+    goes to a temporary file) read from and written to where it takes effect."""
+
+    @property
+    def MEMBER_SPOOL_BYTES(self) -> int:
+        return _backend.MEMBER_SPOOL_BYTES
+
+    @MEMBER_SPOOL_BYTES.setter
+    def MEMBER_SPOOL_BYTES(self, value: int) -> None:
+        _backend.MEMBER_SPOOL_BYTES = value
+
+
+_sys.modules[__name__].__class__ = _Module

pathapi/archive/_backend.py

@@ -0,0 +1,393 @@
+from __future__ import annotations
+
+import contextlib as _contextlib
+import io as _io
+import os as _os
+import shutil as _shutil
+import tempfile as _tempfile
+import threading as _threading
+import weakref as _weakref
+
+from ..path import Path
+from ..protocols._relation import _local_file
+from .._filestat import FileStat
+from ._snapshot import _Snapshot
+
+_registry_lock = _threading.Lock()
+_registry: "_weakref.WeakValueDictionary[tuple[type, object], _ArchiveBackend]" = (
+    _weakref.WeakValueDictionary()
+)
+
+
+def _local_outer_path(outer: "Path") -> "str | None":
+    """The local filesystem path of an outer archive that is a file of this
+    machine (a `LocalPath`, or a path holding one such as a `file:` URI), else
+    None."""
+    local = _local_file(outer)
+    if local is None:
+        return None
+    try:
+        return str(local)
+    except (NotImplementedError, ValueError):
+        return None
+
+
+def _outer_identity(outer: "Path") -> object:
+    """What names a non-local outer archive in the registry: its node, for a
+    path that answers `_node_key()`, else its URI."""
+    try:
+        node = outer._node_key()
+    except Exception:
+        node = None
+    if node is not None:
+        space, names = node
+        return id(space), names
+    return outer.as_uri()
+
+
+def _registry_key(
+    backend_cls: type, outer: "Path", identity: object = None
+) -> "tuple[type, object]":
+    local = _local_outer_path(outer)
+    if local is not None:
+        # One key per file, not per spelling: `C:` vs `c:` or a symlink used
+        # to get independent handles onto the same file.
+        return backend_cls, _os.path.normcase(_os.path.realpath(local))
+    return backend_cls, _outer_identity(outer) if identity is None else identity
+
+
+def _get_backend(
+    backend_cls: "type[_ArchiveBackend]", outer: "Path", identity: object = None
+) -> "_ArchiveBackend":
+    """Return the shared `_ArchiveBackend` for `outer`, creating one if this
+    is the first live reference. Keyed by (backend class, canonical local
+    file path -- or `identity`, else the node or URI of a non-local outer) so
+    independently-constructed paths pointing at the same archive share one
+    handle instead of each opening their own -- avoids the stale-read/
+    corrupt-write hazard of two handles writing the same underlying file.
+    Backed by a `WeakValueDictionary`: once every path referencing a given
+    backend is garbage-collected, the backend itself is collected (its
+    `__del__` closes the underlying handle) and the registry entry is
+    dropped automatically -- no explicit refcounting needed."""
+    key = _registry_key(backend_cls, outer, identity)
+    with _registry_lock:
+        backend = _registry.get(key)
+        if backend is None:
+            backend = backend_cls(outer)
+            _registry[key] = backend
+        return backend
+
+
+MEMBER_SPOOL_BYTES = 16 * 1024 * 1024
+"""A member read through an archive path is held in memory up to this many
+bytes and spooled to an unnamed temporary file beyond that, so a member
+larger than memory can be read."""
+
+_CHUNK = 64 * 1024
+
+
+def _spool(source) -> "_io.IOBase":
+    """A seekable copy of the stream `source`, in memory up to
+    `MEMBER_SPOOL_BYTES` and in a temporary file past it. The copy is what a
+    caller keeps, so the archive's shared handle is not tied up by a reader
+    that keeps its stream open across later changes to the archive."""
+    memory = _io.BytesIO()
+    while True:
+        chunk = source.read(_CHUNK)
+        if not chunk:
+            memory.seek(0)
+            return memory
+        memory.write(chunk)
+        if memory.tell() > MEMBER_SPOOL_BYTES:
+            break
+    disk = _tempfile.TemporaryFile()
+    try:
+        with memory.getbuffer() as written:
+            disk.write(written)
+        memory.close()
+        _shutil.copyfileobj(source, disk, _CHUNK)
+        disk.seek(0)
+    except BaseException:
+        disk.close()
+        raise
+    return disk
+
+
+class _LazyReadFile:
+    """A read-only, seekable view of a local file that holds an OS handle
+    only while an operation runs: `release()` closes it, and the next
+    read/seek reopens it at the same position. A shared `ZipFile` or
+    `TarFile` over one therefore never keeps the archive open between
+    operations (on Windows an open handle blocks deleting or replacing the
+    file)."""
+
+    def __init__(self, path: str):
+        self.name = path
+        self._pos = 0
+        # Opened eagerly: a missing file raises FileNotFoundError here,
+        # which `zipfile` would otherwise turn into BadZipFile.
+        self._fp = open(path, "rb")
+
+    def _file(self):
+        if self._fp is None:
+            self._fp = open(self.name, "rb")
+            self._fp.seek(self._pos)
+        return self._fp
+
+    def read(self, size=-1):
+        data = self._file().read(size)
+        self._pos = self._fp.tell()
+        return data
+
+    def seek(self, offset, whence=0):
+        self._pos = self._file().seek(offset, whence)
+        return self._pos
+
+    def tell(self):
+        return self._pos
+
+    def seekable(self):
+        return True
+
+    def readable(self):
+        return True
+
+    def release(self):
+        fp, self._fp = self._fp, None
+        if fp is not None:
+            fp.close()
+
+    close = release
+
+
+class _ArchiveBackend:
+    """Lazily opens+caches the archive handle for one outer archive.
+    Shared by every archive path derived from the same one, both through
+    normal backend propagation (`with_segments`/`joinpath`/...) AND across
+    independently-constructed paths via the module-level `_get_backend`
+    registry above.
+
+    For a local outer the cached handle is revalidated against the file's
+    (inode, size, mtime) on every access and reopened when it changed, so a
+    write by another process or tool is seen instead of being overwritten
+    from a stale central directory. `_lock` serializes every use of the
+    shared handle."""
+
+    __slots__ = (
+        "outer",
+        "_handle",
+        "_signature",
+        "_lock",
+        "_snapshot",
+        "_prefetched",
+        "__weakref__",
+    )
+
+    #: What a damaged archive raises, whatever decoder noticed it.
+    _FORMAT_ERROR: "type[Exception]" = OSError
+
+    def __init__(self, outer: "Path"):
+        self.outer = outer
+        self._handle = None
+        self._signature = None
+        self._snapshot = None
+        self._prefetched = None
+        self._lock = _threading.RLock()
+
+    @property
+    def lock(self):
+        """Held across a read of the shared handle and the use of its
+        result: another thread's rewrite reopens the handle."""
+        return self._lock
+
+    def _offer(self, data: bytes) -> None:
+        """Hands over the bytes of a non-local outer that were read to decide
+        its format, so opening does not fetch them a second time."""
+        with self._lock:
+            if self._handle is None and self._prefetched is None:
+                self._prefetched = data
+
+    def _outer_bytes(self) -> bytes:
+        """The whole outer archive of a non-local outer."""
+        with self._lock:
+            data, self._prefetched = self._prefetched, None
+        return self.outer.read_bytes() if data is None else data
+
+    def _is_format_error(self, error: Exception) -> bool:
+        """Whether `error`, raised by the decoder, says the archive is damaged."""
+        return False
+
+    @_contextlib.contextmanager
+    def _damage_as_format_error(self):
+        """Re-raises what a decoder raises for a damaged archive as the
+        format's own error, so a caller handles one family per format.
+        Anything else (a missing member, an I/O error) passes unchanged."""
+        try:
+            yield
+        except Exception as error:
+            if isinstance(error, self._FORMAT_ERROR) or not self._is_format_error(
+                error
+            ):
+                raise
+            raise self._FORMAT_ERROR(str(error) or type(error).__name__) from error
+
+    def __del__(self):
+        try:
+            self._close_handle()
+        except Exception:
+            pass
+
+    def _open(self):
+        raise NotImplementedError
+
+    def _close_handle(self):
+        # Drop the cached handle so the next operation (through any instance sharing this
+        # backend) reopens and sees the change. The snapshot derives from that handle's
+        # entries, so it goes too; every mutation replaces the archive (`_replace_outer`).
+        handle = self._handle
+        self._handle = None
+        self._snapshot = None
+        if handle is not None:
+            handle.close()
+
+    def refresh(self) -> None:
+        """Forget the handle and any bytes kept from the outer archive; the
+        next use reads the outer again."""
+        with self._lock:
+            self._prefetched = None
+            self._close_handle()
+
+    def _outer_signature(self):
+        local = _local_outer_path(self.outer)
+        if local is None:
+            return None
+        try:
+            st = _os.stat(local)
+        except OSError:
+            return None
+        return st.st_ino, st.st_size, st.st_mtime_ns
+
+    def _outer_file(self) -> str:
+        """The local file of the outer archive; `ValueError` for any other."""
+        local = _local_outer_path(self.outer)
+        if local is None:
+            raise ValueError(f"the archive {self.outer} is not a local file")
+        return local
+
+    @property
+    def handle(self):
+        with self._lock:
+            signature = self._outer_signature()
+            if self._handle is not None and signature != self._signature:
+                self._close_handle()
+            if self._handle is None:
+                self._handle = self._open()
+                self._signature = signature
+            return self._handle
+
+    @property
+    def writable(self) -> bool:
+        return False
+
+    def snapshot(self) -> _Snapshot:
+        """The members of the open handle, derived once and cached with it
+        (see `_Snapshot`). Revalidates the handle once; a reopen drops the
+        cached one. Every listing, stat, read and write starts here, and
+        rebuilding it from the entries each time made an operation on a
+        20k-member archive scan all of them."""
+        with self._lock:
+            self.handle  # revalidates, and clears the snapshot if it reopened
+            if self._snapshot is None:
+                entries = self._entries()
+                self._snapshot = _Snapshot(
+                    type(self), entries, self._link_texts(entries)
+                )
+            return self._snapshot
+
+    def member_index(self) -> "dict[str, str]":
+        """Normalized member name -> the key this backend knows it by."""
+        return self.snapshot().index
+
+    def raw_names(self, normalized: str) -> "tuple[str, ...]":
+        return self.snapshot().raw_names(normalized)
+
+    def _entries(self) -> "list[tuple[str, object]]":
+        """`(key, info)` for every entry of the open handle, in archive
+        order; reads nothing from the archive."""
+        raise NotImplementedError
+
+    def _link_texts(self, entries) -> "dict[str, str | None]":
+        """`{key: stored text}` for the entries that are symbolic links (the
+        text is None when it cannot be read); empty for a format without."""
+        return {}
+
+    @staticmethod
+    def _is_directory(info) -> bool:
+        raise NotImplementedError
+
+    @staticmethod
+    def _stat_of(info, snapshot: _Snapshot) -> FileStat:
+        raise NotImplementedError
+
+    def read_member(self, path: str):
+        raise NotImplementedError
+
+
+def _detect_backend_cls(outer: "Path") -> "tuple[type, bytes | None]":
+    """Detect zip vs tar for `outer` (extension, then magic-byte sniff --
+    see `detect_format`): the matching backend class and, when
+    the sniff had to read a non-local outer whole, those bytes. An error
+    reading the outer propagates. Lazily imports `._zip`/`._tar` (rather than
+    at module level) to avoid a circular import: both submodules import
+    `_ArchiveBackend` from this module."""
+    from ._pack import detect_format
+    from ._tar import _TarBackend
+    from ._zip import _ZipBackend
+
+    local = _local_outer_path(outer) is not None
+    whole: "list[bytes]" = []
+
+    def _peek() -> bytes:
+        if local:
+            with outer.open("rb") as f:
+                return f.read(4)
+        whole.append(outer.read_bytes())
+        return whole[0][:4]
+
+    fmt = detect_format(outer.name, _peek)
+    return (_ZipBackend if fmt == "zip" else _TarBackend), (whole[0] if whole else None)
+
+
+class _UndecidedBackend:
+    """Stands in for the backend of an `ArchivePath` until something needs
+    it: the format is settled on first use, from the extension or, failing
+    that, from the bytes of the outer archive. Building a path, printing it
+    and joining names onto it never reach the outer. A failure to read the
+    outer propagates and decides nothing, so the next use tries again.
+    `shared` and `identity` are what `_get_backend` is asked with."""
+
+    __slots__ = ("outer", "_lock", "_resolved", "_shared", "_identity", "__weakref__")
+
+    def __init__(self, outer: "Path", shared: bool = True, identity: object = None):
+        self.outer = outer
+        self._lock = _threading.RLock()
+        self._resolved = None
+        self._shared = shared
+        self._identity = identity
+
+    def resolve(self) -> "_ArchiveBackend":
+        resolved = self._resolved
+        if resolved is None:
+            with self._lock:
+                resolved = self._resolved
+                if resolved is None:
+                    backend_cls, data = _detect_backend_cls(self.outer)
+                    if not self._shared:
+                        # It must not borrow a handle another path opened.
+                        resolved = backend_cls(self.outer)
+                    else:
+                        resolved = _get_backend(backend_cls, self.outer, self._identity)
+                    if data is not None:
+                        resolved._offer(data)
+                    self._resolved = resolved
+        return resolved

pathapi/archive/_names.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from .._names import is_safe_child_name
+
+
+def _is_safe_member_name(name: str) -> bool:
+    """Whether archive member `name` is a usable relative path inside the
+    archive: every part a real name, none of them escaping the root.
+
+    Only the rules that hold on every platform are applied here, because an
+    archive has no platform of its own: `\\`, `:` and trailing dots are
+    ordinary filename characters on POSIX, and an archive written there may
+    legitimately contain `C:drive.txt` or `a\\b`. Refusing to *join* such a
+    name onto a destination that reads it differently is the destination's
+    rule, and is applied per target where the joining happens --
+    `Path.copy(recursive=True)`, `PathSyncer` and `unpack_archive()`
+    each check `is_safe_child_name(..., windows=is_windows_flavoured(dest))`.
+    A trailing `/` (a directory marker) is allowed."""
+    if name.endswith("/"):
+        name = name[:-1]
+    return all(is_safe_child_name(part) for part in name.split("/"))
+
+
+def _normalize_member_name(name: str) -> "str | None":
+    """`name` as a normalized POSIX relative path, or None if it does not
+    stay inside the archive root.
+
+    A member name is a relative path, and writers spell it several ways for
+    the same file: `./x` (`tar -C dir .`, `TarFile.add(arcname=".")`,
+    `shutil.make_archive`), `a//b`, `a/./b`, `a/b/../c`. Normalizing it the
+    way a URI reference resolves -- RFC 3986 dot-segment removal, with empty
+    segments dropped -- gives one member one name, so a listing and a lookup
+    agree however the archive was written, and the same spelling addresses
+    the same member in a zip and in a tar.
+
+    A name that would leave the root has no normalized form inside the
+    archive and is rejected (None): absolute (`/abs`), or with more `..`
+    than parts to spend them on. `..` that stays inside is resolved
+    (`pkg/../ok.txt` is `ok.txt`). A drive-shaped part (`C:x`) is NOT
+    rejected here: it is a legal POSIX filename, and only a destination
+    that reads names with Windows rules is endangered by it -- see
+    `_is_safe_member_name`. A trailing `/` (a directory marker) is kept, and
+    the archive root itself normalizes to "".
+    """
+    if name.startswith("/"):
+        return None
+    directory = name.endswith("/")
+    parts: "list[str]" = []
+    for part in name.split("/"):
+        if part in ("", "."):
+            continue
+        if part == "..":
+            if not parts:
+                return None  # escapes the archive root
+            parts.pop()
+            continue
+        parts.append(part)
+    normalized = "/".join(parts)
+    if not normalized:
+        return ""  # the archive root ("", ".", "./", "a/..")
+    if not _is_safe_member_name(normalized):
+        return None
+    return normalized + "/" if directory else normalized