pathapi 0.1.0__py3-none-any.whl

pathapi/sync/_compare.py

@@ -0,0 +1,157 @@
+"""Whether a source and a target already agree: checksum and modification time."""
+
+from __future__ import annotations
+
+import math as _math
+
+from ..path import Path
+from .. import checksum as _checksum
+from ._events import PathAndStat
+
+#: `PathSyncer`'s own default checksum policy (native digest preferred, streaming
+#: fallback), which `sync()` special-cases (`_default_checksums_match()`); a
+#: caller-supplied `checksum` is called as given, one path in, value out, with `==`.
+_DEFAULT_ALGORITHM = "md5"
+
+
+def _default_checksum(entry: "PathAndStat") -> str:
+    # The sentinel that marks the default policy: `sync()` recognizes it and
+    # calls `_default_checksums_match()` instead. It is also the plain
+    # `Callable[[PathAndStat], Any]` that `PathSyncer().checksum` reads as.
+    native = _checksum.native(entry.path, _DEFAULT_ALGORITHM)
+    if native is not None:
+        return native
+    return _checksum.stream(entry.path, _DEFAULT_ALGORITHM)
+
+
+def _shared_native_algorithm(source: Path, target: Path) -> "str | None":
+    """Pick an algorithm both sides advertise via
+    `NativeChecksum.supported_checksums()` (see `protocols/checksum.py`),
+    preferring `_DEFAULT_ALGORITHM` ("md5") when both sides support it,
+    `PathSyncer`'s default algorithm. Returns `None` if either side has no
+    `supported_checksums` at all (a `NativeChecksum` implementation isn't
+    required to override the advisory method -- the base default already
+    returns `frozenset()`, indistinguishable here from "doesn't implement
+    the protocol"), or if the two sides' advertised sets don't overlap.
+    """
+    source_supported = getattr(source, "supported_checksums", None)
+    target_supported = getattr(target, "supported_checksums", None)
+    if source_supported is None or target_supported is None:
+        return None
+    shared = source_supported() & target_supported()
+    if not shared:
+        return None
+    if _DEFAULT_ALGORITHM in shared:
+        return _DEFAULT_ALGORITHM
+    return next(iter(shared))
+
+
+def _default_checksums_match(source: "PathAndStat", target: "PathAndStat") -> bool:
+    """`PathSyncer`'s default in-sync check: prefer each side's
+    `NativeChecksum.checksum()` (no network transfer needed just to
+    *decide* whether a copy is needed), but only trust a native digest from
+    one side if the OTHER side can also produce a digest under the exact
+    same algorithm -- native or streamed. Mixing "native digest from A" with
+    "streamed digest from B" would only be numerically safe if both are
+    guaranteed true content hashes under the same algorithm; the protocol
+    contract for `NativeChecksum.checksum()` already guarantees that
+    (`NotImplementedError` on any doubt, e.g. S3 multipart ETags), so this
+    conservative both-native-or-both-streamed policy is a deliberate
+    simplicity choice, not a soundness requirement -- see
+    `protocols/checksum.py`.
+
+    Algorithm selection: try `supported_checksums()` intersection first
+    (`_shared_native_algorithm`) -- avoids a doomed native attempt when the
+    two sides' capabilities don't overlap on `_DEFAULT_ALGORITHM`. Falls
+    back to trying `_DEFAULT_ALGORITHM` directly (the pre-`supported_checksums`
+    behavior) when either side doesn't implement the advisory method at
+    all, since a `NativeChecksum` implementation is never required to
+    override it.
+    """
+    # Whether each side can answer at all is known without I/O: ask for a
+    # digest only when both can, and ask the second only if the first did,
+    # so a digest that would be thrown away is never computed.
+    if all(getattr(entry.path, "checksum", None) for entry in (source, target)):
+        algorithm = (
+            _shared_native_algorithm(source.path, target.path) or _DEFAULT_ALGORITHM
+        )
+        source_native = _checksum.native(source.path, algorithm)
+        if source_native is not None:
+            target_native = _checksum.native(target.path, algorithm)
+            if target_native is not None:
+                return source_native == target_native
+    return _checksum.stream(source.path, _DEFAULT_ALGORITHM) == _checksum.stream(
+        target.path, _DEFAULT_ALGORITHM
+    )
+
+
+def _is_local(path) -> bool:
+    """`quick_check`'s locality test. `is_local()` is an optional method of a
+    path whose server may be this machine or another one (a URI path, an SFTP
+    path); a plain `LocalPath` or `MemPath` has no such method at all. Both of
+    those ARE effectively local for this purpose (a real local filesystem, or
+    an in-memory structure with no network cost either way), so "no
+    `is_local()` method" is treated as local -- the safe default, since it
+    only preserves the always-checksum behavior rather than skipping a
+    comparison it shouldn't.
+
+    An `is_local()` that resolves a host name can fail with `OSError` when the
+    resolver itself fails (no network at all), as opposed to a name that
+    resolves nowhere, which answers `False`. That is treated as "local" for
+    the same safe-default reason: quick_check does not kick in and the sync
+    falls through to a real checksum comparison, rather than letting an
+    unrelated resolver failure crash the sync outright.
+    """
+    is_local = getattr(path, "is_local", None)
+    if is_local is None:
+        return True
+    try:
+        return is_local()
+    except OSError:
+        return True
+
+
+#: Two floats of one instant differ by less than this after a filesystem has
+#: stored them as nanoseconds and handed them back.
+_MTIME_TICK = 1e-6
+
+
+def _same_mtime(first: float, second: float) -> bool:
+    """Whether two modification times name the same instant at the resolution
+    of the coarser: whole seconds when either has no fractional part (SFTP,
+    FTP and the object stores report seconds, a local filesystem fractions),
+    else to a microsecond."""
+    if first == second:
+        return True
+    if first % 1 == 0 or second % 1 == 0:
+        return _math.floor(first) == _math.floor(second)
+    return abs(first - second) < _MTIME_TICK
+
+
+def _quick_check_in_sync(source: "PathAndStat", target: "PathAndStat") -> bool:
+    """The rsync-style "quick check" pre-check: True only when BOTH
+    `st_size` and `st_mtime` already match between the two cached stats --
+    metadata `PathAndStat` already carries from listing, no extra round
+    trip. Never used to conclude "out of sync" (a caller must fall through
+    to a real checksum comparison on any mismatch) -- see `PathSyncer`'s
+    class docstring for why.
+
+    An `st_mtime` of 0 (or missing) on either side is "unknown", never a
+    timestamp: MemPath, GitHub/GitLab, `data:` and HTTP without
+    Last-Modified all report 0, and two unknowns are not a match. So is a
+    stat whose listing did not report a size (`size_known` False). Times
+    match as `_same_mtime()` says.
+    """
+    source_stat, target_stat = source.stat, target.stat
+    if source_stat is None or target_stat is None:
+        return False
+    if not (
+        getattr(source_stat, "size_known", True)
+        and getattr(target_stat, "size_known", True)
+    ):
+        return False
+    if not source_stat.st_mtime or not target_stat.st_mtime:
+        return False
+    return source_stat.st_size == target_stat.st_size and _same_mtime(
+        source_stat.st_mtime, target_stat.st_mtime
+    )

pathapi/sync/_events.py

@@ -0,0 +1,97 @@
+"""The events a sync reports and the path-and-stat pair it works on."""
+
+from __future__ import annotations
+
+import enum as _enum
+import stat as _stat
+import typing as _ty
+
+from ..path import Path
+from ..protocols._bundles import Readable
+from .._filestat import FileStat
+
+
+class SyncEvent(_enum.Enum):
+    """Events `PathSyncer.hook()` fires during a sync, for progress/logging
+    callbacks."""
+
+    Copy = _enum.auto()
+    RemovedMissing = _enum.auto()
+    Synced = _enum.auto()
+    CreatedDirectory = _enum.auto()
+    SyncStart = _enum.auto()
+    TypeMismatch = _enum.auto()
+    CheckTargetChild = _enum.auto()
+    CheckTargetChildren = _enum.auto()
+    SyncChild = _enum.auto()
+    SyncChildren = _enum.auto()
+    Symlink = _enum.auto()
+    #: Comparing a source/target file pair (quick check or checksum)
+    #: failed. Only ever passed to `ignore_error`; no hook fires for it.
+    Compare = _enum.auto()
+    #: The source entry is neither a regular file, a directory nor a
+    #: symlink (a FIFO, socket or device, or a stat with no file type): it is
+    #: skipped and the target is left untouched.
+    Skipped = _enum.auto()
+    #: `ignore_error` tolerated an error for this source/target pair. Fired
+    #: after the policy returned True; the failing event and the exception
+    #: are in the WARNING record logged on the `pathapi.sync` logger.
+    Error = _enum.auto()
+
+
+class PathAndStat(object):
+    """A `Path` plus its cached `stat()` result (`None` if it doesn't
+    exist). `is_*` attribute access (e.g. `.is_file()`) delegates to the
+    cached stat, returning a false-returning callable if the path doesn't
+    exist; any other unknown attribute raises `AttributeError` as normal.
+
+    `follow_symlink` (in `__init__` and `refresh()`) defaults to `True`,
+    like `pathlib.Path.stat()` and `FileStat.from_path()`: a symlink is
+    described by what it points at. Pass `False` for the link itself
+    (`lstat()`); `PathSyncer` always passes its `follow_symlinks` setting."""
+
+    __slots__ = ("_path", "_stat")
+
+    def __init__(self, path: "Readable", *, follow_symlink: bool = True) -> None:
+        self._path = path
+        self.refresh(follow_symlink)
+
+    @classmethod
+    def from_stat(cls, path: "Readable", stat: FileStat | None) -> "PathAndStat":
+        entry = cls.__new__(cls)
+        entry._path = path
+        entry._stat = stat
+        return entry
+
+    def __str__(self) -> str:
+        return str(self.path)
+
+    def __repr__(self) -> str:
+        return str((self.path, self._stat))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def stat(self):
+        return self._stat
+
+    def exists(self):
+        return self.stat is not None
+
+    def refresh(self, follow_symlink: bool = True):
+        self._stat = FileStat.from_path(self.path, follow_symlink=follow_symlink)
+
+    def __getattr__(self, name: str):
+        if name.startswith("is_"):
+            if self.stat:
+                return getattr(self.stat, name)
+            else:
+                return lambda *args, **kwargs: False
+        raise AttributeError(name)
+
+
+if _ty.TYPE_CHECKING:
+
+    class PathAndStat(PathAndStat, FileStat): ...

pathapi/sync/_links.py

@@ -0,0 +1,93 @@
+"""Symbolic links in a sync: whether they can be made and whether two agree."""
+
+from __future__ import annotations
+
+import errno as _errno
+import os as _os
+import pathlib as _pathlib
+import stat as _stat
+
+from ..path import Path
+from ..protocols import Symlink, supports
+from .._filestat import FileStat
+from ._events import PathAndStat
+
+
+def _supports_symlinks(path: Path) -> bool:
+    """Whether `path`'s class makes links. Decided on the class, before
+    anything is touched; a backend that implements the primitive can still
+    fail at runtime."""
+    return supports(path, Symlink)
+
+
+def _link_text(path: Path) -> str:
+    """The raw, unresolved target string of the symlink at `path`. A path
+    that has a `.path` attribute (a URI path) holds the un-prefixed link text there,
+    since `as_posix()` prepends "host:" when the result carries a host; plain
+    stdlib Path has no `.path`, so `.as_posix()` is the only accessor there.
+    Both sides of a comparison must use this same extraction."""
+    link = path.readlink()
+    return link.path if hasattr(link, "path") else link.as_posix()
+
+
+def _is_windows_local(path: Path) -> bool:
+    return _os.name == "nt" and isinstance(path, _pathlib.Path)
+
+
+def _link_is_directory(path: Path) -> bool:
+    """Whether a symlink at `path` is (or should be recreated as) a
+    directory link. On Windows a local link records its own kind, dangling
+    or not, in its attributes; elsewhere the kind is what the link resolves
+    to (False when dangling), which only Windows targets act on."""
+    if _is_windows_local(path):
+        attributes = getattr(_os.lstat(path), "st_file_attributes", 0)
+        return bool(attributes & _stat.FILE_ATTRIBUTE_DIRECTORY)
+    try:
+        return bool(path.is_dir())
+    except (OSError, NotImplementedError):
+        return False
+
+
+def _same_link(source: Path, target: Path) -> bool:
+    """Whether the symlink `target` already matches the symlink `source`:
+    the same raw target string, and on a Windows target the same link kind
+    (a file link to a directory cannot be traversed). Any failure to read
+    either side means "not known to match", so the link is replaced."""
+    try:
+        if _link_text(source) != _link_text(target):
+            return False
+        if _is_windows_local(target):
+            return _link_is_directory(source) == _link_is_directory(target)
+        return True
+    except (OSError, NotImplementedError):
+        return False
+
+
+def _as_entry(entry) -> "PathAndStat":
+    # Hooks and error policies are declared to receive PathAndStat; a raw
+    # path that never got stat'd (its stat() is what failed) is reported with
+    # an unknown stat.
+    if isinstance(entry, PathAndStat):
+        return entry
+    return PathAndStat.from_stat(entry, None)
+
+
+def _unresolved_link(entry: "PathAndStat") -> "tuple[Exception, PathAndStat] | None":
+    """For a source entry whose followed stat found nothing: the error to
+    offer and the entry as the link itself, when the directory still lists
+    something there that cannot be resolved (a dangling link, a loop, an
+    unmounted volume). None when nothing is left at that name, so it
+    vanished after the listing. Never an instruction to delete anything."""
+    link = FileStat.from_path(entry.path, follow_symlink=False)
+    if link is None:
+        return None
+    error = None
+    try:
+        entry.path.stat()
+    except OSError as caught:
+        error = caught
+    if error is None or isinstance(error, FileNotFoundError):
+        error = FileNotFoundError(
+            _errno.ENOENT, "the link's target does not exist", str(entry.path)
+        )
+    return error, PathAndStat.from_stat(entry.path, link)

pathapi/sync/_offer.py

@@ -0,0 +1,33 @@
+"""Offering a failure to the caller's error policy once."""
+
+from __future__ import annotations
+
+#: Set on an exception once `ignore_error` has declined it, so every
+#: enclosing hook re-raises it instead of consulting the policy again with an
+#: ancestor's paths.
+_OFFERED_ATTR = "_pathapi_sync_offered"
+
+
+def _offer_error(policy, error: Exception, source, target, event) -> bool:
+    """Ask `policy` about `error` once. True means "tolerated". An error the
+    policy already declined (deeper in the tree) is re-raised by every outer
+    level without asking again."""
+    if getattr(error, _OFFERED_ATTR, False):
+        return False
+    try:
+        if policy(error, source, target, event):
+            return True
+    except BaseException as raised:
+        # The policy's own exception ends the sync: an enclosing level must
+        # not offer it back to the policy as a failure of its own.
+        _mark_offered(raised)
+        raise
+    _mark_offered(error)
+    return False
+
+
+def _mark_offered(error: BaseException) -> None:
+    try:
+        setattr(error, _OFFERED_ATTR, True)
+    except Exception:
+        pass

pathapi/sync/_paths.py

@@ -0,0 +1,50 @@
+"""Questions about the paths of a sync: junctions, mounts, names, overlap."""
+
+from __future__ import annotations
+
+from ..path import Path
+from ..protocols._relation import _relation
+
+
+def _is_junction(path: Path) -> bool:
+    """Whether `path` is a directory junction: a link to another directory
+    that lists as an ordinary one. A mount point is not one, even where the
+    platform builds both from the same reparse point: it is a filesystem
+    boundary inside the tree, and a sync writes into it."""
+    try:
+        if not path.is_junction():
+            return False
+    except (NotImplementedError, OSError, ValueError):
+        return False
+    try:
+        return not path.is_mount()
+    except (NotImplementedError, OSError, ValueError):
+        return True
+
+
+def _is_mount(path: Path) -> bool:
+    """Whether `path` is a mount point (a mounted volume, a bind mount); a
+    class that cannot answer says no."""
+    try:
+        return bool(path.is_mount())
+    except (NotImplementedError, OSError, ValueError):
+        return False
+
+
+def _child_name(path: Path) -> str:
+    # A directory child of a URI path can end in "/", leaving `.name`
+    # empty; its last real component is then the parent's name.
+    return path.name or path.parent.name
+
+
+def _paths_overlap(source: Path, target: Path) -> bool:
+    """Whether `source` and `target` are the same tree, one contains the
+    other, or they are two names of one file. Decided by `path._relation()`:
+    two files of this machine by where each really is (a symlink cannot hide
+    the overlap), whatever their classes; a type answering `_node_key()`
+    (`MemPath`, archive members) by the node, across classes and spellings;
+    anything else only between two paths of one type that
+    `_same_filesystem()` places together, so two `MemPath` trees on separate
+    backends, or two hosts behind one path type, do not overlap even with
+    equal segments."""
+    return _relation(source, target, follow=True, identity=False) is not None

pathapi/sync/_replace.py

@@ -0,0 +1,98 @@
+"""Writing a target through a temporary sibling and swapping it in."""
+
+from __future__ import annotations
+
+import pathlib as _pathlib
+import re as _re
+import uuid as _uuid
+
+from ..path import Path
+from .._filestat import FileStat
+from ._paths import _child_name
+
+#: The longest name of one directory entry, in UTF-8 bytes, that NTFS and the
+#: usual POSIX filesystems accept (NTFS counts UTF-16 units, never more).
+_NAME_MAX = 255
+
+#: A temporary sibling is `.NAME.<12 hex digits>.pathapi-tmp`, with NAME
+#: cut short when the whole would not fit in `_NAME_MAX`. Only a name matching
+#: all of this is the library's own.
+_LEFTOVER = _re.compile(r"\..+\.[0-9a-f]{12}\.pathapi-tmp")
+
+#: A leftover is stale once it has not been modified for this long (seconds).
+#: A transfer that is running writes to its file continuously, so a younger
+#: one may belong to a sync running now, in this process or in another.
+_STALE_AFTER = 24 * 60 * 60
+
+#: Temporary files (`str(path)`) this process is writing now: never stale,
+#: whatever the store reports as their modification time.
+_LIVE_TEMPS: "set[str]" = set()
+
+#: Set on an exception raised while the previous version of a file is gone
+#: and the new one exists only under its temporary name.
+_ONLY_COPY_ATTR = "_pathapi_sync_only_copy"
+
+
+def _is_leftover(name: str) -> bool:
+    return _LEFTOVER.fullmatch(name) is not None
+
+
+def _encoded_length(name: str) -> int:
+    return len(name.encode("utf-8", "surrogatepass"))
+
+
+def _temp_sibling(path: Path) -> Path:
+    # Hidden and unique, in the same directory so a rename stays on one
+    # filesystem/connection. The embedded name is cut (never the random part
+    # or the suffix) so that a name the target accepts still gets a sibling.
+    tail = f".{_uuid.uuid4().hex[:12]}.pathapi-tmp"
+    room = _NAME_MAX - 1 - len(tail)
+    name = _child_name(path)[:room]
+    while _encoded_length(name) > room:
+        name = name[:-1]
+    return path.with_name(f".{name}{tail}")
+
+
+def _only_copy(error: BaseException, temp: Path, target: Path) -> BaseException:
+    """`error`, marked so that the clean-up keeps `temp`, and, for an
+    `Exception`, replaced by an `OSError` naming `temp` (the kind follows the
+    errno, as `OSError()` maps it). Used once the previous `target` is gone,
+    when `temp` is the only copy of the new version."""
+    kept = error
+    if isinstance(error, Exception):
+        detail = (
+            f"{target} was removed to make room for the new version, which is "
+            f"kept at {temp}: {error}"
+        )
+        errno = getattr(error, "errno", None)
+        kept = OSError(errno, detail) if errno else OSError(detail)
+    try:
+        setattr(kept, _ONLY_COPY_ATTR, True)
+    except Exception:
+        pass
+    return kept
+
+
+def _replace(source: Path, target: Path) -> None:
+    """Rename `source` over the existing `target`. `os.replace` locally
+    (atomic on POSIX and Windows); elsewhere `rename()`, which replaces on
+    backends with POSIX rename semantics, and otherwise refuses -- then the
+    old target is removed first, after the new content is complete. If the
+    second rename then fails `source` is the only copy: it is kept, and the
+    error names it."""
+    if isinstance(source, _pathlib.Path):
+        source.replace(target)
+        return
+    try:
+        source.rename(target)
+    except OSError:
+        if FileStat.from_path(target, follow_symlink=False) is None:
+            raise
+        target.unlink()
+        try:
+            source.rename(target)
+        except BaseException as error:
+            kept = _only_copy(error, source, target)
+            if kept is error:
+                raise
+            raise kept from error