hlsync 1.17.2__py3-none-any.whl

hlsync/progress.py

@@ -0,0 +1,57 @@
+"""Directory progress based on work discovered by the remote traversal."""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import PurePosixPath
+from typing import TextIO
+
+
+class DirectoryReadProgress:
+    def __init__(self, output: TextIO) -> None:
+        self.output = output
+        self.terminal = output.isatty() and os.environ.get("TERM") != "dumb"
+        self.directory = "."
+        self.completed = 0
+        self.discovered = 1
+
+    def _draw(self) -> None:
+        text = (
+            f"  {self.completed}/{self.discovered} discovered directories read"
+            f" · {self.directory}/"
+        )
+        if self.terminal:
+            width = max(1, shutil.get_terminal_size().columns - 1)
+            print(f"\r\033[2K{text[:width]}", end="", file=self.output, flush=True)
+        else:
+            print(text, file=self.output, flush=True)
+
+    def __enter__(self) -> DirectoryReadProgress:
+        return self
+
+    def reading(self, directory: PurePosixPath) -> None:
+        self.directory = directory.as_posix()
+        self._draw()
+
+    def counts(self, completed: int, discovered: int) -> None:
+        self.completed = completed
+        self.discovered = discovered
+        if self.terminal:
+            self._draw()
+
+    def message(self, message: str) -> None:
+        if self.terminal:
+            print("\r\033[2K", end="", file=self.output)
+        print(f"  {message}", file=self.output, flush=True)
+        if self.terminal:
+            self._draw()
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        if self.terminal:
+            print("\r\033[2K", end="", file=self.output, flush=True)
+        if exc_type is None:
+            print(
+                f"  Read {self.completed}/{self.discovered} directories.",
+                file=self.output, flush=True,
+            )

hlsync/rules.py

@@ -0,0 +1,276 @@
+from __future__ import annotations
+
+import glob
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any, Literal
+
+from hlsync.gitignore import GitIgnores
+from hlsync.selection import FileSelector, SelectionError
+
+RuleAction = Literal["include", "exclude"]
+RuleTarget = Literal["local", "remote"]
+
+
+class RuleError(ValueError):
+    """Raised when an HLSync synchronization rule is unsafe or malformed."""
+
+
+def _relative_base(profile_root: Path, current_directory: Path) -> PurePosixPath:
+    try:
+        relative = current_directory.relative_to(profile_root)
+    except ValueError:
+        return PurePosixPath()
+    return PurePosixPath(*relative.parts)
+
+
+def expand_path_operands(
+    values: tuple[str, ...],
+    *,
+    profile_root: Path,
+    current_directory: Path,
+) -> tuple[str, ...]:
+    """Expand wildcard operands to the local paths they currently match."""
+    try:
+        current_directory.relative_to(profile_root)
+    except ValueError:
+        expansion_root = profile_root
+    else:
+        expansion_root = current_directory
+    expanded: list[str] = []
+    for value in values:
+        supplied = PurePosixPath(value)
+        if supplied.is_absolute() or ".." in supplied.parts:
+            raise RuleError(
+                "rule paths must be relative to the profile or current directory"
+            )
+        if "*" not in value:
+            expanded.append(value)
+            continue
+        matches = glob.glob(
+            value,
+            root_dir=expansion_root,
+            recursive=True,
+            include_hidden=False,
+        )
+        if not matches:
+            raise RuleError(f"path expression '{value}' matched no local paths")
+        expanded.extend(PurePosixPath(match).as_posix() for match in sorted(matches))
+    return tuple(dict.fromkeys(expanded))
+
+
+def patterns_from_operands(
+    values: tuple[str, ...],
+    *,
+    profile_root: Path,
+    current_directory: Path,
+) -> tuple[str, ...]:
+    """Translate CLI operands into profile-rooted HLSync patterns."""
+    base = _relative_base(profile_root, current_directory)
+    normalized: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            raise RuleError("rule patterns must be non-empty strings")
+        pattern = value.strip()
+        supplied = PurePosixPath(pattern)
+        if supplied.is_absolute() or ".." in supplied.parts:
+            raise RuleError(
+                "rule patterns must be relative to the profile or current directory"
+            )
+        rooted = base / supplied
+        local_path = profile_root.joinpath(*rooted.parts)
+        is_directory = pattern.endswith("/") or (
+            "*" not in pattern and local_path.is_dir() and not local_path.is_symlink()
+        )
+        if is_directory:
+            rooted = rooted / "**"
+        candidate = rooted.as_posix()
+        try:
+            candidate = FileSelector(candidate).pattern
+        except SelectionError as error:
+            raise RuleError(str(error)) from error
+        normalized.append(candidate)
+    return tuple(normalized)
+
+
+def patterns_from_global_operands(
+    values: tuple[str, ...],
+    *,
+    trailing_slash_tree: bool = True,
+) -> tuple[str, ...]:
+    """Normalize reusable patterns rooted at every profile's local root."""
+    normalized: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            raise RuleError("global rule patterns must be non-empty strings")
+        pattern = value.strip()
+        supplied = PurePosixPath(
+            pattern if trailing_slash_tree else pattern.rstrip("/")
+        )
+        if supplied.is_absolute() or ".." in supplied.parts:
+            raise RuleError("global rules must be relative to the profile root")
+        if trailing_slash_tree and pattern.endswith("/"):
+            supplied = supplied / "**"
+        try:
+            normalized.append(FileSelector(supplied.as_posix()).pattern)
+        except SelectionError as error:
+            raise RuleError(str(error)) from error
+    return tuple(normalized)
+
+
+def patterns_from_remote_operands(
+    values: tuple[str, ...],
+    *,
+    profile_root: Path,
+    current_directory: Path,
+) -> tuple[str, ...]:
+    """Normalize declarative remote paths without consulting either filesystem."""
+    base = _relative_base(profile_root, current_directory)
+    normalized: list[str] = []
+    for value in values:
+        if not isinstance(value, str) or not value.strip():
+            raise RuleError("remote rule paths must be non-empty strings")
+        pattern = value.strip()
+        supplied = PurePosixPath(pattern.rstrip("/"))
+        if supplied.is_absolute() or not supplied.parts or ".." in supplied.parts:
+            raise RuleError(
+                "remote rule paths must be relative to the profile or current directory"
+            )
+        try:
+            normalized.append(FileSelector((base / supplied).as_posix()).pattern)
+        except SelectionError as error:
+            raise RuleError(str(error)) from error
+    return tuple(normalized)
+
+
+@dataclass(frozen=True)
+class SyncRule:
+    id: int
+    action: RuleAction
+    pattern: str
+    target: RuleTarget = "local"
+
+    def __post_init__(self) -> None:
+        if isinstance(self.id, bool) or not isinstance(self.id, int) or self.id < 1:
+            raise RuleError("rule id must be a positive integer")
+        if not isinstance(self.action, str) or self.action not in {
+            "include",
+            "exclude",
+        }:
+            raise RuleError("rule action must be 'include' or 'exclude'")
+        if not isinstance(self.pattern, str):
+            raise RuleError("rule pattern must be a string")
+        if self.target not in {"local", "remote"}:
+            raise RuleError("rule target must be 'local' or 'remote'")
+        try:
+            normalized = FileSelector(self.pattern).pattern
+        except SelectionError as error:
+            raise RuleError(str(error)) from error
+        object.__setattr__(self, "pattern", normalized)
+        object.__setattr__(self, "_selector", FileSelector(normalized))
+
+    def matches(self, profile_relative_path: str) -> bool:
+        return self._selector.matches(profile_relative_path)
+
+    def may_match_descendant(self, profile_relative_directory: str) -> bool:
+        return self._selector.may_match_descendant(profile_relative_directory)
+
+    def to_dict(self) -> dict[str, Any]:
+        value = {
+            "id": self.id,
+            "action": self.action,
+            "pattern": self.pattern,
+        }
+        if self.target == "remote":
+            value["target"] = self.target
+        return value
+
+    @classmethod
+    def from_dict(cls, value: Any) -> SyncRule:
+        required = {"id", "action", "pattern"}
+        if (
+            not isinstance(value, dict)
+            or not required.issubset(value)
+            or set(value) - required != ({"target"} if "target" in value else set())
+        ):
+            raise RuleError(
+                "each rule must contain id, action, and pattern, with only an "
+                "optional target"
+            )
+        return cls(
+            id=value["id"],
+            action=value["action"],
+            pattern=value["pattern"],
+            target=value.get("target", "local"),
+        )
+
+
+@dataclass(frozen=True)
+class RuleSet:
+    rules: tuple[SyncRule, ...] = ()
+    gitignores: GitIgnores | None = None
+
+    def __post_init__(self) -> None:
+        if any(not isinstance(rule, SyncRule) for rule in self.rules):
+            raise RuleError("rules must contain structured synchronization rules")
+        ids = tuple(rule.id for rule in self.rules)
+        if len(set(ids)) != len(ids):
+            raise RuleError("rule ids must be unique")
+        if ids != tuple(sorted(ids)):
+            raise RuleError("rules must be ordered by increasing id")
+
+    @classmethod
+    def layered(
+        cls, *layers: tuple[SyncRule, ...], gitignores: GitIgnores | None = None
+    ) -> RuleSet:
+        return cls(
+            tuple(
+                SyncRule(index, rule.action, rule.pattern, rule.target)
+                for index, rule in enumerate(
+                    (rule for layer in layers for rule in layer),
+                    start=1,
+                )
+            ),
+            gitignores=gitignores,
+        )
+
+    def excludes(
+        self,
+        relative_path: str,
+        *,
+        target: RuleTarget = "local",
+        is_directory: bool = False,
+    ) -> bool:
+        profile_relative_path = relative_path
+        path = PurePosixPath(profile_relative_path)
+        if path.is_absolute() or not path.parts or ".." in path.parts:
+            raise RuleError("evaluated paths must be non-empty profile-relative paths")
+        candidate = path.as_posix()
+        winner = next(
+            (
+                rule
+                for rule in reversed(self.rules)
+                if rule.target == target and rule.matches(candidate)
+            ),
+            None,
+        )
+        if winner is not None:
+            return winner.action == "exclude"
+        return bool(
+            target == "local"
+            and self.gitignores is not None
+            and self.gitignores.excludes(candidate, is_directory=is_directory)
+        )
+
+    def may_include_descendant(
+        self,
+        relative_directory: str,
+        *,
+        target: RuleTarget = "local",
+    ) -> bool:
+        return any(
+            rule.action == "include"
+            and rule.target == target
+            and rule.may_match_descendant(relative_directory)
+            for rule in self.rules
+        )

hlsync/selection.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from functools import cache
+from pathlib import Path, PurePosixPath
+
+
+class SelectionError(ValueError):
+    """Raised when a file selector is unsafe or matches no paths."""
+
+
+@cache
+def _matches_parts(pattern: tuple[str, ...], path: tuple[str, ...]) -> bool:
+    if not pattern:
+        return not path
+    head, *tail = pattern
+    if head == "**":
+        return _matches_parts(tuple(tail), path) or (
+            bool(path) and _matches_parts(pattern, path[1:])
+        )
+    return bool(path) and fnmatchcase(path[0], head) and _matches_parts(
+        tuple(tail), path[1:]
+    )
+
+
+@cache
+def _can_match_descendant(
+    pattern: tuple[str, ...],
+    directory: tuple[str, ...],
+) -> bool:
+    if not directory:
+        return bool(pattern)
+    if not pattern:
+        return False
+    head, *tail = pattern
+    if head == "**":
+        return _can_match_descendant(tuple(tail), directory) or (
+            _can_match_descendant(pattern, directory[1:])
+        )
+    return fnmatchcase(directory[0], head) and _can_match_descendant(
+        tuple(tail), directory[1:]
+    )
+
+
+@dataclass(frozen=True)
+class FileSelector:
+    pattern: str
+
+    def __post_init__(self) -> None:
+        path = PurePosixPath(self.pattern)
+        if path.is_absolute() or not path.parts or ".." in path.parts:
+            raise SelectionError(
+                "file selector must be a non-empty relative path without '..'"
+            )
+        if any(character in self.pattern for character in "?["):
+            raise SelectionError("file selector supports only '*' and '**' wildcards")
+        if any("**" in part and part != "**" for part in path.parts):
+            raise SelectionError("'**' must occupy an entire path segment")
+        object.__setattr__(self, "pattern", path.as_posix())
+
+    @classmethod
+    def from_argument(
+        cls,
+        value: str,
+        *,
+        profile_root: Path,
+        current_directory: Path,
+    ) -> FileSelector:
+        if not value:
+            raise SelectionError("file selector cannot be empty")
+        try:
+            current_relative = current_directory.relative_to(profile_root)
+        except ValueError:
+            base = PurePosixPath()
+        else:
+            base = PurePosixPath(*current_relative.parts)
+        supplied = PurePosixPath(value)
+        if supplied.is_absolute() or ".." in supplied.parts:
+            raise SelectionError(
+                "file selector must be relative to the profile or current directory"
+            )
+        return cls((base / supplied).as_posix())
+
+    def matches(self, profile_relative_path: str) -> bool:
+        path = PurePosixPath(profile_relative_path)
+        if path.is_absolute() or ".." in path.parts:
+            raise SelectionError("selected paths must remain inside the profile")
+        return _matches_parts(PurePosixPath(self.pattern).parts, path.parts)
+
+    def may_match_descendant(self, profile_relative_directory: str) -> bool:
+        directory = PurePosixPath(profile_relative_directory)
+        if directory.is_absolute() or ".." in directory.parts:
+            raise SelectionError("selected paths must remain inside the profile")
+        return _can_match_descendant(
+            PurePosixPath(self.pattern).parts,
+            directory.parts,
+        )
+
+
+@dataclass(frozen=True)
+class FileSelectorSet:
+    selectors: tuple[FileSelector, ...]
+
+    def __post_init__(self) -> None:
+        if not self.selectors:
+            raise SelectionError("file selector set cannot be empty")
+        unique = {selector.pattern: selector for selector in self.selectors}
+        object.__setattr__(
+            self,
+            "selectors",
+            tuple(unique[pattern] for pattern in sorted(unique)),
+        )
+
+    @property
+    def pattern(self) -> str:
+        return ", ".join(selector.pattern for selector in self.selectors)
+
+    def matches(self, profile_relative_path: str) -> bool:
+        return any(
+            selector.matches(profile_relative_path) for selector in self.selectors
+        )
+
+    def may_match_descendant(self, profile_relative_directory: str) -> bool:
+        return any(
+            selector.may_match_descendant(profile_relative_directory)
+            for selector in self.selectors
+        )
+
+
+FileSelection = FileSelector | FileSelectorSet
+
+
+@dataclass(frozen=True)
+class DirectoryContentsSelection:
+    """Select operands while treating matched directories as containers."""
+
+    traversal: FileSelection
+    containers: FileSelection | None = None
+
+    @property
+    def pattern(self) -> str:
+        return self.traversal.pattern
+
+    def matches(self, profile_relative_path: str) -> bool:
+        return self.traversal.matches(profile_relative_path)
+
+    def may_match_descendant(self, profile_relative_directory: str) -> bool:
+        return self.traversal.may_match_descendant(profile_relative_directory)
+
+    def includes_result(self, profile_relative_path: str, *, directory: bool) -> bool:
+        return not (
+            directory
+            and self.containers is not None
+            and self.containers.matches(profile_relative_path)
+        )
+
+
+FileSelection = FileSelector | FileSelectorSet | DirectoryContentsSelection

hlsync/snapshot.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+from hlsync.rules import RuleSet
+from hlsync.selection import FileSelection
+
+EntryKind = Literal["directory", "file", "symlink"]
+
+
+class SnapshotError(RuntimeError):
+    """Raised when a deterministic tree snapshot cannot be produced."""
+
+
+@dataclass(frozen=True)
+class TreeEntry:
+    path: str
+    kind: EntryKind
+    size: int | None = None
+    modified_ns: int | None = None
+    timestamp_precision_ns: int | None = None
+    excluded: bool = False
+    remote_excluded: bool = False
+
+    def __post_init__(self) -> None:
+        path = PurePosixPath(self.path)
+        if path.is_absolute() or not path.parts or ".." in path.parts:
+            raise SnapshotError(f"snapshot path must be relative: {self.path!r}")
+        object.__setattr__(self, "path", path.as_posix())
+        if not isinstance(self.excluded, bool):
+            raise SnapshotError("snapshot exclusion state must be a boolean")
+        if not isinstance(self.remote_excluded, bool):
+            raise SnapshotError("remote exclusion state must be a boolean")
+        metadata = (self.size, self.modified_ns, self.timestamp_precision_ns)
+        if self.kind == "file":
+            if any(value is None for value in metadata):
+                raise SnapshotError(
+                    f"file snapshot metadata is incomplete: {self.path!r}"
+                )
+            if self.size is not None and self.size < 0:
+                raise SnapshotError(f"file size cannot be negative: {self.path!r}")
+            if (
+                self.timestamp_precision_ns is not None
+                and self.timestamp_precision_ns <= 0
+            ):
+                raise SnapshotError(
+                    f"timestamp precision must be positive: {self.path!r}"
+                )
+        elif any(value is not None for value in metadata):
+            raise SnapshotError(
+                f"{self.kind} snapshot entry cannot have file metadata: "
+                f"{self.path!r}"
+            )
+
+
+@dataclass(frozen=True)
+class TreeSnapshot:
+    entries: tuple[TreeEntry, ...] = ()
+
+    def __post_init__(self) -> None:
+        ordered = tuple(sorted(self.entries, key=lambda entry: entry.path))
+        if len({entry.path for entry in ordered}) != len(ordered):
+            raise SnapshotError("snapshot contains duplicate paths")
+        object.__setattr__(self, "entries", ordered)
+
+
+def snapshot_local(
+    root: Path,
+    rules: RuleSet,
+    selector: FileSelection | None = None,
+    *,
+    include_excluded: bool = False,
+    traverse_excluded: bool = False,
+    respect_remote_boundaries: bool = False,
+    directory_progress: Callable[[PurePosixPath], None] | None = None,
+) -> TreeSnapshot:
+    if not root.is_dir():
+        raise SnapshotError(f"local root is not an accessible directory: {root}")
+    entries: list[TreeEntry] = []
+
+    def walk(relative_directory: PurePosixPath) -> None:
+        if directory_progress is not None:
+            directory_progress(relative_directory)
+        listing = list_local_directory(root, relative_directory, rules)
+        descendants: list[PurePosixPath] = []
+        for entry in listing.entries:
+            relative_path = entry.path
+            relative = PurePosixPath(relative_path)
+            kind = entry.kind
+            excluded = entry.excluded
+            visible = not excluded or include_excluded
+            selected = visible and (
+                selector is None or selector.matches(relative_path)
+            )
+            if selected:
+                entries.append(entry)
+            selection_may_descend = (
+                selector is None or selector.may_match_descendant(relative_path)
+            )
+            exclusion_may_descend = (
+                traverse_excluded
+                or not excluded
+                or rules.may_include_descendant(relative_path)
+            )
+            remote_may_descend = (
+                not respect_remote_boundaries or not entry.remote_excluded
+            )
+            if (
+                kind == "directory"
+                and selection_may_descend
+                and exclusion_may_descend
+                and remote_may_descend
+            ):
+                descendants.append(relative)
+        for descendant in descendants:
+            walk(descendant)
+
+    walk(PurePosixPath())
+    return TreeSnapshot(tuple(entries))
+
+
+def list_local_directory(
+    root: Path,
+    relative_directory: PurePosixPath,
+    rules: RuleSet,
+) -> TreeSnapshot:
+    """Inspect one directory and return all immediate children."""
+    directory = root / relative_directory
+    try:
+        with os.scandir(directory) as iterator:
+            children = sorted(iterator, key=lambda entry: entry.name)
+    except OSError as error:
+        raise SnapshotError(
+            f"could not read local directory '{directory}': {error}"
+        ) from error
+
+    entries: list[TreeEntry] = []
+    for child in children:
+        relative = relative_directory / child.name
+        relative_path = relative.as_posix()
+        try:
+            if child.is_symlink():
+                kind: EntryKind = "symlink"
+            elif child.is_dir(follow_symlinks=False):
+                kind = "directory"
+            elif child.is_file(follow_symlinks=False):
+                kind = "file"
+            else:
+                raise SnapshotError(
+                    f"unsupported local entry type: {directory / child.name}"
+                )
+        except OSError as error:
+            raise SnapshotError(
+                f"could not inspect local path '{directory / child.name}': {error}"
+            ) from error
+
+        excluded = rules.excludes(relative_path, is_directory=kind == "directory")
+        remote_excluded = rules.excludes(
+            relative_path, target="remote", is_directory=kind == "directory"
+        )
+        if kind == "file":
+            try:
+                stat = child.stat(follow_symlinks=False)
+            except OSError as error:
+                raise SnapshotError(
+                    f"could not read local file metadata '{directory / child.name}': "
+                    f"{error}"
+                ) from error
+            entries.append(
+                TreeEntry(
+                    relative_path,
+                    kind,
+                    size=stat.st_size,
+                    modified_ns=stat.st_mtime_ns,
+                    timestamp_precision_ns=1,
+                    excluded=excluded,
+                    remote_excluded=remote_excluded,
+                )
+            )
+        else:
+            entries.append(
+                TreeEntry(
+                    relative_path,
+                    kind,
+                    excluded=excluded,
+                    remote_excluded=remote_excluded,
+                )
+            )
+    return TreeSnapshot(tuple(entries))