hlsync 1.16.2__py3-none-any.whl

hlsync/storage.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def write_json_atomic(path: Path, document: dict[str, Any]) -> None:
+    """Atomically replace a JSON configuration document."""
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    payload = json.dumps(document, indent=2, sort_keys=True) + "\n"
+    descriptor, temporary_name = tempfile.mkstemp(
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        text=True,
+    )
+    temporary_path = Path(temporary_name)
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            os.chmod(temporary_path, 0o600)
+            handle.write(payload)
+            handle.flush()
+            os.fsync(handle.fileno())
+        temporary_path.replace(path)
+    except BaseException:
+        temporary_path.unlink(missing_ok=True)
+        raise

hlsync/transfer.py

@@ -0,0 +1,465 @@
+from __future__ import annotations
+
+import os
+import stat
+import tempfile
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+from hlsync.comparison import ComparisonEntry, ComparisonPlan
+from hlsync.snapshot import EntryKind, TreeEntry, TreeSnapshot
+from hlsync.transport import PathOperationError, RemoteTransport
+
+
+class TransferError(RuntimeError):
+    """Raised when a transfer plan cannot be executed safely."""
+
+
+@dataclass(frozen=True)
+class TransferIssue:
+    path: str
+    status: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class TransferOperation:
+    action: Literal["add", "update", "delete", "create"]
+    path: str
+    kind: EntryKind
+
+
+TransferProgress = Callable[[TransferOperation | TransferIssue], None]
+
+
+def _record_issue(
+    issues: list[TransferIssue],
+    issue: TransferIssue,
+    progress: TransferProgress | None,
+) -> None:
+    issues.append(issue)
+    if progress is not None:
+        progress(issue)
+
+
+@dataclass(frozen=True)
+class TransferResult:
+    plan: ComparisonPlan
+    completed_paths: frozenset[str]
+    issues: tuple[TransferIssue, ...] = ()
+
+    @property
+    def changed_count(self) -> int:
+        return len(self.completed_paths)
+
+    @property
+    def unchanged_file_count(self) -> int:
+        return sum(
+            entry.action == "unchanged" and entry.local_kind == "file"
+            for entry in self.plan.entries
+        )
+
+    @property
+    def failed_count(self) -> int:
+        return sum(issue.status == "failed" for issue in self.issues)
+
+    @property
+    def skipped_count(self) -> int:
+        return sum(issue.status == "skipped" for issue in self.issues)
+
+    @property
+    def succeeded(self) -> bool:
+        return not self.issues
+
+
+def _entry_map(snapshot: TreeSnapshot) -> dict[str, TreeEntry]:
+    return {entry.path: entry for entry in snapshot.entries}
+
+
+def _preflight(plan: ComparisonPlan, local_root: Path, local: TreeSnapshot) -> None:
+    conflicts = [entry for entry in plan.entries if entry.action == "conflict"]
+    if conflicts:
+        message = "transfer has unresolved conflict(s): " + ", ".join(
+            entry.path for entry in conflicts
+        )
+        if any(entry.state == "destination-newer" for entry in conflicts):
+            message += (
+                "; destination files not older than their source are kept"
+                " unless --force is given"
+            )
+        raise TransferError(message)
+    local_entries = _entry_map(local)
+    for entry in plan.entries:
+        if entry.action not in {"upload", "replace-remote", "replace-local"}:
+            continue
+        expected = local_entries[entry.path]
+        path = local_root / Path(*PurePosixPath(entry.path).parts)
+        try:
+            current = path.stat(follow_symlinks=False)
+        except OSError as error:
+            raise TransferError(
+                f"could not validate local source '{path}': {error}"
+            ) from error
+        if not stat.S_ISREG(current.st_mode):
+            raise TransferError(f"local source is no longer a file: {path}")
+        if (
+            current.st_size != expected.size
+            or current.st_mtime_ns != expected.modified_ns
+        ):
+            raise TransferError(f"local source changed after snapshot: {path}")
+
+
+def _remote_directories_to_create(
+    plan: ComparisonPlan,
+    remote: TreeSnapshot,
+) -> tuple[str, ...]:
+    existing = {
+        entry.path for entry in remote.entries if entry.kind == "directory"
+    }
+    required = {
+        entry.path
+        for entry in plan.entries
+        if entry.action == "create-remote"
+    }
+    for entry in plan.entries:
+        if entry.action not in {"upload", "replace-remote"}:
+            continue
+        parent = PurePosixPath(entry.path).parent
+        while parent.parts:
+            if parent.as_posix() not in existing:
+                required.add(parent.as_posix())
+            parent = parent.parent
+    return tuple(
+        sorted(
+            required,
+            key=lambda path: (len(PurePosixPath(path).parts), path),
+        )
+    )
+
+
+def planned_transfer_operations(
+    plan: ComparisonPlan,
+    remote: TreeSnapshot,
+) -> tuple[TransferOperation, ...]:
+    """Return successful-path operations in their execution order."""
+    if plan.direction == "pull":
+        return tuple(
+            TransferOperation("update", entry.path, "file")
+            for entry in plan.entries
+            if entry.action == "replace-local"
+        )
+    creations = tuple(
+        TransferOperation("create", path, "directory")
+        for path in _remote_directories_to_create(plan, remote)
+    )
+    uploads = tuple(
+        TransferOperation(
+            "add" if entry.action == "upload" else "update",
+            entry.path,
+            "file",
+        )
+        for entry in plan.entries
+        if entry.action in {"upload", "replace-remote"}
+    )
+    deletions = tuple(
+        entry for entry in plan.entries if entry.action == "delete-remote"
+    )
+    files = sorted(
+        (entry for entry in deletions if entry.remote_kind != "directory"),
+        key=lambda entry: entry.path,
+    )
+    directories = sorted(
+        (entry for entry in deletions if entry.remote_kind == "directory"),
+        key=lambda entry: (-len(PurePosixPath(entry.path).parts), entry.path),
+    )
+    deletes = tuple(
+        TransferOperation("delete", entry.path, entry.remote_kind or "file")
+        for entry in (*files, *directories)
+    )
+    return creations + uploads + deletes
+
+
+def _push_files(
+    plan: ComparisonPlan,
+    local_root: Path,
+    local: TreeSnapshot,
+    transport: RemoteTransport,
+    progress: TransferProgress | None,
+    operations: tuple[TransferOperation, ...],
+    dry_run: bool,
+    byte_progress: Callable[[int], None] | None,
+) -> tuple[set[str], list[TransferIssue]]:
+    local_entries = _entry_map(local)
+    completed: set[str] = set()
+    issues: list[TransferIssue] = []
+    unavailable_directories: set[str] = set()
+    planned_creations = {
+        entry.path
+        for entry in plan.entries
+        if entry.action == "create-remote"
+    }
+
+    def unavailable_parent(path: str) -> str | None:
+        candidate = PurePosixPath(path)
+        return next(
+            (
+                directory
+                for directory in sorted(unavailable_directories)
+                if PurePosixPath(directory) == candidate
+                or PurePosixPath(directory) in candidate.parents
+            ),
+            None,
+        )
+
+    for operation in (item for item in operations if item.action == "create"):
+        directory = operation.path
+        blocked_by = unavailable_parent(directory)
+        if blocked_by is not None:
+            unavailable_directories.add(directory)
+            if directory in planned_creations:
+                _record_issue(
+                    issues,
+                    TransferIssue(
+                        directory,
+                        "skipped",
+                        f"parent directory '{blocked_by}' could not be created",
+                    ),
+                    progress,
+                )
+            continue
+        try:
+            if not dry_run:
+                transport.make_directory(directory)
+        except PathOperationError as error:
+            unavailable_directories.add(directory)
+            _record_issue(
+                issues, TransferIssue(directory, "failed", str(error)), progress
+            )
+            continue
+        if progress is not None:
+            progress(operation)
+        if directory in planned_creations:
+            completed.add(directory)
+    for operation in (
+        item for item in operations if item.action in {"add", "update"}
+    ):
+        blocked_by = unavailable_parent(operation.path)
+        if blocked_by is not None:
+            _record_issue(
+                issues,
+                TransferIssue(
+                    operation.path,
+                    "skipped",
+                    f"parent directory '{blocked_by}' is unavailable",
+                ),
+                progress,
+            )
+            continue
+        metadata = local_entries[operation.path]
+        if metadata.size is None or metadata.modified_ns is None:
+            raise TransferError(
+                f"local file metadata is incomplete: {operation.path}"
+            )
+        path = local_root / Path(*PurePosixPath(operation.path).parts)
+        try:
+            if not dry_run:
+                with path.open("rb") as source:
+                    transport.upload_file(
+                        source,
+                        operation.path,
+                        size=metadata.size,
+                        modified_ns=metadata.modified_ns,
+                        replace=operation.action == "update",
+                        byte_progress=byte_progress,
+                    )
+        except PathOperationError as error:
+            _record_issue(
+                issues, TransferIssue(operation.path, "failed", str(error)), progress
+            )
+            continue
+        except OSError as error:
+            _record_issue(
+                issues,
+                TransferIssue(
+                    operation.path,
+                    "failed",
+                    f"could not read local file '{path}': {error}",
+                ),
+                progress,
+            )
+            continue
+        completed.add(operation.path)
+        if progress is not None:
+            progress(operation)
+    return completed, issues
+
+
+def _replace_local_file(
+    root: Path,
+    comparison: ComparisonEntry,
+    local_entry: TreeEntry,
+    remote_entry: TreeEntry,
+    transport: RemoteTransport,
+) -> None:
+    destination = root / Path(*PurePosixPath(comparison.path).parts)
+    try:
+        mode = stat.S_IMODE(destination.stat(follow_symlinks=False).st_mode)
+    except OSError as error:
+        raise TransferError(
+            f"could not inspect local destination '{destination}': {error}"
+        ) from error
+    descriptor, temporary_name = tempfile.mkstemp(
+        dir=destination.parent,
+        prefix=f".{destination.name}.hlsync-pull-",
+    )
+    temporary = Path(temporary_name)
+    try:
+        with os.fdopen(descriptor, "wb") as target:
+            transport.download_file(comparison.path, target)
+            target.flush()
+            os.fsync(target.fileno())
+        actual_size = temporary.stat().st_size
+        if actual_size != remote_entry.size:
+            raise TransferError(
+                f"download size mismatch for '{comparison.path}': expected "
+                f"{remote_entry.size}, got {actual_size}"
+            )
+        os.chmod(temporary, mode)
+        if remote_entry.modified_ns is None:
+            raise TransferError(
+                f"remote file timestamp is missing: {comparison.path}"
+            )
+        os.utime(
+            temporary,
+            ns=(remote_entry.modified_ns, remote_entry.modified_ns),
+        )
+        current = destination.stat(follow_symlinks=False)
+        if (
+            not stat.S_ISREG(current.st_mode)
+            or current.st_size != local_entry.size
+            or current.st_mtime_ns != local_entry.modified_ns
+        ):
+            raise TransferError(
+                f"local destination changed during download: {destination}"
+            )
+        temporary.replace(destination)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def _pull_files(
+    plan: ComparisonPlan,
+    local_root: Path,
+    local: TreeSnapshot,
+    remote: TreeSnapshot,
+    transport: RemoteTransport,
+    progress: TransferProgress | None,
+    operations: tuple[TransferOperation, ...],
+) -> set[str]:
+    completed: set[str] = set()
+    local_entries = _entry_map(local)
+    remote_entries = _entry_map(remote)
+    entries = {entry.path: entry for entry in plan.entries}
+    for operation in operations:
+        entry = entries[operation.path]
+        _replace_local_file(
+            local_root,
+            entry,
+            local_entries[operation.path],
+            remote_entries[operation.path],
+            transport,
+        )
+        completed.add(operation.path)
+        if progress is not None:
+            progress(operation)
+    return completed
+
+
+def _delete_remote(
+    transport: RemoteTransport,
+    progress: TransferProgress | None,
+    operations: tuple[TransferOperation, ...],
+    dry_run: bool,
+) -> tuple[set[str], list[TransferIssue]]:
+    completed: set[str] = set()
+    issues: list[TransferIssue] = []
+    for operation in (item for item in operations if item.action == "delete"):
+        try:
+            if not dry_run:
+                transport.delete_path(
+                    operation.path,
+                    is_directory=operation.kind == "directory",
+                )
+        except PathOperationError as error:
+            _record_issue(
+                issues, TransferIssue(operation.path, "failed", str(error)), progress
+            )
+            continue
+        completed.add(operation.path)
+        if progress is not None:
+            progress(operation)
+    return completed, issues
+
+
+def execute_transfer(
+    plan: ComparisonPlan,
+    *,
+    local_root: Path,
+    local: TreeSnapshot,
+    remote: TreeSnapshot,
+    transport: RemoteTransport,
+    progress: TransferProgress | None = None,
+    dry_run: bool = False,
+    byte_progress: Callable[[int], None] | None = None,
+) -> TransferResult:
+    if dry_run and plan.direction != "push":
+        raise TransferError("dry run is supported only for push")
+    _preflight(plan, local_root, local)
+    operations = planned_transfer_operations(plan, remote)
+    if plan.direction == "push":
+        completed, issues = _push_files(
+            plan,
+            local_root,
+            local,
+            transport,
+            progress,
+            operations,
+            dry_run,
+            byte_progress,
+        )
+    else:
+        completed = _pull_files(
+            plan,
+            local_root,
+            local,
+            remote,
+            transport,
+            progress,
+            operations,
+        )
+        issues = []
+    if issues:
+        for entry in plan.entries:
+            if entry.action == "delete-remote":
+                _record_issue(
+                    issues,
+                    TransferIssue(
+                        entry.path,
+                        "skipped",
+                        "remote pruning was suppressed after another transfer failed",
+                    ),
+                    progress,
+                )
+    else:
+        deleted, deletion_issues = _delete_remote(
+            transport,
+            progress,
+            operations,
+            dry_run,
+        )
+        completed.update(deleted)
+        issues.extend(deletion_issues)
+    return TransferResult(plan, frozenset(completed), tuple(issues))