batchvault 0.1.0__py3-none-any.whl

batchvault/signing.py

@@ -0,0 +1,152 @@
+"""Ed25519 signatures for manifests (needs ``batchvault[crypto]``).
+
+The signature is stored next to the manifest as ``<manifest>.sig`` and covers
+the exact bytes of the manifest file. Anyone with the public key can verify it;
+only the holder of the private key can create it.
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+from pathlib import Path
+
+from batchvault.crypto import require_cryptography
+from batchvault.errors import InputError, SignatureError
+
+SIGNATURE_SUFFIX = ".sig"
+
+
+def signature_path(manifest_path: Path) -> Path:
+    return manifest_path.with_name(manifest_path.name + SIGNATURE_SUFFIX)
+
+
+def _write_private(path: Path, data: bytes) -> None:
+    # Create the file readable by its owner only, before any key is written.
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "wb") as handle:
+        handle.write(data)
+
+
+def generate_keypair(
+    private_path: Path,
+    public_path: Path,
+    *,
+    password: str | None = None,
+    overwrite: bool = False,
+) -> None:
+    """Create an Ed25519 key pair as PEM files.
+
+    The private key is encrypted with ``password`` when one is given.
+    """
+    require_cryptography()
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+    for path in (private_path, public_path):
+        if path.exists() and not overwrite:
+            raise InputError(f"{path} already exists. Use --force to replace it.")
+
+    key = Ed25519PrivateKey.generate()
+    encryption = (
+        serialization.BestAvailableEncryption(password.encode("utf-8"))
+        if password
+        else serialization.NoEncryption()
+    )
+    _write_private(
+        private_path,
+        key.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.PKCS8,
+            encryption,
+        ),
+    )
+    public_path.write_bytes(
+        key.public_key().public_bytes(
+            serialization.Encoding.PEM,
+            serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+    )
+
+
+def load_private_key(path: Path, password: str | None = None):
+    require_cryptography()
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+    try:
+        key = serialization.load_pem_private_key(
+            path.read_bytes(), password.encode("utf-8") if password else None
+        )
+    except OSError as exc:
+        raise SignatureError(f"Cannot read private key {path}: {exc}") from exc
+    except (ValueError, TypeError) as exc:
+        raise SignatureError(
+            f"Cannot load private key {path} (wrong password or not a key): {exc}"
+        ) from exc
+
+    if not isinstance(key, Ed25519PrivateKey):
+        raise SignatureError(f"{path} is not an Ed25519 private key.")
+    return key
+
+
+def load_public_key(path: Path):
+    require_cryptography()
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
+
+    try:
+        key = serialization.load_pem_public_key(path.read_bytes())
+    except OSError as exc:
+        raise SignatureError(f"Cannot read public key {path}: {exc}") from exc
+    except (ValueError, TypeError) as exc:
+        raise SignatureError(f"Cannot load public key {path}: {exc}") from exc
+
+    if not isinstance(key, Ed25519PublicKey):
+        raise SignatureError(f"{path} is not an Ed25519 public key.")
+    return key
+
+
+def sign_manifest(manifest_path: Path, private_key) -> Path:
+    """Sign the manifest with a loaded private key; return the signature file."""
+    signature = private_key.sign(manifest_path.read_bytes())
+    target = signature_path(manifest_path)
+    target.write_text(base64.b64encode(signature).decode("ascii") + "\n")
+    return target
+
+
+def verify_manifest_bytes(
+    data: bytes, manifest_path: Path, public_key_path: Path
+) -> None:
+    """Raise SignatureError unless ``data`` carries a valid signature.
+
+    ``data`` is the manifest content, already read from ``manifest_path``; the
+    signature file sits next to it. Checking the bytes that are then parsed
+    leaves no gap in which the file could be swapped.
+    """
+    from cryptography.exceptions import InvalidSignature
+
+    public_key = load_public_key(public_key_path)
+    sig_file = signature_path(manifest_path)
+    if not sig_file.is_file():
+        raise SignatureError(
+            f"{manifest_path.name} is not signed (no {sig_file.name})."
+        )
+
+    try:
+        signature = base64.b64decode(sig_file.read_text().strip(), validate=True)
+        public_key.verify(signature, data)
+    except (ValueError, InvalidSignature) as exc:
+        raise SignatureError(
+            f"Signature check failed for {manifest_path.name}: the manifest was "
+            "changed or signed with a different key."
+        ) from exc
+
+
+def verify_manifest_signature(manifest_path: Path, public_key_path: Path) -> None:
+    """Raise SignatureError unless the manifest has a valid signature."""
+    try:
+        data = manifest_path.read_bytes()
+    except OSError as exc:
+        raise SignatureError(f"Cannot read {manifest_path}: {exc}") from exc
+    verify_manifest_bytes(data, manifest_path, public_key_path)

batchvault/ui.py

@@ -0,0 +1,298 @@
+"""Rich terminal output. Only the command-line interface imports this module."""
+
+from __future__ import annotations
+
+import threading
+from pathlib import Path
+
+from rich.console import Console
+from rich.markup import escape
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    Progress,
+    ProgressColumn,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from batchvault.doctor import Check
+from batchvault.eta import EtaEstimator, format_duration
+from batchvault.models import InspectEntry, ProgressEvent, RunResult, Status
+from batchvault.options import format_size
+
+STATUS_STYLE = {
+    Status.SUCCESS: ("OK", "green"),
+    Status.SKIPPED: ("SKIPPED", "yellow"),
+    Status.FAILED: ("FAILED", "bold red"),
+    Status.PARTIAL: ("PARTIAL", "yellow"),
+}
+
+
+class EtaColumn(ProgressColumn):
+    """Shows the run's own ETA for tasks created with an ``estimator`` field."""
+
+    def render(self, task):
+        estimator = task.fields.get("estimator")
+        if estimator is None:
+            return Text("")
+
+        remaining = estimator.estimate(task.completed, task.total or 0)
+        if remaining is None:
+            return Text("ETA estimating…", style="dim")
+        return Text(f"ETA {format_duration(remaining)}", style="progress.remaining")
+
+
+def status_text(status: Status) -> Text:
+    label, style = STATUS_STYLE[status]
+    return Text(label, style=style)
+
+
+class RunProgress:
+    """Progress callback that draws an overall bar and one line per active file.
+
+    Use as a context manager and pass the instance as ``progress=``. With
+    ``enabled=False`` it does nothing.
+    """
+
+    def __init__(self, console: Console, label: str, enabled: bool = True):
+        self._enabled = enabled
+        self._label = label
+        self._lock = threading.Lock()
+        self._eta = EtaEstimator()
+        self._by_bytes = False
+        self._overall = None
+        self._tasks: dict[Path, int] = {}
+        self._progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold cyan]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+            EtaColumn(),
+            console=console,
+            transient=True,
+            disable=not enabled,
+        )
+
+    def __enter__(self):
+        if self._enabled:
+            self._progress.start()
+        return self
+
+    def __exit__(self, *exc_info):
+        self._progress.stop()
+
+    def pause(self) -> None:
+        """Stop drawing, e.g. before asking the user a question."""
+        self._progress.stop()
+
+    def __call__(self, event: ProgressEvent) -> None:
+        if not self._enabled:
+            return
+
+        with self._lock:
+            if event.kind == "plan":
+                self._by_bytes = event.size > 0
+                total = event.size if self._by_bytes else event.count
+                self._overall = self._progress.add_task(
+                    self._label, total=total, estimator=self._eta
+                )
+                self._eta.start()
+            elif event.kind == "start":
+                task = self._progress.add_task(escape(event.source.name), total=None)
+                self._tasks[event.source] = task
+            elif event.kind == "finish":
+                self._finish(event)
+
+    def _finish(self, event: ProgressEvent) -> None:
+        task = self._tasks.pop(event.source, None)
+        if task is not None:
+            self._progress.remove_task(task)
+        if self._overall is not None:
+            self._progress.advance(
+                self._overall, event.size if self._by_bytes else 1
+            )
+
+        label, style = STATUS_STYLE.get(event.status, ("", ""))
+        self._progress.console.print(
+            Text.assemble((f"{label:<8}", style), event.source.name)
+        )
+
+
+def print_run_info(console: Console, title: str, rows: list[tuple[str, str]]) -> None:
+    info = Table.grid(padding=(0, 2))
+    info.add_column(style="bold cyan")
+    info.add_column()
+    for key, value in rows:
+        info.add_row(key, Text(value))
+    console.print(Panel(info, title=title, border_style="cyan"))
+
+
+# Terms used on screen: a SOURCE is an original file; an ARCHIVE is the set of
+# volumes made from it. Each table names its columns accordingly.
+SOURCE_STYLE = "cyan"
+ARCHIVE_STYLE = "magenta"
+
+SUMMARY_TITLES = {
+    "pack": "Pack: source files -> archives",
+    "unpack": "Unpack: archives -> restored files",
+    "verify": "Verify: archives",
+}
+
+
+def _shown(folder: str, name: str) -> str:
+    return f"{folder}/{name}" if folder else name
+
+
+def _detail_text(entry) -> Text:
+    style = "dim" if entry.status == Status.SUCCESS else ""
+    text = Text(entry.message, style=style)
+    for warning in entry.warnings:
+        text.append(f"\n{warning}", style="yellow")
+    return text
+
+
+def print_summary(
+    console: Console, result: RunResult, title: str | None = None, kind: str = "pack"
+) -> None:
+    """Print one row per file. ``kind`` is pack, unpack or verify."""
+    table = Table(title=title or SUMMARY_TITLES.get(kind, ""), show_lines=False)
+
+    # The same columns as `inspect`: the original file, its archive, the outcome
+    # and an explanation.
+    table.add_column("Source", style=SOURCE_STYLE, overflow="fold")
+    table.add_column("Archive", style=ARCHIVE_STYLE, overflow="fold")
+    table.add_column("Status")
+    table.add_column("Details", overflow="fold")
+
+    for entry in result.entries:
+        # pack: the source is the original file and the archive its first volume.
+        # unpack and verify: the "source" is the archive's first volume.
+        if kind == "pack":
+            archive = entry.archive.name if entry.archive else "-"
+            original = entry.name or entry.source.name
+        else:
+            archive = entry.source.name
+            original = entry.name or "-"
+        table.add_row(
+            Text(_shown(entry.folder, original)),
+            Text(archive),
+            status_text(entry.status),
+            _detail_text(entry),
+        )
+    console.print(table)
+
+    for warning in result.warnings:
+        console.print(Text(f"Warning: {warning}", style="yellow"))
+
+    counts = ", ".join(
+        f"{result.count(status)} {label.lower()}"
+        for status, (label, _) in STATUS_STYLE.items()
+        if status != Status.PARTIAL and result.count(status)
+    )
+    console.print(f"[bold]Summary:[/bold] {escape(counts or 'nothing to do')}")
+    if result.manifest and any(e.status == Status.SUCCESS for e in result.entries):
+        console.print(f"Manifest: {escape(str(result.manifest))}")
+
+
+def _archive_status(entry: InspectEntry) -> Text:
+    if entry.complete is True:
+        return Text("OK", style="green")
+    if entry.complete is False:
+        return Text("MISSING", style="bold red")
+    return Text("-", style="dim")  # no manifest, so completeness is unknown
+
+
+def _inspect_details(entry: InspectEntry) -> Text:
+    notes = []
+    if entry.encrypted:
+        notes.append(("encrypted", "yellow"))
+    if entry.complete is False:
+        notes.append(("some volumes are missing", "bold red"))
+    if entry.complete is None:
+        notes.append(("no manifest: sizes and checks unknown", "dim"))
+
+    text = Text()
+    for index, (note, style) in enumerate(notes):
+        if index:
+            text.append(", ")
+        text.append(note, style=style)
+    return text
+
+
+def print_inspect(
+    console: Console, entries: list[InspectEntry], source: str | None = None
+) -> None:
+    """Describe archives with the same layout and colours as the result tables."""
+    original = sum(e.size or 0 for e in entries)
+    stored = sum(e.stored_size or 0 for e in entries)
+    missing = sum(1 for e in entries if e.complete is False)
+    encrypted = sum(1 for e in entries if e.encrypted)
+
+    rows = []
+    if source:
+        rows.append(("Archives", source))
+    rows.append(("Files", str(len(entries))))
+    if original:
+        sizes = f"{format_size(original)} -> {format_size(stored)}"
+        rows.append(("Size", f"{sizes} ({stored / original:.0%})"))
+    else:
+        rows.append(("Stored size", format_size(stored)))
+    if encrypted:
+        rows.append(("Encrypted", f"{encrypted} of {len(entries)}"))
+    print_run_info(console, "BatchVault inspect", rows)
+
+    table = Table(title="Inspect: archives", show_lines=False)
+    table.add_column("Source", style=SOURCE_STYLE, overflow="fold")
+    table.add_column("Archive", style=ARCHIVE_STYLE, overflow="fold")
+    table.add_column("Status")
+    table.add_column("Engine")
+    table.add_column("Format")
+    table.add_column("Level", justify="right")
+    table.add_column("Original", justify="right")
+    table.add_column("Stored", justify="right")
+    table.add_column("Ratio", justify="right")
+    table.add_column("Volumes", justify="right")
+    table.add_column("Details", overflow="fold")
+
+    for entry in entries:
+        ratio = f"{entry.ratio:.0%}" if entry.ratio is not None else "-"
+        table.add_row(
+            Text(_shown(entry.folder, entry.name)),
+            Text(entry.path.name),
+            _archive_status(entry),
+            entry.engine,
+            entry.format or "-",
+            "-" if entry.level is None else str(entry.level),
+            format_size(entry.size),
+            format_size(entry.stored_size),
+            ratio,
+            str(entry.volumes),
+            _inspect_details(entry),
+        )
+    console.print(table)
+
+    summary = f"{len(entries)} archive{'' if len(entries) == 1 else 's'}"
+    if missing:
+        summary += f", {missing} with missing volumes"
+    console.print(f"[bold]Summary:[/bold] {escape(summary)}")
+
+
+def print_checks(console: Console, checks: list[Check]) -> None:
+    table = Table(title="BatchVault doctor")
+    table.add_column("Check")
+    table.add_column("Result")
+    table.add_column("Details", overflow="fold")
+
+    for check in checks:
+        if check.ok:
+            result = Text("OK", style="green")
+        else:
+            result = Text("FAILED" if check.required else "MISSING", style="red")
+        table.add_row(check.name, result, Text(check.detail))
+    console.print(table)

batchvault/volumes.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import io
+import os
+import re
+import shutil
+import time
+import unicodedata
+from pathlib import Path
+
+from batchvault.errors import VolumeError
+from batchvault.security import restrict_permissions
+
+VOLUME_RE = re.compile(r"^(?P<base>.+\.(?:7z|bv))\.(?P<number>\d{3,})$", re.IGNORECASE)
+
+
+def split_volume_name(name: str) -> tuple[str, int] | None:
+    """Return ``(base, number)`` for ``intro.mp4.7z.003``, or None if not a volume."""
+    match = VOLUME_RE.match(name)
+    if not match:
+        return None
+    return match.group("base"), int(match.group("number"))
+
+
+def _same_name(a: str, b: str) -> bool:
+    """Compare names ignoring case and Unicode form (macOS decomposes accents)."""
+    return (
+        unicodedata.normalize("NFC", a).casefold()
+        == unicodedata.normalize("NFC", b).casefold()
+    )
+
+
+def existing_volumes(directory: Path, base: str) -> list[Path]:
+    """Volumes of ``base`` already present in ``directory``, in volume order."""
+    if not directory.is_dir():
+        return []
+
+    found = []
+    for path in directory.iterdir():
+        parsed = split_volume_name(path.name)
+        if path.is_file() and parsed and _same_name(parsed[0], base):
+            found.append((parsed[1], path))
+    return [path for _, path in sorted(found)]
+
+
+def find_volumes(path: Path) -> list[Path]:
+    """Find every volume of the set that ``path`` belongs to.
+
+    Any volume of the set may be given. Raises VolumeError if the path is not a
+    volume or if a volume inside the numbered range is missing. A missing
+    *trailing* volume cannot be detected without a manifest.
+    """
+    parsed = split_volume_name(path.name)
+    if parsed is None:
+        raise VolumeError(f"Not an archive volume: {path}")
+
+    volumes = existing_volumes(path.parent, parsed[0])
+    numbers = [split_volume_name(volume.name)[1] for volume in volumes]
+
+    missing = sorted(set(range(1, max(numbers) + 1)) - set(numbers))
+    if missing:
+        listed = ", ".join(f"{number:03d}" for number in missing)
+        raise VolumeError(
+            f"Missing volume(s) {listed} for {parsed[0]} in {path.parent}"
+        )
+
+    return volumes
+
+
+STAGING_PREFIX = ".batchvault-"
+
+
+def cleanup_staging(root: Path, max_age: float = 86400.0) -> int:
+    """Remove staging folders left behind by a killed run; return how many.
+
+    Looks through the whole tree under ``root``. Only folders untouched for
+    ``max_age`` seconds are removed, so a run in progress is never disturbed.
+    """
+    if not root.is_dir():
+        return 0
+
+    removed = 0
+    now = time.time()
+    for current, dirnames, _ in os.walk(root):  # does not follow links
+        for name in list(dirnames):
+            if not name.startswith(STAGING_PREFIX):
+                continue
+            dirnames.remove(name)  # never look inside a staging folder
+            item = Path(current) / name
+            try:
+                if not item.is_symlink() and now - item.stat().st_mtime > max_age:
+                    shutil.rmtree(item)
+                    removed += 1
+            except OSError:
+                continue
+    return removed
+
+
+def install_volumes(built: list[Path], dest_dir: Path, existing: list[Path]):
+    """Move a finished volume set into ``dest_dir``, replacing ``existing``.
+
+    Old volumes are removed only here, after the new set was built completely.
+    """
+    for old in existing:
+        old.unlink()
+
+    installed = []
+    for volume in built:
+        # Archives may hold private data; do not let umask make them readable by
+        # other users (no effect on Windows, where folder permissions apply).
+        restrict_permissions(volume)
+        target = dest_dir / volume.name
+        os.replace(volume, target)
+        installed.append(target)
+    return installed
+
+
+class VolumeWriter:
+    """Write-only file object that rolls over to a new volume every N bytes."""
+
+    def __init__(self, directory: Path, base: str, volume_size: int):
+        self._directory = directory
+        self._base = base
+        self._volume_size = volume_size
+        self._file = None
+        self._room = 0
+        self.volumes: list[Path] = []
+
+    def write(self, data) -> int:
+        view = memoryview(data)
+        total = len(view)
+
+        while len(view):
+            if self._room == 0:
+                self._next_volume()
+            chunk = view[: self._room]
+            self._file.write(chunk)
+            self._room -= len(chunk)
+            view = view[len(chunk) :]
+
+        return total
+
+    def _next_volume(self) -> None:
+        self._close_current()
+        number = len(self.volumes) + 1
+        path = self._directory / f"{self._base}.{number:03d}"
+        self._file = path.open("wb")
+        self.volumes.append(path)
+        self._room = self._volume_size
+
+    def _close_current(self) -> None:
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+    def close(self) -> None:
+        self._close_current()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.close()
+
+
+class VolumeReader(io.RawIOBase):
+    """Read-only file object that reads a volume set as one continuous stream."""
+
+    def __init__(self, paths):
+        super().__init__()
+        self._paths = list(paths)
+        self._index = 0
+        self._file = None
+
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, buffer) -> int:
+        view = memoryview(buffer).cast("B")
+        if not len(view):
+            return 0
+
+        while True:
+            if self._file is None:
+                if self._index >= len(self._paths):
+                    return 0
+                self._file = self._paths[self._index].open("rb")
+                self._index += 1
+
+            count = self._file.readinto(view)
+            if count:
+                return count
+
+            self._file.close()
+            self._file = None
+
+    def close(self) -> None:
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+        super().close()