batchvault 0.1.0__py3-none-any.whl

batchvault/engines/native.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+import os
+import tarfile
+import tempfile
+import zlib
+from pathlib import Path, PurePosixPath
+
+from batchvault import crypto
+from batchvault.blocks import (
+    BLOCK_SIZE,
+    BlockWriter,
+    check_format,
+    open_decompressed,
+    sniff_archive,
+)
+from batchvault.engines.base import Capabilities, Engine
+from batchvault.errors import EngineError, OutputExistsError
+from batchvault.names import unsafe_on_windows as _unsafe_on_windows
+from batchvault.options import MIB, PackOptions
+from batchvault.volumes import (
+    VolumeReader,
+    VolumeWriter,
+    existing_volumes,
+    find_volumes,
+    install_volumes,
+)
+
+try:
+    import lzma
+
+    _LZMA_ERRORS: tuple[type[Exception], ...] = (lzma.LZMAError,)
+except ImportError:
+    _LZMA_ERRORS = ()
+
+# Errors raised while reading damaged, truncated or unreadable archive data.
+_READ_ERRORS = (tarfile.TarError, EOFError, zlib.error, OSError, *_LZMA_ERRORS)
+
+_STREAM_BUFFER = MIB
+_CHUNK = MIB
+# Data allowed after the tar end marker; a real archive has only padding.
+_MAX_TRAILING = 4 * MIB
+
+
+def _normalize(info: tarfile.TarInfo) -> tarfile.TarInfo:
+    """Drop ownership so archives do not depend on the user who made them."""
+    info.uid = info.gid = 0
+    info.uname = info.gname = ""
+    return info
+
+
+class _Budget:
+    """Caps the total size of the files restored from one archive."""
+
+    def __init__(self, limit: int | None):
+        self.remaining = limit
+
+    def charge(self, member: tarfile.TarInfo) -> None:
+        if self.remaining is None:
+            return
+        if member.size > self.remaining:
+            raise EngineError(
+                f"{member.name!r} would expand beyond the allowed size; "
+                "the archive was not restored."
+            )
+        self.remaining -= member.size
+
+
+def _check_member(member: tarfile.TarInfo, windows: bool | None = None) -> None:
+    """Accept only a plain, single-part regular file name that is safe to write."""
+    if windows is None:
+        windows = os.name == "nt"
+
+    path = PurePosixPath(member.name)
+    if (
+        not member.isfile()
+        or path.is_absolute()
+        or len(path.parts) != 1
+        or path.name in ("", ".", "..")
+        or "\\" in member.name
+        or (windows and _unsafe_on_windows(member.name))
+    ):
+        raise EngineError(f"Unsupported or unsafe entry in archive: {member.name!r}")
+
+
+class NativeEngine(Engine):
+    """Standard-library engine: tar stream, block-parallel compression."""
+
+    name = "native"
+    capabilities = Capabilities(
+        levels=range(0, 10), threads_per_file=True, archive_suffix=".bv"
+    )
+
+    def __init__(self, block_size: int = BLOCK_SIZE):
+        self._block_size = block_size
+
+    def is_available(self) -> bool:
+        return True
+
+    def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]:
+        if not source.is_file():
+            raise FileNotFoundError(f"Source file not found: {source}")
+        check_format(options.format)
+
+        base = f"{source.name}{self.capabilities.archive_suffix}"
+        existing = existing_volumes(dest_dir, base)
+        if existing and not options.force:
+            raise OutputExistsError(
+                f"Archive volumes already exist in {dest_dir}. Use --force to replace."
+            )
+
+        header = cipher = None
+        if options.password:
+            header = crypto.make_header(options.format)
+            cipher = crypto.BlockCipher(options.password, header)
+
+        dest_dir.mkdir(parents=True, exist_ok=True)
+
+        # Build in a temporary folder so a failed run never damages what exists.
+        with tempfile.TemporaryDirectory(dir=dest_dir, prefix=".batchvault-") as tmp:
+            with VolumeWriter(Path(tmp), base, options.volume_size) as volumes:
+                with BlockWriter(
+                    volumes,
+                    options.format,
+                    options.level,
+                    self._block_size,
+                    options.jobs,
+                    cipher=cipher,
+                    header=header or b"",
+                ) as blocks:
+                    with tarfile.open(
+                        fileobj=blocks,
+                        mode="w|",
+                        format=tarfile.PAX_FORMAT,
+                        bufsize=_STREAM_BUFFER,
+                    ) as tar:
+                        tar.add(source, arcname=source.name, filter=_normalize)
+
+            return install_volumes(volumes.volumes, dest_dir, existing)
+
+    def unpack(
+        self,
+        volume: Path,
+        output_dir: Path,
+        *,
+        force: bool = False,
+        password: str | None = None,
+        max_bytes: int | None = None,
+    ) -> list[Path]:
+        volumes = find_volumes(volume)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        budget = _Budget(max_bytes)
+
+        def handle(member, tar) -> None:
+            budget.charge(member)  # refuse a decompression bomb before writing
+            self._extract(member, tar, staging)
+
+        with tempfile.TemporaryDirectory(dir=output_dir, prefix=".batchvault-") as tmp:
+            staging = Path(tmp)
+            self._read(volumes, handle, password)
+
+            extracted = sorted(staging.iterdir())
+            if not extracted:
+                raise EngineError(f"No files found in archive {volumes[0]}")
+
+            conflicts = [
+                output_dir / item.name
+                for item in extracted
+                if (output_dir / item.name).exists()
+            ]
+            if conflicts and not force:
+                names = ", ".join(path.name for path in conflicts)
+                raise OutputExistsError(
+                    f"Destination already exists: {names}. Use --force to overwrite."
+                )
+
+            restored = []
+            for item in extracted:
+                target = output_dir / item.name
+                os.replace(item, target)
+                restored.append(target)
+
+        return restored
+
+    def test(self, volume: Path, *, password: str | None = None) -> None:
+        self._read(find_volumes(volume), self._drain_member, password)
+
+    @staticmethod
+    def _extract(member, tar, staging: Path) -> None:
+        # Belt and braces next to _check_member and the "data" filter: whatever
+        # the name looks like, the file must end up inside the staging folder.
+        root = staging.resolve()
+        if root not in (root / member.name).resolve().parents:
+            raise EngineError(f"Unsafe entry in archive: {member.name!r}")
+        tar.extract(member, staging, filter="data")
+
+    @staticmethod
+    def _drain_member(member, tar) -> None:
+        handle = tar.extractfile(member)
+        while handle.read(_CHUNK):
+            pass
+
+    @staticmethod
+    def _read(volumes: list[Path], handle_member, password: str | None) -> None:
+        """Stream the archive and call ``handle_member(member, tar)`` for each file."""
+        fmt, encrypted = sniff_archive(volumes[0])
+        if encrypted and not password:
+            raise EngineError(
+                f"{volumes[0].name} is encrypted. A password is required."
+            )
+
+        try:
+            with VolumeReader(volumes) as reader:
+                source = reader
+                if encrypted:
+                    fmt, source = crypto.open_decrypted(reader, password)
+
+                with open_decompressed(fmt, source) as stream:
+                    with tarfile.open(fileobj=stream, mode="r|") as tar:
+                        for member in tar:
+                            _check_member(member)
+                            handle_member(member, tar)
+
+                    # Read to the very end so a truncated stream is detected. A
+                    # real archive has almost nothing after the tar end marker.
+                    trailing = 0
+                    while chunk := stream.read(_CHUNK):
+                        trailing += len(chunk)
+                        if trailing > _MAX_TRAILING:
+                            raise EngineError(
+                                f"{volumes[0].name} has unexpected data after the "
+                                "end of the archive."
+                            )
+        except _READ_ERRORS as exc:
+            raise EngineError(
+                f"Archive is corrupt or incomplete ({volumes[0].name}): {exc}"
+            ) from exc

batchvault/engines/registry.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from batchvault.engines.base import Engine
+from batchvault.engines.native import NativeEngine
+from batchvault.engines.sevenzip import SevenZipEngine, find_7zip
+from batchvault.errors import InputError
+from batchvault.volumes import split_volume_name
+
+ENGINE_CHOICES = ("auto", "native", "7zip")
+
+
+def get_engine(name: str = "auto", seven_zip: str | Path | None = None) -> Engine:
+    """Return the engine to create archives with.
+
+    ``auto`` uses 7-Zip when it is found (or when a path to it was given) and
+    falls back to the native engine.
+    """
+    if name == "native":
+        return NativeEngine()
+
+    if name in ("7zip", "7z"):
+        find_7zip(seven_zip)  # raises EngineNotFoundError with install help
+        return SevenZipEngine(seven_zip)
+
+    if name == "auto":
+        if seven_zip:
+            find_7zip(seven_zip)
+            return SevenZipEngine(seven_zip)
+        sevenzip = SevenZipEngine()
+        return sevenzip if sevenzip.is_available() else NativeEngine()
+
+    choices = ", ".join(ENGINE_CHOICES)
+    raise InputError(f"Unknown engine {name!r}. Choose from: {choices}.")
+
+
+def engine_for_volume(volume: Path, seven_zip: str | Path | None = None) -> Engine:
+    """Pick the engine that can read a volume, from its file name."""
+    parsed = split_volume_name(volume.name)
+    if parsed is None:
+        raise InputError(f"Not an archive volume: {volume}")
+    if parsed[0].lower().endswith(".7z"):
+        find_7zip(seven_zip)
+        return SevenZipEngine(seven_zip)
+    return NativeEngine()
+
+
+def option_warnings(
+    engine: Engine, format_given: str | None, level: int
+) -> list[str]:
+    """Settings the chosen engine cannot honour exactly."""
+    warnings = []
+    if engine.name == "7zip" and format_given not in (None, "xz"):
+        warnings.append(
+            f"The 7-Zip engine always writes 7z (LZMA2); --format {format_given} "
+            "is ignored."
+        )
+    if engine.name == "native" and format_given == "bzip2" and level == 0:
+        warnings.append("bzip2 has no level 0; level 1 was used.")
+    return warnings

batchvault/engines/sevenzip.py

@@ -0,0 +1,303 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from batchvault.engines.base import Capabilities, Engine
+from batchvault.errors import EngineError, EngineNotFoundError, OutputExistsError
+from batchvault.options import PackOptions
+from batchvault.volumes import existing_volumes, find_volumes, install_volumes
+
+ENV_VAR = "BATCHVAULT_7Z"
+COMMANDS = ("7zz", "7z", "7za", "7zr")
+WINDOWS_PATHS = (
+    Path(r"C:\Program Files\7-Zip\7z.exe"),
+    Path(r"C:\Program Files (x86)\7-Zip\7z.exe"),
+)
+OUTPUT_TAIL_LINES = 15
+LIMIT_POLL_SECONDS = 0.05
+
+
+def _search_path(command: str) -> Path | None:
+    """Find ``command`` in the folders listed in ``PATH``.
+
+    Unlike ``shutil.which``, this never looks in the current folder (Windows does
+    that first) and ignores empty and relative ``PATH`` entries, so a program
+    planted in the folder you happen to be working in is never run.
+    """
+    cwd = Path.cwd().resolve()
+    if sys.platform == "win32":
+        extensions = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";")
+        names = [command + extension.lower() for extension in extensions if extension]
+    else:
+        names = [command]
+
+    for entry in os.environ.get("PATH", "").split(os.pathsep):
+        folder = Path(entry) if entry else None
+        if folder is None or not folder.is_absolute():
+            continue
+        try:
+            if folder.resolve() == cwd:
+                continue
+        except OSError:
+            continue
+
+        for name in names:
+            candidate = folder / name
+            if candidate.is_file() and os.access(candidate, os.X_OK):
+                return candidate
+    return None
+
+
+def find_7zip(explicit: str | Path | None = None) -> Path:
+    """Locate a 7-Zip command-line executable.
+
+    Order: explicit path, ``BATCHVAULT_7Z``, ``PATH`` (never the current folder),
+    then well-known Windows install folders.
+    """
+    configured = explicit or os.environ.get(ENV_VAR)
+    if configured:
+        path = Path(configured).expanduser()
+        if path.is_file():
+            return path
+        raise EngineNotFoundError(f"7-Zip executable not found: {configured}")
+
+    for command in COMMANDS:
+        found = _search_path(command)
+        if found:
+            return found
+
+    if sys.platform == "win32":
+        for candidate in WINDOWS_PATHS:
+            if candidate.is_file():
+                return candidate
+
+    raise EngineNotFoundError(
+        "Could not find 7-Zip. Install it from https://www.7-zip.org/ (7zz on "
+        f"Linux and macOS), or set the path with --7z or {ENV_VAR}."
+    )
+
+
+def build_pack_command(
+    executable: Path, archive: Path, source: Path, options: PackOptions
+) -> list[str]:
+    command = [
+        str(executable),
+        "a",
+        "-t7z",
+        f"-mx={options.level}",
+        "-m0=LZMA2",
+        f"-mmt={max(options.jobs, 1)}",
+        f"-v{options.volume_size}b",
+        "-y",
+        # A file name containing * or ? is a name, not a pattern; without this
+        # 7-Zip would archive every neighbour it matches.
+        "-spd",
+    ]
+    if options.password:
+        # 7-Zip only takes the password on its command line, where other users
+        # of this machine could see it in the process list while it runs.
+        command += [f"-p{options.password}", "-mhe=on"]
+    return [*command, str(archive), str(source)]
+
+
+def _password_args(password: str | None) -> list[str]:
+    return [f"-p{password}"] if password else []
+
+
+def build_unpack_command(
+    executable: Path, first_volume: Path, target: Path, password: str | None = None
+):
+    return [
+        str(executable),
+        "e",
+        str(first_volume),
+        f"-o{target}",
+        "-y",
+        "-spd",
+        *_password_args(password),
+    ]
+
+
+def build_test_command(
+    executable: Path, first_volume: Path, password: str | None = None
+) -> list[str]:
+    return [
+        str(executable), "t", str(first_volume), "-spd", *_password_args(password)
+    ]  # fmt: skip
+
+
+def folder_size(folder: Path) -> int:
+    """Total size of the files below ``folder`` (files that vanish are ignored)."""
+    total = 0
+    for root, _, names in os.walk(folder):
+        for name in names:
+            try:
+                total += (Path(root) / name).stat().st_size
+            except OSError:
+                continue
+    return total
+
+
+class SevenZipEngine(Engine):
+    name = "7zip"
+    capabilities = Capabilities(
+        levels=range(0, 10), threads_per_file=True, archive_suffix=".7z"
+    )
+
+    def __init__(self, executable: str | Path | None = None):
+        self._explicit = executable
+        self._executable: Path | None = None
+
+    @property
+    def executable(self) -> Path:
+        if self._executable is None:
+            self._executable = find_7zip(self._explicit)
+        return self._executable
+
+    def is_available(self) -> bool:
+        try:
+            find_7zip(self._explicit)
+        except EngineNotFoundError:
+            return False
+        return True
+
+    @staticmethod
+    def _failure(code: int, output: str) -> EngineError:
+        tail = "\n".join(output.strip().splitlines()[-OUTPUT_TAIL_LINES:])
+        return EngineError(f"7-Zip failed (exit code {code}).\n{tail}".rstrip())
+
+    def _run(self, command: list[str]) -> str:
+        """Run 7-Zip; return its standard output, raise EngineError on failure."""
+        result = subprocess.run(
+            command,
+            capture_output=True,
+            stdin=subprocess.DEVNULL,  # never wait for a password prompt
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            check=False,
+        )
+        if result.returncode != 0:
+            raise self._failure(result.returncode, result.stderr or result.stdout)
+        return result.stdout
+
+    def _run_limited(self, command: list[str], watch: Path, max_bytes: int) -> str:
+        """Like ``_run``, but stop the program if ``watch`` grows past ``max_bytes``.
+
+        7-Zip writes as it extracts, so a decompression bomb is cut off while it
+        is still small instead of filling the disk. Output goes to a temporary
+        file so a full pipe can never stall the program.
+        """
+        too_big = EngineError(
+            f"The archive would expand beyond the allowed size ({max_bytes} bytes); "
+            "extraction was stopped."
+        )
+
+        with tempfile.TemporaryFile() as output:
+            process = subprocess.Popen(
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=output,
+                stderr=subprocess.STDOUT,
+            )
+            try:
+                while True:
+                    try:
+                        code = process.wait(timeout=LIMIT_POLL_SECONDS)
+                        break
+                    except subprocess.TimeoutExpired:
+                        if folder_size(watch) > max_bytes:
+                            raise too_big from None
+            finally:
+                if process.poll() is None:
+                    process.kill()
+                    process.wait()
+
+            output.seek(0)
+            text = output.read().decode("utf-8", errors="replace")
+
+        if folder_size(watch) > max_bytes:  # it finished between two checks
+            raise too_big
+        if code != 0:
+            raise self._failure(code, text)
+        return text
+
+    def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]:
+        if not source.is_file():
+            raise FileNotFoundError(f"Source file not found: {source}")
+
+        base = f"{source.name}{self.capabilities.archive_suffix}"
+        existing = existing_volumes(dest_dir, base)
+        if existing and not options.force:
+            raise OutputExistsError(
+                f"Archive volumes already exist in {dest_dir}. Use --force to replace."
+            )
+
+        dest_dir.mkdir(parents=True, exist_ok=True)
+
+        # Build in a temporary folder so a failed run never damages what exists.
+        with tempfile.TemporaryDirectory(dir=dest_dir, prefix=".batchvault-") as tmp:
+            staging = Path(tmp)
+            self._run(
+                build_pack_command(self.executable, staging / base, source, options)
+            )
+
+            built = existing_volumes(staging, base)
+            if not built:
+                raise EngineError(f"7-Zip produced no volumes for {source}")
+
+            return install_volumes(built, dest_dir, existing)
+
+    def unpack(
+        self,
+        volume: Path,
+        output_dir: Path,
+        *,
+        force: bool = False,
+        password: str | None = None,
+        max_bytes: int | None = None,
+    ) -> list[Path]:
+        first = find_volumes(volume)[0]
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        with tempfile.TemporaryDirectory(dir=output_dir, prefix=".batchvault-") as tmp:
+            staging = Path(tmp)
+            command = build_unpack_command(self.executable, first, staging, password)
+            if max_bytes is None:
+                self._run(command)
+            else:
+                self._run_limited(command, staging, max_bytes)
+
+            # Never accept links or other special files from an archive.
+            if any(p.is_symlink() or not p.is_file() for p in staging.iterdir()):
+                raise EngineError(
+                    f"{first.name} contains a link or special file, which is refused."
+                )
+
+            extracted = sorted(staging.iterdir())
+            if not extracted:
+                raise EngineError(f"7-Zip extracted no files from {first}")
+
+            conflicts = [output_dir / p.name for p in extracted]
+            conflicts = [path for path in conflicts if path.exists()]
+            if conflicts and not force:
+                names = ", ".join(path.name for path in conflicts)
+                raise OutputExistsError(
+                    f"Destination already exists: {names}. Use --force to overwrite."
+                )
+
+            restored = []
+            for item in extracted:
+                target = output_dir / item.name
+                os.replace(item, target)
+                restored.append(target)
+
+        return restored
+
+    def test(self, volume: Path, *, password: str | None = None) -> None:
+        first = find_volumes(volume)[0]
+        self._run(build_test_command(self.executable, first, password))

batchvault/errors.py

@@ -0,0 +1,38 @@
+class BatchVaultError(Exception):
+    """Base class for all BatchVault errors."""
+
+
+class EngineNotFoundError(BatchVaultError):
+    """The requested compression engine is not available."""
+
+
+class OptionsError(BatchVaultError, ValueError):
+    """An option value is invalid."""
+
+
+class ConfigError(BatchVaultError):
+    """The configuration file is unreadable or contains invalid settings."""
+
+
+class SignatureError(BatchVaultError):
+    """A manifest signature is missing, invalid or cannot be created."""
+
+
+class InputError(BatchVaultError):
+    """An input file, list or path is missing or unusable."""
+
+
+class EngineError(BatchVaultError):
+    """A compression engine failed to run or reported an error."""
+
+
+class OutputExistsError(BatchVaultError):
+    """The destination already exists and replacing it was not requested."""
+
+
+class ManifestError(BatchVaultError):
+    """A manifest is missing, malformed or fails validation."""
+
+
+class VolumeError(BatchVaultError):
+    """Archive volumes are missing, incomplete or corrupt."""

batchvault/eta.py

@@ -0,0 +1,39 @@
+import time
+
+# Wait for this much real run time before trusting the measured speed.
+MIN_SECONDS = 5.0
+
+
+def format_duration(seconds):
+    seconds = max(0, int(seconds))
+    hours, remainder = divmod(seconds, 3600)
+    minutes, secs = divmod(remainder, 60)
+    return f"{hours}:{minutes:02d}:{secs:02d}"
+
+
+class EtaEstimator:
+    """ETA from the average speed of the whole run so far.
+
+    Speed is bytes completed divided by the time since the first file started,
+    so per-file overhead is included.
+    """
+
+    def __init__(self, clock=time.monotonic):
+        self.clock = clock
+        self.started_at = None
+
+    def start(self):
+        if self.started_at is None:
+            self.started_at = self.clock()
+
+    def estimate(self, done_bytes, total_bytes):
+        """Seconds remaining, or None while there is not enough data yet."""
+        if self.started_at is None or done_bytes <= 0:
+            return None
+
+        elapsed = self.clock() - self.started_at
+        if elapsed < MIN_SECONDS:
+            return None
+
+        remaining = max(0, total_bytes - done_bytes)
+        return remaining / (done_bytes / elapsed)