batchvault 0.1.0__py3-none-any.whl

batchvault/config.py

@@ -0,0 +1,169 @@
+"""Project configuration from ``batchvault.toml`` or ``[tool.batchvault]``.
+
+Layout (keys use the same names as the command-line options)::
+
+    [defaults]            # applies to every command that has the option
+    jobs = 4
+
+    [log]
+    level = "DEBUG"       # --log-level
+    file = "run.log"      # --log-file
+
+    [notify]
+    url = "https://hooks.example.com/..."   # --notify
+    on = "failure"                          # --notify-on
+
+    [pack]                # per command: pack, unpack, verify, inspect
+    level = 7
+    volume-size = "100m"
+    include = ["*.mp4"]
+
+Command-line options override the file. List options (``include``,
+``exclude``) from the file and the command line are combined.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+
+from batchvault.errors import ConfigError
+
+CONFIG_FILE = "batchvault.toml"
+PYPROJECT = "pyproject.toml"
+
+COMMAND_TABLES = ("pack", "unpack", "verify", "inspect")
+# Tables that map their own key names onto command-line options.
+SECTION_KEYS = {
+    "log": {"level": "log_level", "file": "log_file"},
+    "notify": {"url": "notify", "on": "notify_on"},
+}
+KNOWN_TABLES = {"defaults", *COMMAND_TABLES, *SECTION_KEYS}
+
+# Options that run a program, send data out, choose which keys to trust, read a
+# password from somewhere or delete files. A config file found automatically in
+# the current or a parent folder may belong to someone else (a cloned
+# repository, a downloaded folder), so it may not set these. A file named with
+# --config or BATCHVAULT_CONFIG is trusted.
+RESTRICTED_OPTIONS = frozenset(
+    {
+        "seven_zip",
+        "notify",
+        "delete_source",
+        "yes",
+        "public_key",
+        "sign_key",
+        "password_file",
+        "log_file",
+    }
+)
+
+
+def _tool_section(pyproject: Path) -> dict | None:
+    try:
+        data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError):
+        return None
+    section = data.get("tool", {}).get("batchvault")
+    return section if isinstance(section, dict) else None
+
+
+def find_config(start: Path | None = None) -> Path | None:
+    """Look for a config file in ``start`` and then each parent folder."""
+    folder = (start or Path.cwd()).resolve()
+    for candidate in (folder, *folder.parents):
+        if (candidate / CONFIG_FILE).is_file():
+            return candidate / CONFIG_FILE
+        pyproject = candidate / PYPROJECT
+        if pyproject.is_file() and _tool_section(pyproject) is not None:
+            return pyproject
+    return None
+
+
+def load_config(path: Path) -> dict:
+    try:
+        text = path.read_text(encoding="utf-8")
+        data = tomllib.loads(text)
+    except OSError as exc:
+        raise ConfigError(f"Cannot read config file {path}: {exc}") from exc
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(f"Config file {path} is not valid TOML: {exc}") from exc
+
+    if path.name == PYPROJECT:
+        data = data.get("tool", {}).get("batchvault", {})
+
+    unknown = sorted(set(data) - KNOWN_TABLES)
+    if unknown:
+        raise ConfigError(
+            f"{path}: unknown section(s): {', '.join(unknown)}. "
+            f"Known: {', '.join(sorted(KNOWN_TABLES))}."
+        )
+    for name, table in data.items():
+        if not isinstance(table, dict):
+            raise ConfigError(f"{path}: [{name}] must be a table")
+    return data
+
+
+def reject_restricted(config: dict, path: Path) -> None:
+    """Refuse restricted options in a config file that was found automatically."""
+    for table_name, table in config.items():
+        mapping = SECTION_KEYS.get(table_name)
+        for key in table:
+            dest = mapping.get(key, key) if mapping else key.replace("-", "_")
+            if dest in RESTRICTED_OPTIONS:
+                raise ConfigError(
+                    f"{path} sets '{key}' in [{table_name}]. A config file found "
+                    "automatically cannot set options that run programs, send data, "
+                    "choose trusted keys, read passwords or delete files. If you "
+                    f"trust this file, run with: --config {path}"
+                )
+
+
+def _check_value(dest: str, value, action, where: str) -> None:
+    if action.choices is not None and value not in action.choices:
+        choices = ", ".join(str(choice) for choice in action.choices)
+        raise ConfigError(f"{where}: '{dest}' must be one of: {choices}")
+
+    if action.nargs == 0:  # on/off flag
+        if not isinstance(value, bool):
+            raise ConfigError(f"{where}: '{dest}' must be true or false")
+    elif action.type is int:
+        if isinstance(value, bool) or not isinstance(value, int):
+            raise ConfigError(f"{where}: '{dest}' must be a whole number")
+    elif action.default is not None and isinstance(action.default, list):
+        if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+            raise ConfigError(f"{where}: '{dest}' must be a list of strings")
+    elif not isinstance(value, str | int | float) or isinstance(value, bool):
+        raise ConfigError(f"{where}: '{dest}' has an unsupported value")
+
+
+def defaults_for(config: dict, command: str, actions: dict) -> dict:
+    """Option defaults for ``command`` (``actions`` maps option name to Action)."""
+    merged: dict = {}
+
+    for key, value in config.get("defaults", {}).items():
+        dest = key.replace("-", "_")
+        if dest in actions:
+            _check_value(dest, value, actions[dest], "[defaults]")
+            merged[dest] = value
+
+    for section, mapping in SECTION_KEYS.items():
+        for key, value in config.get(section, {}).items():
+            dest = mapping.get(key)
+            if dest is None:
+                raise ConfigError(
+                    f"Unknown key '{key}' in [{section}]. "
+                    f"Known: {', '.join(mapping)}."
+                )
+            if dest in actions:
+                _check_value(dest, value, actions[dest], f"[{section}]")
+                merged[dest] = value
+
+    for key, value in config.get(command, {}).items():
+        dest = key.replace("-", "_")
+        if dest not in actions:
+            raise ConfigError(f"Unknown option '{key}' in [{command}].")
+        _check_value(dest, value, actions[dest], f"[{command}]")
+        merged[dest] = value
+
+    return merged

batchvault/crypto.py

@@ -0,0 +1,193 @@
+"""Optional encryption for the native engine.
+
+Needs the ``cryptography`` package (``pip install "batchvault[crypto]"``).
+
+An encrypted archive is a header followed by one record per compressed block
+and a final terminator record. Each record is AES-256-GCM encrypted with a
+unique nonce; the header, the record number and a "last record" flag are
+authenticated, so reordering, dropping or truncating records is detected. The
+key comes from the password with scrypt and a random per-archive salt.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import os
+import struct
+
+from batchvault.errors import EngineError, EngineNotFoundError
+
+MAGIC = b"BVENC1\x00\x00"
+FORMAT_IDS = {"xz": 1, "gzip": 2, "bzip2": 3, "store": 4}
+_ID_FORMATS = {value: key for key, value in FORMAT_IDS.items()}
+SALT_SIZE = 16
+PREFIX_SIZE = 4
+HEADER_SIZE = len(MAGIC) + 2 + SALT_SIZE + PREFIX_SIZE
+
+# scrypt cost: N = 2**log2_n (about 128 MiB of memory at the default), r = 8, p = 1.
+# The value used is stored in the archive header so the default can rise later,
+# and header values outside this range are refused: a damaged or hostile header
+# must not be able to ask for terabytes of memory.
+DEFAULT_LOG2_N = 17
+KDF_LOG2_N_RANGE = (14, 18)
+_SCRYPT_R = 8
+_SCRYPT_P = 1
+
+_LENGTH = struct.Struct(">I")
+_INDEX = struct.Struct(">Q")
+# A record holds one compressed block (16 MiB by default); anything near this
+# size is damage, and the limit stops a bad length from allocating gigabytes.
+_MAX_RECORD = 128 * 1024 * 1024
+
+INSTALL_HINT = (
+    "Encryption and signing need the optional 'cryptography' package: "
+    'pip install "batchvault[crypto]"'
+)
+
+
+def require_cryptography() -> None:
+    try:
+        import cryptography  # noqa: F401
+    except ImportError as exc:
+        raise EngineNotFoundError(INSTALL_HINT) from exc
+
+
+def make_header(fmt: str) -> bytes:
+    return (
+        MAGIC
+        + bytes([FORMAT_IDS[fmt], DEFAULT_LOG2_N])
+        + os.urandom(SALT_SIZE + PREFIX_SIZE)
+    )
+
+
+def is_encrypted_header(data: bytes) -> bool:
+    return data.startswith(MAGIC)
+
+
+def parse_header(data: bytes) -> tuple[str, int, bytes, bytes]:
+    """Return ``(format, log2_n, salt, nonce_prefix)`` from an archive header."""
+    if len(data) < HEADER_SIZE or not is_encrypted_header(data):
+        raise EngineError("Not an encrypted BatchVault archive.")
+    fmt = _ID_FORMATS.get(data[len(MAGIC)])
+    if fmt is None:
+        raise EngineError("Unknown format in encrypted archive header.")
+
+    log2_n = data[len(MAGIC) + 1]
+    low, high = KDF_LOG2_N_RANGE
+    if not low <= log2_n <= high:
+        raise EngineError("Unsupported key setting in the encrypted archive header.")
+
+    start = len(MAGIC) + 2
+    salt = data[start : start + SALT_SIZE]
+    return fmt, log2_n, salt, data[start + SALT_SIZE : HEADER_SIZE]
+
+
+class BlockCipher:
+    """Encrypts and decrypts numbered records for one archive."""
+
+    def __init__(self, password: str, header: bytes):
+        require_cryptography()
+        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+        _, log2_n, salt, prefix = parse_header(header)
+        n = 1 << log2_n
+        key = hashlib.scrypt(
+            password.encode("utf-8"),
+            salt=salt,
+            n=n,
+            r=_SCRYPT_R,
+            p=_SCRYPT_P,
+            maxmem=130 * n * _SCRYPT_R,  # scrypt needs about 128 * n * r bytes
+            dklen=32,
+        )
+        self._aes = AESGCM(key)
+        self._prefix = prefix
+        self._header = header
+
+    def _nonce(self, index: int) -> bytes:
+        return self._prefix + _INDEX.pack(index)
+
+    def _aad(self, index: int, last: bool) -> bytes:
+        return self._header + _INDEX.pack(index) + (b"\x01" if last else b"\x00")
+
+    def encrypt(self, index: int, data: bytes, last: bool = False) -> bytes:
+        """Return a length-prefixed encrypted record."""
+        sealed = self._aes.encrypt(self._nonce(index), data, self._aad(index, last))
+        return _LENGTH.pack(len(sealed)) + sealed
+
+    def decrypt(self, index: int, sealed: bytes) -> tuple[bytes, bool]:
+        """Return ``(data, is_last)``; raise EngineError on a wrong password."""
+        from cryptography.exceptions import InvalidTag
+
+        for last in (False, True):
+            try:
+                data = self._aes.decrypt(
+                    self._nonce(index), sealed, self._aad(index, last)
+                )
+            except InvalidTag:
+                continue
+            return data, last
+        raise EngineError("Wrong password, or the encrypted archive is damaged.")
+
+
+def _read_exact(source, count: int) -> bytes:
+    chunks = []
+    while count:
+        chunk = source.read(count)
+        if not chunk:
+            break
+        chunks.append(chunk)
+        count -= len(chunk)
+    return b"".join(chunks)
+
+
+class DecryptingReader(io.RawIOBase):
+    """Read-only stream of the decrypted (still compressed) block data."""
+
+    def __init__(self, source, cipher: BlockCipher):
+        super().__init__()
+        self._source = source
+        self._cipher = cipher
+        self._buffer = memoryview(b"")
+        self._index = 0
+        self._done = False
+
+    def readable(self) -> bool:
+        return True
+
+    def _next_record(self) -> None:
+        head = _read_exact(self._source, _LENGTH.size)
+        if len(head) < _LENGTH.size:
+            raise EngineError("Encrypted archive is truncated (no end marker).")
+
+        (length,) = _LENGTH.unpack(head)
+        if length > _MAX_RECORD:
+            raise EngineError("Encrypted archive is damaged (bad record size).")
+
+        sealed = _read_exact(self._source, length)
+        if len(sealed) < length:
+            raise EngineError("Encrypted archive is truncated.")
+
+        data, self._done = self._cipher.decrypt(self._index, sealed)
+        self._index += 1
+        self._buffer = memoryview(data)
+
+    def readinto(self, buffer) -> int:
+        view = memoryview(buffer).cast("B")
+        while not len(self._buffer):
+            if self._done:
+                return 0
+            self._next_record()
+
+        count = min(len(view), len(self._buffer))
+        view[:count] = self._buffer[:count]
+        self._buffer = self._buffer[count:]
+        return count
+
+
+def open_decrypted(source, password: str) -> tuple[str, DecryptingReader]:
+    """Read the header from ``source``; return the inner format and a reader."""
+    header = _read_exact(source, HEADER_SIZE)
+    fmt = parse_header(header)[0]
+    return fmt, DecryptingReader(source, BlockCipher(password, header))

batchvault/doctor.py

@@ -0,0 +1,67 @@
+"""Checks that BatchVault can run on this machine."""
+
+from __future__ import annotations
+
+import os
+import platform
+import sys
+import tempfile
+from dataclasses import dataclass
+
+from batchvault.engines.sevenzip import find_7zip
+from batchvault.errors import EngineNotFoundError
+from batchvault.logs import default_log_dir
+
+
+@dataclass(frozen=True)
+class Check:
+    name: str
+    ok: bool
+    detail: str
+    required: bool = False  # a failed required check makes doctor exit non-zero
+
+
+def _module(name: str, label: str, required: bool = False) -> Check:
+    try:
+        __import__(name)
+    except ImportError:
+        return Check(label, False, "not available in this Python build", required)
+    return Check(label, True, "available", required)
+
+
+def _log_dir_check() -> Check:
+    folder = default_log_dir()
+    try:
+        folder.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=folder):
+            pass
+    except OSError as exc:
+        return Check("Log folder", False, f"{folder} is not writable: {exc}")
+    return Check("Log folder", True, str(folder))
+
+
+def run_checks(seven_zip: str | None = None) -> list[Check]:
+    checks = [
+        Check(
+            "Python",
+            sys.version_info >= (3, 11, 4),  # safe tar extraction filters
+            f"{platform.python_version()} on {platform.system()}",
+            required=True,
+        ),
+        _module("zlib", "zlib (gzip, store)", required=True),
+        _module("lzma", "lzma (xz format)"),
+        _module("bz2", "bz2 (bzip2 format)"),
+    ]
+
+    try:
+        checks.append(Check("7-Zip (optional)", True, str(find_7zip(seven_zip))))
+    except EngineNotFoundError as exc:
+        checks.append(Check("7-Zip (optional)", False, str(exc)))
+
+    checks.append(_log_dir_check())
+    checks.append(Check("CPU threads", True, str(os.cpu_count() or 1)))
+    return checks
+
+
+def passed(checks: list[Check]) -> bool:
+    return all(check.ok for check in checks if check.required)

batchvault/engines/base.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+
+from batchvault.options import PackOptions
+
+
+@dataclass(frozen=True)
+class Capabilities:
+    """What an engine supports, used to warn about unsupported settings."""
+
+    levels: range
+    threads_per_file: bool
+    archive_suffix: str
+
+
+class Engine(ABC):
+    """Common interface implemented by every compression engine."""
+
+    name: str
+    capabilities: Capabilities
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """True if the engine can run on this machine."""
+
+    @abstractmethod
+    def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]:
+        """Archive ``source`` into volumes inside ``dest_dir``; return the volumes.
+
+        Encrypts when ``options.password`` is set. Raises OutputExistsError if
+        volumes already exist and ``options.force`` is false. Existing volumes
+        are only removed once the new set is complete.
+        """
+
+    @abstractmethod
+    def unpack(
+        self,
+        volume: Path,
+        output_dir: Path,
+        *,
+        force: bool = False,
+        password: str | None = None,
+        max_bytes: int | None = None,
+    ) -> list[Path]:
+        """Restore the files in the volume set containing ``volume``.
+
+        Raises OutputExistsError if a destination exists and ``force`` is false.
+        With ``max_bytes``, an archive that would expand beyond that many bytes
+        is refused before anything is written.
+        """
+
+    @abstractmethod
+    def test(self, volume: Path, *, password: str | None = None) -> None:
+        """Check the volume set's integrity; raise EngineError if it fails."""