batchvault 0.1.0__py3-none-any.whl

batchvault/manifest.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from batchvault import __version__
+from batchvault.errors import ManifestError
+from batchvault.names import validate_folder
+
+SCHEMA_VERSION = 1
+MANIFEST_NAME = "manifest.json"
+_CHUNK = 1024 * 1024
+
+
+def hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(_CHUNK):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _require(data: dict, key: str, kind: type, where: str):
+    value = data.get(key)
+    # bool is an int subclass; never accept it as a number.
+    if not isinstance(value, kind) or isinstance(value, bool) and kind is int:
+        raise ManifestError(f"{where}: '{key}' is missing or not a {kind.__name__}")
+    return value
+
+
+def _folder(value, where: str) -> str:
+    if not isinstance(value, str):
+        raise ManifestError(f"{where}: 'folder' must be a string")
+    try:
+        return validate_folder(value)
+    except ValueError as exc:
+        raise ManifestError(f"{where}: {exc}") from exc
+
+
+@dataclass(frozen=True)
+class VolumeInfo:
+    file: str  # relative to the manifest folder, with forward slashes
+    size: int
+    sha256: str
+
+    @classmethod
+    def from_dict(cls, data, where: str) -> VolumeInfo:
+        if not isinstance(data, dict):
+            raise ManifestError(f"{where}: volume entry must be an object")
+        return cls(
+            file=_require(data, "file", str, where),
+            size=_require(data, "size", int, where),
+            sha256=_require(data, "sha256", str, where),
+        )
+
+
+@dataclass
+class ManifestEntry:
+    name: str
+    size: int
+    sha256: str
+    engine: str
+    format: str
+    level: int
+    volume_size: int
+    volumes: list[VolumeInfo] = field(default_factory=list)
+    encrypted: bool = False
+    mtime_ns: int = 0  # source modification time when archived; 0 if unknown
+    folder: str = ""  # where the file lived, relative to --root ("" = flat)
+
+    @property
+    def path(self) -> str:
+        """The file's path relative to the root, with forward slashes."""
+        return f"{self.folder}/{self.name}" if self.folder else self.name
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name,
+            "size": self.size,
+            "sha256": self.sha256,
+            "engine": self.engine,
+            "format": self.format,
+            "level": self.level,
+            "volume_size": self.volume_size,
+            "encrypted": self.encrypted,
+            "mtime_ns": self.mtime_ns,
+            "folder": self.folder,
+            "volumes": [vars(volume) for volume in self.volumes],
+        }
+
+    @classmethod
+    def from_dict(cls, data, index: int) -> ManifestEntry:
+        where = f"entries[{index}]"
+        if not isinstance(data, dict):
+            raise ManifestError(f"{where}: must be an object")
+
+        name = _require(data, "name", str, where)
+        if not name or "/" in name or "\\" in name or name in (".", ".."):
+            raise ManifestError(f"{where}: unsafe file name {name!r}")
+
+        raw_volumes = _require(data, "volumes", list, where)
+        if not raw_volumes:
+            raise ManifestError(f"{where}: no volumes listed")
+
+        return cls(
+            name=name,
+            size=_require(data, "size", int, where),
+            sha256=_require(data, "sha256", str, where),
+            engine=_require(data, "engine", str, where),
+            format=_require(data, "format", str, where),
+            level=_require(data, "level", int, where),
+            volume_size=_require(data, "volume_size", int, where),
+            volumes=[VolumeInfo.from_dict(v, where) for v in raw_volumes],
+            encrypted=bool(data.get("encrypted", False)),
+            mtime_ns=int(data.get("mtime_ns", 0) or 0),
+            folder=_folder(data.get("folder", ""), where),
+        )
+
+    def check_volumes(self, base_dir: Path) -> list[str]:
+        """Problems found with the volumes on disk; empty if all are intact."""
+        problems = []
+        for volume in self.volumes:
+            path = base_dir / volume.file
+            if not path.is_file():
+                problems.append(f"missing volume {volume.file}")
+            elif path.stat().st_size != volume.size:
+                problems.append(f"volume {volume.file} has the wrong size")
+            elif hash_file(path) != volume.sha256:
+                problems.append(f"volume {volume.file} fails its checksum")
+        return problems
+
+
+@dataclass
+class Manifest:
+    entries: dict[str, ManifestEntry] = field(default_factory=dict)
+    created: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "created": self.created or datetime.now(UTC).isoformat(timespec="seconds"),
+            "batchvault_version": __version__,
+            "entries": [entry.to_dict() for entry in self.entries.values()],
+        }
+
+    @classmethod
+    def from_dict(cls, data) -> Manifest:
+        if not isinstance(data, dict):
+            raise ManifestError("Manifest must be a JSON object")
+
+        version = data.get("schema_version")
+        if version != SCHEMA_VERSION:
+            raise ManifestError(
+                f"Unsupported manifest schema_version {version!r} "
+                f"(this version reads {SCHEMA_VERSION})."
+            )
+
+        entries = {}
+        for index, raw in enumerate(_require(data, "entries", list, "manifest")):
+            entry = ManifestEntry.from_dict(raw, index)
+            if entry.path in entries:
+                raise ManifestError(f"Duplicate entry for {entry.path!r}")
+            entries[entry.path] = entry
+
+        return cls(entries=entries, created=str(data.get("created", "")))
+
+    @classmethod
+    def from_bytes(cls, raw: bytes, source: Path) -> Manifest:
+        """Parse manifest bytes already read from ``source``."""
+        try:
+            data = json.loads(raw.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise ManifestError(f"Manifest {source} is not valid JSON: {exc}") from exc
+
+        try:
+            return cls.from_dict(data)
+        except ManifestError as exc:
+            raise ManifestError(f"{source}: {exc}") from exc
+
+    @classmethod
+    def load(cls, path: Path) -> Manifest:
+        try:
+            raw = path.read_bytes()
+        except OSError as exc:
+            raise ManifestError(f"Cannot read manifest {path}: {exc}") from exc
+        return cls.from_bytes(raw, path)
+
+    def save(self, path: Path) -> None:
+        """Write atomically so an interrupted run cannot leave a half-written file."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        text = json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n"
+
+        # A fresh, unpredictable, owner-only file (not a fixed ".tmp" name that
+        # someone could pre-create as a link); the manifest lists file names.
+        fd, name = tempfile.mkstemp(
+            dir=path.parent, prefix=path.name + ".", suffix=".tmp"
+        )
+        temp = Path(name)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+                handle.write(text)
+            os.replace(temp, path)
+        except BaseException:
+            temp.unlink(missing_ok=True)
+            raise

batchvault/models.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+
+class Status(StrEnum):
+    SUCCESS = "success"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+    PARTIAL = "partial"
+
+
+@dataclass
+class FileResult:
+    """Outcome of one file in a pack, unpack or verify run.
+
+    For ``pack`` the ``source`` is the original file and ``archive`` its first
+    volume. For ``unpack`` and ``verify`` the ``source`` is the archive (its first
+    volume) and, for ``unpack``, ``restored`` lists the files written.
+    """
+
+    source: Path
+    status: Status
+    message: str = ""
+    size: int = 0
+    volumes: list[Path] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    archive: Path | None = None  # first volume of the archive (pack)
+    stored_size: int = 0  # total size of the volumes (pack)
+    restored: list[Path] = field(default_factory=list)  # files written (unpack)
+    folder: str = ""  # folder of the file under the root ("" = flat)
+    name: str = ""  # the original file's name (pack, unpack and verify)
+    level: int | None = None  # compression level used (pack)
+
+    def to_dict(self) -> dict:
+        return {
+            "source": str(self.source),
+            "name": self.name,
+            "level": self.level,
+            "folder": self.folder,
+            "status": self.status.value,
+            "message": self.message,
+            "size": self.size,
+            "volumes": [str(volume) for volume in self.volumes],
+            "archive": str(self.archive) if self.archive else None,
+            "stored_size": self.stored_size,
+            "restored": [str(path) for path in self.restored],
+            "warnings": self.warnings,
+        }
+
+
+@dataclass
+class RunResult:
+    """Structured outcome of a whole run."""
+
+    entries: list[FileResult] = field(default_factory=list)
+    manifest: Path | None = None
+    warnings: list[str] = field(default_factory=list)
+    engine: str = ""
+
+    @property
+    def status(self) -> Status:
+        statuses = {entry.status for entry in self.entries}
+        if not statuses or statuses == {Status.SUCCESS}:
+            return Status.SUCCESS
+        if statuses == {Status.SKIPPED}:
+            return Status.SKIPPED
+        if Status.SUCCESS in statuses:
+            return Status.PARTIAL
+        return Status.FAILED
+
+    def count(self, status: Status) -> int:
+        return sum(1 for entry in self.entries if entry.status == status)
+
+    def to_dict(self) -> dict:
+        return {
+            "status": self.status.value,
+            "engine": self.engine,
+            "manifest": str(self.manifest) if self.manifest else None,
+            "warnings": self.warnings,
+            "entries": [entry.to_dict() for entry in self.entries],
+        }
+
+
+@dataclass(frozen=True)
+class FileEntry:
+    """A file found by ``list_files`` or an archive found by ``list_archives``."""
+
+    path: Path
+    size: int | None = None
+    root: Path | None = None  # folder a file was listed from (``list files``)
+
+    def to_dict(self) -> dict:
+        data = {"path": str(self.path), "size": self.size}
+        if self.root is not None:
+            # Lets ``pack`` keep the listed folder's own name in the archives.
+            data["root"] = str(self.root)
+        return data
+
+
+@dataclass(frozen=True)
+class ProgressEvent:
+    """Progress report. Callbacks run on worker threads and must be thread-safe.
+
+    ``kind`` is ``"plan"`` (once, before work: ``size`` is the total bytes if
+    known and ``count`` the number of files), then ``"start"`` and ``"finish"``
+    for each file.
+    """
+
+    kind: str
+    source: Path
+    size: int = 0
+    status: Status | None = None
+    count: int = 0
+
+    def to_dict(self) -> dict:
+        return {
+            "event": self.kind,
+            "source": str(self.source),
+            "size": self.size,
+            "status": self.status.value if self.status else None,
+            "count": self.count,
+        }
+
+
+@dataclass(frozen=True)
+class InspectEntry:
+    """What the manifest (or, without one, the file names) say about an archive."""
+
+    name: str
+    path: Path  # first volume
+    engine: str
+    format: str | None = None
+    level: int | None = None
+    encrypted: bool | None = None
+    size: int | None = None  # original size
+    stored_size: int | None = None  # total size of the volumes
+    volumes: int = 0
+    complete: bool | None = None  # every listed volume is present
+    folder: str = ""  # folder under the root the file came from ("" = flat)
+
+    @property
+    def ratio(self) -> float | None:
+        if self.size and self.stored_size is not None:
+            return self.stored_size / self.size
+        return None
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name,
+            "folder": self.folder,
+            "path": str(self.path),
+            "engine": self.engine,
+            "format": self.format,
+            "level": self.level,
+            "encrypted": self.encrypted,
+            "size": self.size,
+            "stored_size": self.stored_size,
+            "ratio": self.ratio,
+            "volumes": self.volumes,
+            "complete": self.complete,
+        }

batchvault/names.py

@@ -0,0 +1,36 @@
+"""Checks on file and folder names before they are written to disk."""
+
+from __future__ import annotations
+
+_WINDOWS_RESERVED = {
+    "con", "prn", "aux", "nul",
+    *(f"com{n}" for n in range(1, 10)),
+    *(f"lpt{n}" for n in range(1, 10)),
+}  # fmt: skip
+_WINDOWS_BAD_CHARACTERS = '<>:"|?*'
+
+
+def unsafe_on_windows(name: str) -> bool:
+    """Names Windows treats specially: drives (``C:x``), streams (``a:b``),
+    devices (``NUL``, ``con.txt``), and names ending in a dot or space."""
+    stem = name.split(".")[0].rstrip(" ").lower()
+    return (
+        any(char in name for char in _WINDOWS_BAD_CHARACTERS)
+        or name != name.rstrip(" .")
+        or stem in _WINDOWS_RESERVED
+    )
+
+
+def validate_folder(folder: str) -> str:
+    """Check a stored folder path (forward slashes, relative, no ``..``).
+
+    Returns it unchanged; raises ValueError if it could point outside the folder
+    it is restored into.
+    """
+    if folder == "":
+        return folder
+    if "\\" in folder or "\x00" in folder or folder.startswith("/"):
+        raise ValueError(f"unsafe folder {folder!r}")
+    if any(part in ("", ".", "..") for part in folder.split("/")):
+        raise ValueError(f"unsafe folder {folder!r}")
+    return folder

batchvault/notify.py

@@ -0,0 +1,73 @@
+"""Webhook notifications. Used by the command-line interface only."""
+
+from __future__ import annotations
+
+import json
+import platform
+import urllib.error
+import urllib.request
+from urllib.parse import urlparse
+
+from batchvault.errors import OptionsError
+from batchvault.models import RunResult, Status
+
+NOTIFY_ON = ("failure", "success", "always")
+TIMEOUT_SECONDS = 10
+
+
+def should_notify(when: str, result: RunResult) -> bool:
+    if when == "always":
+        return True
+    ok = result.status == Status.SUCCESS
+    return ok if when == "success" else not ok
+
+
+def validate_url(url: str) -> str:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.netloc:
+        raise OptionsError("The notification URL must start with http:// or https://")
+    return url
+
+
+def is_insecure(url: str) -> bool:
+    """True for plain http:// to anything other than this machine."""
+    parsed = urlparse(url)
+    local = parsed.hostname in ("localhost", "127.0.0.1", "::1")
+    return parsed.scheme == "http" and not local
+
+
+def build_payload(command: str, result: RunResult) -> dict:
+    counts = ", ".join(
+        f"{result.count(status)} {status.value}"
+        for status in (Status.SUCCESS, Status.SKIPPED, Status.FAILED)
+        if result.count(status)
+    )
+    text = (
+        f"BatchVault {command} on {platform.node()}: "
+        f"{result.status.value} ({counts or 'nothing to do'})"
+    )
+    # "text" is read by Slack and Teams, "content" by Discord.
+    return {
+        "text": text,
+        "content": text,
+        "command": command,
+        "host": platform.node(),
+        "result": result.to_dict(),
+    }
+
+
+def send(url: str, payload: dict) -> str | None:
+    """POST the payload as JSON. Returns an error message, or None on success."""
+    request = urllib.request.Request(
+        validate_url(url),
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS):
+            return None
+    except urllib.error.HTTPError as exc:
+        return f"the server answered {exc.code}"
+    except (urllib.error.URLError, OSError, ValueError) as exc:
+        return str(getattr(exc, "reason", exc))

batchvault/options.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from batchvault.errors import OptionsError
+
+KIB = 1024
+MIB = 1024 * KIB
+GIB = 1024 * MIB
+
+_SIZE_UNITS = {"": 1, "k": KIB, "m": MIB, "g": GIB}
+_SIZE_RE = re.compile(r"^\s*(\d+)\s*([kmg]?)b?\s*$", re.IGNORECASE)
+
+
+# Compression formats of the native engine. The 7-Zip engine always writes 7z.
+FORMATS = ("xz", "gzip", "bzip2", "store")
+
+
+def parse_size(value: str | int) -> int:
+    """Convert a size such as ``10m`` or ``1048576`` to bytes."""
+    if isinstance(value, int):
+        size = value
+    else:
+        match = _SIZE_RE.match(value)
+        if not match:
+            raise OptionsError(
+                f"Invalid size {value!r}. Use a number with an optional "
+                "k, m or g suffix, e.g. 500k, 10m, 2g."
+            )
+        size = int(match.group(1)) * _SIZE_UNITS[match.group(2).lower()]
+
+    if size < 1:
+        raise OptionsError("Size must be at least 1 byte.")
+    return size
+
+
+def format_size(size: int | None) -> str:
+    """Human-readable size such as ``12.3 MiB``; ``-`` when unknown."""
+    if size is None:
+        return "-"
+    value = float(size)
+    for unit in ("B", "KiB", "MiB", "GiB"):
+        if value < 1024:
+            return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}"
+        value /= 1024
+    return f"{value:.1f} TiB"
+
+
+@dataclass(frozen=True)
+class PackOptions:
+    """Settings for creating an archive."""
+
+    level: int = 5  # a fixed level; "auto" is resolved per file before this
+    volume_size: int = 10 * MIB
+    jobs: int = 0
+    force: bool = False
+    format: str = "xz"
+    password: str | None = field(default=None, repr=False)
+
+    def __post_init__(self):
+        if self.format not in FORMATS:
+            raise OptionsError(
+                f"Unknown format {self.format!r}. Choose from: {', '.join(FORMATS)}."
+            )
+        if not 0 <= self.level <= 9:
+            raise OptionsError("Compression level must be between 0 and 9.")
+        if self.volume_size < 1:
+            raise OptionsError("Volume size must be at least 1 byte.")
+        if self.jobs < 0:
+            raise OptionsError("Jobs cannot be negative.")

batchvault/security.py

@@ -0,0 +1,40 @@
+import os
+import re
+from pathlib import Path
+
+# Credentials embedded in a URL: scheme://user:password@host
+_URL_CREDENTIALS = re.compile(r"(?<=://)[^/\s:@]+(:[^/\s@]*)?@")
+
+
+def redact(text: str) -> str:
+    """Hide credentials embedded in URLs."""
+    return _URL_CREDENTIALS.sub("***@", text)
+
+
+SECRET_OPTIONS = ("--notify",)
+
+
+def redact_arguments(argv: list[str]) -> list[str]:
+    """Hide the values of options that carry secrets (webhook URLs contain one)."""
+    hidden = []
+    mask_next = False
+    for arg in argv:
+        if mask_next:
+            hidden.append("***")
+            mask_next = False
+        elif arg in SECRET_OPTIONS:
+            hidden.append(arg)
+            mask_next = True
+        elif any(arg.startswith(f"{option}=") for option in SECRET_OPTIONS):
+            hidden.append(arg.split("=", 1)[0] + "=***")
+        else:
+            hidden.append(redact(arg))
+    return hidden
+
+
+def restrict_permissions(path: Path) -> None:
+    """Make a file readable by its owner only (no effect on Windows)."""
+    try:
+        os.chmod(path, 0o600)
+    except OSError:
+        pass