batchvault 0.1.0__py3-none-any.whl

batchvault/filters.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+import pathspec
+
+from batchvault.errors import InputError
+
+
+def load_patterns(values: Iterable[str]) -> list[str]:
+    """Expand ``@file`` references into the patterns listed in that file."""
+    patterns = []
+    for value in values:
+        if value.startswith("@"):
+            path = Path(value[1:]).expanduser()
+            try:
+                lines = path.read_text(encoding="utf-8-sig").splitlines()
+            except OSError as exc:
+                raise InputError(f"Cannot read pattern file {path}: {exc}") from exc
+            patterns.extend(
+                line.strip()
+                for line in lines
+                if line.strip() and not line.lstrip().startswith("#")
+            )
+        elif value.strip():
+            patterns.append(value.strip())
+    return patterns
+
+
+def _spec(patterns: list[str]):
+    return pathspec.GitIgnoreSpec.from_lines(patterns) if patterns else None
+
+
+class PathFilter:
+    """Include/exclude filter using gitignore-style patterns (``*``, ``**``, ``/``).
+
+    Exclude wins. If any include pattern is given, only matching paths are kept.
+    """
+
+    def __init__(self, include: Iterable[str] = (), exclude: Iterable[str] = ()):
+        self._include = _spec(load_patterns(include))
+        self._exclude = _spec(load_patterns(exclude))
+
+    def matches(self, path: str) -> bool:
+        path = path.replace("\\", "/").lstrip("/")
+        if self._exclude and self._exclude.match_file(path):
+            return False
+        if self._include and not self._include.match_file(path):
+            return False
+        return True

batchvault/inputs.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import json
+import os
+import sys
+from collections.abc import Iterable
+from pathlib import Path
+
+from batchvault.errors import InputError
+from batchvault.filters import PathFilter
+from batchvault.manifest import MANIFEST_NAME
+from batchvault.volumes import split_volume_name
+
+# A text list written by ``list files`` starts with this comment. It records the
+# folder the files were listed from, so ``pack`` keeps that folder's name.
+ROOT_HEADER = "# batchvault-root:"
+
+ListItem = tuple[str, str, str | None]  # (label, path, root the list recorded)
+
+
+def read_list_items(list_path: Path) -> list[ListItem]:
+    """Read a list file as ``(label, path, root)`` items.
+
+    Text lists have one path per line; blank lines and ``#`` comments are
+    ignored, except the ``# batchvault-root:`` header written by ``list files``.
+    JSON lists are an array of paths or ``{"path": ..., "root": ...}`` objects (as
+    printed by ``batchvault list files``), or an object with such an array under
+    ``"files"``. ``root`` is the folder the file was listed from, if recorded.
+    """
+    if str(list_path) == "-":
+        list_path = Path("<stdin>")
+        text = sys.stdin.read().lstrip("")
+    else:
+        try:
+            text = list_path.read_text(encoding="utf-8-sig")
+        except OSError as exc:
+            raise InputError(f"Cannot read list file {list_path}: {exc}") from exc
+
+    if list_path.suffix.lower() == ".json" or text.lstrip().startswith(("[", "{")):
+        return _json_items(list_path, text)
+
+    items = []
+    root = None
+    for number, line in enumerate(text.splitlines(), start=1):
+        entry = line.strip()
+        if entry.startswith(ROOT_HEADER):
+            root = entry[len(ROOT_HEADER) :].strip() or None
+        elif entry and not entry.startswith("#"):
+            items.append((f"{list_path}:{number}", entry, root))
+    return items
+
+
+def read_list_entries(list_path: Path) -> list[tuple[str, str]]:
+    """Read a list file as ``(label, path)`` pairs (see ``read_list_items``)."""
+    return [(label, entry) for label, entry, _ in read_list_items(list_path)]
+
+
+def _json_items(list_path: Path, text: str) -> list[ListItem]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise InputError(f"{list_path} is not valid JSON: {exc}") from exc
+
+    default_root = None
+    if isinstance(data, dict):
+        default_root = data.get("root") if isinstance(data.get("root"), str) else None
+        data = data.get("files")
+    if not isinstance(data, list):
+        raise InputError(f"{list_path}: expected a JSON array of paths")
+
+    items = []
+    for index, item in enumerate(data, start=1):
+        value = item.get("path") if isinstance(item, dict) else item
+        if not isinstance(value, str) or not value.strip():
+            raise InputError(f"{list_path}: item {index} has no path")
+        root = item.get("root") if isinstance(item, dict) else None
+        if not isinstance(root, str) or not root.strip():
+            root = default_root
+        items.append((f"{list_path}[{index}]", value.strip(), root))
+    return items
+
+
+def relative_folder(path: Path, root: Path) -> str:
+    """The folder of ``path`` relative to ``root``, with forward slashes.
+
+    Returns ``""`` when the file sits directly in ``root``.
+    """
+    try:
+        relative = path.parent.relative_to(root)
+    except ValueError:
+        raise InputError(
+            f"{path} is not inside the root folder {root}. Choose a root that "
+            "contains all the files, or leave --root out."
+        ) from None
+    return relative.as_posix() if relative.parts else ""
+
+
+def assign_folders(
+    items: list[tuple[Path, Path | None]], root: Path | None = None
+) -> dict[Path, str]:
+    """Decide the folder each file is stored under (``""`` = not in a subfolder).
+
+    - With an explicit ``root``, every file is stored relative to it.
+    - A file from a ``list files`` list is stored relative to the folder that
+      listing recorded, so the listed folder's own name is kept.
+    - Other files are stored relative to the deepest folder they all share, so the
+      folders below it are kept and files directly in it are not nested.
+    """
+    if root is not None:
+        return {path: relative_folder(path, root) for path, _ in items}
+
+    folders: dict[Path, str] = {}
+    unlisted = []
+    for path, recorded in items:
+        if recorded is not None:
+            try:
+                folders[path] = relative_folder(path, recorded)
+                continue
+            except InputError:
+                pass  # the list was edited since it was made; use the shared folder
+        unlisted.append(path)
+
+    if unlisted:
+        try:
+            shared = Path(os.path.commonpath([str(p.parent) for p in unlisted]))
+        except ValueError:
+            raise InputError(
+                "These files are on different drives, so there is no folder they "
+                "share. Archive each drive separately, or pass --root."
+            ) from None
+        for path in unlisted:
+            folders[path] = relative_folder(path, shared)
+    return folders
+
+
+def looks_like_archive(path: Path) -> bool:
+    return split_volume_name(path.name) is not None or path.name == MANIFEST_NAME
+
+
+def gather_sources(
+    files: Iterable[str | Path] = (),
+    list_file: str | Path | None = None,
+    *,
+    path_filter: PathFilter | None = None,
+    allow_archives: bool = False,
+    root: Path | None = None,
+) -> list[tuple[Path, str]]:
+    """Turn arguments and/or a list file into unique, existing source files.
+
+    Returns ``(file, folder)`` pairs, where ``folder`` is the folder the file is
+    stored under (see ``assign_folders``). Relative paths are resolved from the
+    current directory. Files with the same name are fine when their folders
+    differ.
+    """
+    if root is not None:
+        root = Path(root).expanduser().resolve()
+
+    items: list[ListItem] = [("argument", str(file), None) for file in files]
+    if list_file is not None:
+        items += read_list_items(
+            Path(list_file) if str(list_file) == "-" else Path(list_file).expanduser()
+        )
+
+    sources: dict[str, tuple[Path, Path | None]] = {}
+
+    for label, entry, recorded in items:
+        path = Path(entry).expanduser()
+        if not path.is_absolute():
+            path = Path.cwd() / path
+        if not path.is_file():
+            raise InputError(f"{label}: source file not found: {entry}")
+
+        path = path.resolve()
+        if looks_like_archive(path) and not allow_archives:
+            raise InputError(
+                f"{label}: {path.name} looks like an archive volume or manifest. "
+                "Use allow_archives / --allow-archives to archive it anyway."
+            )
+        if path_filter and not path_filter.matches(path.as_posix()):
+            continue
+
+        key = os.path.normcase(str(path))
+        if key not in sources:
+            hint = Path(recorded).expanduser().resolve() if recorded else None
+            sources[key] = (path, hint)
+
+    if not sources:
+        raise InputError("No input files to archive.")
+
+    folders = assign_folders(list(sources.values()), root)
+
+    # Each file gets its own archive folder named after it, inside its stored
+    # folder; two files may not end up in the same place.
+    taken: dict[tuple[str, str], Path] = {}
+    for path, _ in sources.values():
+        place = (folders[path].casefold(), path.name.casefold())
+        if place in taken:
+            raise InputError(
+                "Two source files would use the same archive folder: "
+                f"{taken[place]} and {path}. Archive them separately, or keep "
+                "their folders apart with --root."
+            )
+        taken[place] = path
+
+    return [(path, folders[path]) for path, _ in sources.values()]
+
+
+def collect_sources(
+    files: Iterable[str | Path] = (),
+    list_file: str | Path | None = None,
+    *,
+    path_filter: PathFilter | None = None,
+    allow_archives: bool = False,
+    root: Path | None = None,
+) -> list[Path]:
+    """Like ``gather_sources``, returning only the files."""
+    return [
+        path
+        for path, _ in gather_sources(
+            files,
+            list_file,
+            path_filter=path_filter,
+            allow_archives=allow_archives,
+            root=root,
+        )
+    ]

batchvault/levels.py

@@ -0,0 +1,169 @@
+"""Pick a compression level for each file from its type (``--level auto``).
+
+The choice uses only the file's signature (its first bytes) and extension, so it
+is instant and predictable. Level 0 means "do not try": the file is already
+compressed, so a stronger level would only cost time and memory.
+"""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from batchvault.errors import OptionsError
+
+AUTO = "auto"
+DEFAULT_LEVEL = 5
+
+TEXT_LEVEL = 6
+MEDIA_LEVEL = 6
+BINARY_LEVEL = 5
+PDF_LEVEL = 3
+INCOMPRESSIBLE_LEVEL = 0
+
+
+@dataclass(frozen=True)
+class LevelChoice:
+    level: int
+    kind: str  # a short reason, shown next to the result
+
+
+def parse_level(value: int | str) -> int | str:
+    """Validate a level: 0 to 9, or ``"auto"``."""
+    if isinstance(value, str):
+        text = value.strip().lower()
+        if text == AUTO:
+            return AUTO
+        try:
+            value = int(text)
+        except ValueError:
+            raise OptionsError(
+                f"Invalid compression level {value!r}. Use 0-9 or auto."
+            ) from None
+
+    if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 9:
+        raise OptionsError("Compression level must be between 0 and 9, or auto.")
+    return value
+
+
+def level_argument(text: str) -> int | str:
+    """argparse type for ``--level``; the range is checked by ``choices``."""
+    value = text.strip().lower()
+    if value == AUTO:
+        return AUTO
+    try:
+        return int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(
+            f"invalid level {text!r}: use 0-9 or auto"
+        ) from None
+
+
+def _extensions(text: str) -> frozenset[str]:
+    return frozenset(text.split())
+
+
+# Formats that are already compressed (or encrypted): nothing left to squeeze.
+INCOMPRESSIBLE = _extensions(
+    """
+    zip 7z rar gz tgz xz txz bz2 tbz2 zst lz4 lzma z cab jar war ear apk ipa
+    docx xlsx pptx odt ods odp epub
+    jpg jpeg png gif webp heic heif avif cr2 nef arw dng orf rw2
+    mp4 m4v mov mkv webm avi wmv flv mpg mpeg m2ts 3gp
+    mp3 aac m4a ogg oga opus flac wma wv
+    """
+)
+
+# Plain text and similar: very repetitive, so a strong level pays off.
+TEXT = _extensions(
+    """
+    txt log csv tsv json jsonl ndjson xml html htm xhtml css scss sass less
+    js mjs jsx tsx md markdown rst yaml yml toml ini cfg conf properties sql
+    py pyi java kt c h cc cpp hpp cs go rs rb php swift sh bash zsh bat cmd ps1
+    tex bib srt vtt ass svg rtf eml ics vcf gpx kml diff patch gradle tf
+    ipynb r pl lua dart scala vue
+    """
+)
+
+# Uncompressed pictures and sound.
+UNCOMPRESSED_MEDIA = _extensions(
+    "wav aiff aif bmp tif tiff ppm pgm pbm tga dds pcm psd"
+)
+
+# Programs, disk images, databases and containers: compressible, but the
+# strongest levels gain little for the memory they need.
+BINARY = _extensions(
+    """
+    exe dll so dylib o a lib class pyc wasm iso img vhd vhdx vmdk qcow2 vdi
+    db sqlite sqlite3 mdb bak dump pak tar
+    """
+)
+
+PDF = _extensions("pdf")
+
+# (offset, bytes) at the start of a file that mark an already compressed format.
+_SIGNATURES = (
+    (0, b"PK\x03\x04"),  # zip, docx, jar, apk, epub, ...
+    (0, b"PK\x05\x06"),
+    (0, b"\x1f\x8b"),  # gzip
+    (0, b"\xfd7zXZ\x00"),  # xz
+    (0, b"7z\xbc\xaf\x27\x1c"),
+    (0, b"Rar!\x1a\x07"),
+    (0, b"\x28\xb5\x2f\xfd"),  # zstd
+    (0, b"\x04\x22\x4d\x18"),  # lz4
+    (0, b"MSCF"),  # cab
+    (0, b"\xff\xd8\xff"),  # jpeg
+    (0, b"\x89PNG\r\n\x1a\n"),
+    (0, b"GIF87a"),
+    (0, b"GIF89a"),
+    (4, b"ftyp"),  # mp4, mov, heic, m4a
+    (0, b"\x1a\x45\xdf\xa3"),  # matroska, webm
+    (0, b"OggS"),
+    (0, b"fLaC"),
+    (0, b"\x00\x00\x01\xba"),  # mpeg program stream
+)
+
+
+def _has_compressed_signature(head: bytes) -> bool:
+    if any(head[at : at + len(mark)] == mark for at, mark in _SIGNATURES):
+        return True
+    if head[:4] == b"RIFF" and head[8:12] == b"WEBP":
+        return True
+    # bzip2: "BZh", a block size digit and the block magic. The longer check keeps
+    # a text file that happens to start with "BZh" from being misjudged.
+    if head[:3] == b"BZh" and head[3:4] in b"123456789" and head[4:10] == b"1AY&SY":
+        return True
+    # ID3v2 tag (mp3): version, flags, then a size whose bytes are below 0x80.
+    return (
+        head[:3] == b"ID3"
+        and head[3:4] in (b"\x02", b"\x03", b"\x04")
+        and head[4:5] == b"\x00"
+        and len(head) >= 10
+        and all(byte < 0x80 for byte in head[6:10])
+    )
+
+
+def choose_level(path: Path) -> LevelChoice:
+    """The best compression level for this file, judged by its type."""
+    try:
+        with path.open("rb") as handle:
+            head = handle.read(16)
+    except OSError:
+        head = b""
+
+    if _has_compressed_signature(head):
+        return LevelChoice(INCOMPRESSIBLE_LEVEL, "already compressed")
+
+    extension = path.suffix.lower().lstrip(".")
+    if extension in INCOMPRESSIBLE:
+        return LevelChoice(INCOMPRESSIBLE_LEVEL, "already compressed")
+    if extension in TEXT:
+        return LevelChoice(TEXT_LEVEL, "text")
+    if extension in UNCOMPRESSED_MEDIA:
+        return LevelChoice(MEDIA_LEVEL, "uncompressed media")
+    if extension in BINARY:
+        return LevelChoice(BINARY_LEVEL, "binary data")
+    if extension in PDF:
+        return LevelChoice(PDF_LEVEL, "pdf")
+    return LevelChoice(DEFAULT_LEVEL, "unknown type")

batchvault/locate.py

@@ -0,0 +1,140 @@
+"""Work out which archives a path given to unpack, verify or list refers to."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from batchvault.errors import InputError, ManifestError
+from batchvault.inputs import read_list_entries
+from batchvault.manifest import MANIFEST_NAME, Manifest, ManifestEntry
+from batchvault.volumes import split_volume_name
+
+
+@dataclass
+class Target:
+    """One archive to process. ``entry`` is None when no manifest is available."""
+
+    volume: Path  # first volume (or the given one when there is no manifest)
+    entry: ManifestEntry | None = None
+    base_dir: Path | None = None  # folder the entry's volume paths are relative to
+    manifest: Path | None = None  # the manifest file the entry came from
+
+    @property
+    def name(self) -> str:
+        if self.entry:
+            return self.entry.name
+        base = split_volume_name(self.volume.name)[0]
+        return base.rsplit(".", 1)[0]
+
+
+def _same(a: Path, b: Path) -> bool:
+    try:
+        return a.resolve() == b.resolve()
+    except OSError:
+        return False
+
+
+def _find_manifest(*folders: Path) -> Path | None:
+    for folder in folders:
+        candidate = folder / MANIFEST_NAME
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+Loader = Callable[[Path], Manifest]
+
+
+def _manifest_targets(
+    manifest_path: Path, under: Path | None = None, load: Loader = Manifest.load
+) -> list[Target]:
+    manifest = load(manifest_path)
+    base = manifest_path.parent
+
+    targets = []
+    for entry in manifest.entries.values():
+        first = base / entry.volumes[0].file
+        if under is None or _is_under(first, under):
+            targets.append(Target(first, entry, base, manifest_path))
+    return targets
+
+
+def _is_under(path: Path, folder: Path) -> bool:
+    try:
+        return path.resolve().is_relative_to(folder.resolve())
+    except OSError:
+        return False
+
+
+def _volume_target(volume: Path, load: Loader = Manifest.load) -> Target:
+    """Target for one volume path, using a nearby manifest when it knows it."""
+    if split_volume_name(volume.name) is None:
+        raise InputError(f"Not an archive volume: {volume}")
+    if not volume.is_file():
+        raise InputError(f"Archive volume not found: {volume}")
+
+    manifest_path = _find_manifest(volume.parent, volume.parent.parent)
+    if manifest_path:
+        base = manifest_path.parent
+        for entry in load(manifest_path).entries.values():
+            if any(_same(base / v.file, volume) for v in entry.volumes):
+                return Target(base / entry.volumes[0].file, entry, base, manifest_path)
+
+    return Target(volume)
+
+
+def resolve_targets(
+    source: str | Path, load: Loader = Manifest.load
+) -> list[Target]:
+    """Archives referred to by a manifest, folder, volume file or list file.
+
+    Every manifest is read through ``load``, so a caller can verify a manifest's
+    signature on the very bytes that are then parsed.
+    """
+    path = Path(source).expanduser()
+    if not path.exists():
+        raise InputError(f"Path not found: {path}")
+
+    if path.is_dir():
+        manifest_path = _find_manifest(path, path.parent)
+        if manifest_path:
+            under = None if manifest_path.parent == path else path
+            targets = _manifest_targets(manifest_path, under, load)
+        else:
+            targets = [
+                Target(found)
+                for found in sorted(path.rglob("*"))
+                if found.is_file()
+                and (parsed := split_volume_name(found.name))
+                and parsed[1] == 1
+            ]
+        if not targets:
+            raise InputError(f"No archives found in {path}")
+        return targets
+
+    if split_volume_name(path.name):
+        return [_volume_target(path, load)]
+
+    if path.name == MANIFEST_NAME or path.suffix.lower() == ".json":
+        try:
+            return _manifest_targets(path, None, load)
+        except ManifestError:
+            # A JSON list of volume paths is also accepted; fall through.
+            if path.name == MANIFEST_NAME:
+                raise
+
+    targets = []
+    for label, entry in read_list_entries(path):
+        volume = Path(entry).expanduser()
+        if not volume.is_absolute():
+            volume = path.parent / volume
+        try:
+            targets.append(_volume_target(volume, load))
+        except InputError as exc:
+            raise InputError(f"{label}: {exc}") from exc
+
+    if not targets:
+        raise InputError(f"No archive paths found in {path}")
+    return targets

batchvault/logs.py

@@ -0,0 +1,86 @@
+import logging
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from batchvault.security import redact, restrict_permissions
+
+LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR")
+DEFAULT_LEVEL = "INFO"
+LOGGER_NAME = "batchvault"
+ENV_LOG_DIR = "BATCHVAULT_LOG_DIR"
+KEEP_LOGS = 20
+
+
+class RedactingFormatter(logging.Formatter):
+    """Hides credentials embedded in URLs, including in tracebacks."""
+
+    def format(self, record):
+        return redact(super().format(record))
+
+
+def default_log_dir() -> Path:
+    """The conventional per-user log folder for the current OS."""
+    override = os.environ.get(ENV_LOG_DIR)
+    if override:
+        return Path(override).expanduser()
+
+    home = Path.home()
+
+    if sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA") or home / "AppData" / "Local")
+        return base / "BatchVault" / "Logs"
+    if sys.platform == "darwin":
+        return home / "Library" / "Logs" / "BatchVault"
+
+    state = os.environ.get("XDG_STATE_HOME")
+    return (Path(state) if state else home / ".local" / "state") / "batchvault"
+
+
+def prune_logs(directory: Path, keep: int = KEEP_LOGS) -> None:
+    """Delete the oldest per-run log files beyond ``keep``."""
+    try:
+        logs = sorted(directory.glob("batchvault-*.log"))
+        for old in logs[:-keep] if keep else logs:
+            old.unlink(missing_ok=True)
+    except OSError:
+        pass
+
+
+def setup_logging(level=DEFAULT_LEVEL, log_file=None):
+    """Log to a file for this run. Returns its path, or None if it can't be opened."""
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(getattr(logging, level.upper()))
+    close_logging()
+
+    if log_file:
+        path = Path(log_file).expanduser()
+    else:
+        stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        path = default_log_dir() / f"batchvault-{stamp}-{os.getpid()}.log"
+
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        handler = logging.FileHandler(path, encoding="utf-8")
+    except OSError:
+        logger.addHandler(logging.NullHandler())
+        return None
+
+    restrict_permissions(path)
+    handler.setFormatter(
+        RedactingFormatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s")
+    )
+    logger.addHandler(handler)
+
+    if not log_file:
+        prune_logs(path.parent)
+    return path.resolve()
+
+
+def close_logging() -> None:
+    logger = logging.getLogger(LOGGER_NAME)
+    for handler in list(logger.handlers):
+        handler.close()
+        logger.removeHandler(handler)
+    logger.addHandler(logging.NullHandler())