zipctl 0.1.0__py3-none-any.whl

zipctl/cli/commands/helpers/progress.py

@@ -0,0 +1,215 @@
+"""Rendering :class:`~zipctl.ProgressEvent` on standard error."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import time
+from collections.abc import Callable, Generator, Iterable
+from contextlib import contextmanager
+from typing import Protocol, TextIO
+from unicodedata import east_asian_width
+
+from zipctl.cli.output import human_size, printable
+from zipctl.zipfile.extract import MemberStatus
+from zipctl.zipfile.progress import ProgressCallback, ProgressEvent, ProgressPhase
+
+__all__ = [
+    "ProgressArgs",
+    "ProgressRenderer",
+    "StepReporter",
+    "add_progress_option",
+    "progress_renderer",
+]
+
+_FALLBACK_WIDTH = 80
+_REDRAW_INTERVAL = 0.05  # seconds between redraws of the live status line
+_STATUS_WORDS = {
+    MemberStatus.EXTRACTED: "OK",
+    MemberStatus.PREVIEWED: "OK",
+    MemberStatus.SKIPPED: "SKIP",
+    MemberStatus.FAILED: "FAILED",
+}
+
+
+def _columns(text: str) -> int:
+    """Terminal columns *text* takes: East Asian wide characters take two."""
+    return sum(2 if east_asian_width(char) in "WF" else 1 for char in text)
+
+
+def _terminal_width(stream: TextIO) -> int:
+    """Columns of the terminal *stream* is attached to (it is not standard output)."""
+    try:
+        return os.get_terminal_size(stream.fileno()).columns or _FALLBACK_WIDTH
+    except (OSError, ValueError, AttributeError):  # not a terminal, or no fileno
+        return _FALLBACK_WIDTH
+
+
+def _prefix(text: str, columns: int) -> str:
+    """The longest start of *text* that fits in *columns*."""
+    used = 0
+    for index, char in enumerate(text):
+        used += _columns(char)
+        if used > columns:
+            return text[:index]
+    return text
+
+
+def _shorten(text: str, width: int) -> str:
+    """Trim *text* to *width* columns, keeping both ends of long names."""
+    if width < 8 or _columns(text) <= width:
+        return _prefix(text, max(width, 0))
+    keep = width - 3
+    head = keep // 2
+    tail = _prefix(text[::-1], keep - head)[::-1]
+    return f"{_prefix(text, head)}...{tail}"
+
+
+class ProgressRenderer:
+    """A ``progress=`` callback that reports on *stream*.
+
+    On a terminal it keeps one status line up to date (``\\r`` rewrites); when
+    output is redirected it prints one plain line per finished member instead
+    (``[1/3] OK      name``), so logs stay readable.  The live line is redrawn at
+    most every 50 ms, except for the first event and the last member finishing,
+    which are always drawn.
+    """
+
+    def __init__(
+        self,
+        stream: TextIO,
+        *,
+        live: bool,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self._stream: TextIO = stream
+        self._live: bool = live
+        self._clock: Callable[[], float] = clock
+        self._drawn: int = 0
+        self._last_draw: float | None = None
+
+    def __call__(self, event: ProgressEvent) -> None:
+        if self._live:
+            self._draw(event)
+        elif event.phase is ProgressPhase.FINISH and event.status is not None:
+            print(
+                f"[{event.member_index + 1}/{event.member_count}] "
+                f"{_STATUS_WORDS[event.status]:<8}{printable(event.member)}",
+                file=self._stream,
+            )
+
+    def _due(self, event: ProgressEvent) -> bool:
+        if self._last_draw is None:
+            return True
+        last = (
+            event.phase is ProgressPhase.FINISH
+            and event.member_index + 1 == event.member_count
+        )
+        return last or self._clock() - self._last_draw >= _REDRAW_INTERVAL
+
+    def _draw(self, event: ProgressEvent) -> None:
+        if not self._due(event):
+            return
+        self._last_draw = self._clock()
+        if event.total_bytes:
+            percent = min(100, event.total_bytes_done * 100 // event.total_bytes)
+        else:
+            percent = 100 if event.phase is ProgressPhase.FINISH else 0
+        tail = (
+            f"  {percent:3d}%  {human_size(event.total_bytes_done)}"
+            f"/{human_size(event.total_bytes)}"
+        )
+        head = f"[{event.member_index + 1}/{event.member_count}] "
+        width = _terminal_width(self._stream) - 1
+        name = _shorten(printable(event.member), width - len(head) - len(tail))
+        self._write(head + name + tail)
+
+    def _write(self, line: str) -> None:
+        columns = _columns(line)
+        padding = " " * max(0, self._drawn - columns)
+        self._stream.write(f"\r{line}{padding}")
+        self._stream.flush()
+        self._drawn = columns
+
+    def clear(self) -> None:
+        """Erase the status line, e.g. before a password prompt."""
+        if self._live and self._drawn:
+            self._stream.write("\r" + " " * self._drawn + "\r")
+            self._stream.flush()
+            self._drawn = 0
+        self._last_draw = None  # what comes next is drawn at once
+
+    def close(self) -> None:
+        """Same as :meth:`clear`."""
+        self.clear()
+
+
+class ProgressArgs(Protocol):
+    """What :func:`add_progress_option` leaves on the parsed arguments."""
+
+    progress: bool
+
+
+def add_progress_option(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--progress", action="store_true", help="report progress on standard error"
+    )
+
+
+@contextmanager
+def progress_renderer(
+    stream: TextIO, enabled: bool
+) -> Generator[ProgressRenderer | None]:
+    """The renderer for ``--progress`` (``None`` without it), closed on the way out."""
+    renderer = ProgressRenderer(stream, live=stream.isatty()) if enabled else None
+    try:
+        yield renderer
+    finally:
+        if renderer is not None:
+            renderer.close()
+
+
+class StepReporter:
+    """Progress events for a command that does its own work member by member.
+
+    Extraction reports progress from inside the library; ``test`` and ``create``
+    call :meth:`start` and :meth:`finish` around each member instead.  Sizes are
+    the declared ones, and there is one event pair per member (no byte-level
+    progress inside a member).  Without a *callback* every call does nothing,
+    and *members* is never read (so it may be a lazy iterable).
+    """
+
+    def __init__(
+        self, callback: ProgressCallback | None, members: Iterable[tuple[str, int]]
+    ) -> None:
+        self._callback: ProgressCallback | None = callback
+        self._members: list[tuple[str, int]] = list(members) if callback else []
+        self._total: int = sum(size for _, size in self._members)
+        self._done: int = 0
+
+    def _event(
+        self, phase: ProgressPhase, index: int, status: MemberStatus | None = None
+    ) -> ProgressEvent:
+        name, size = self._members[index]
+        finished = phase is ProgressPhase.FINISH
+        return ProgressEvent(
+            phase=phase,
+            member=name,
+            member_index=index,
+            member_count=len(self._members),
+            member_bytes=size if finished else 0,
+            member_total=size,
+            total_bytes=self._total,
+            total_bytes_done=self._done,
+            status=status,
+        )
+
+    def start(self, index: int) -> None:
+        if self._callback:
+            self._callback(self._event(ProgressPhase.START, index))
+
+    def finish(self, index: int, *, ok: bool) -> None:
+        if self._callback:
+            self._done += self._members[index][1]
+            status = MemberStatus.EXTRACTED if ok else MemberStatus.FAILED
+            self._callback(self._event(ProgressPhase.FINISH, index, status))

zipctl/cli/commands/helpers/selection.py

@@ -0,0 +1,96 @@
+"""Choosing archive members with shell-style patterns."""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_left
+from collections.abc import Callable, Sequence
+
+from zipctl.cli.errors import UsageError
+from zipctl.cli.output import did_you_mean, printable
+from zipctl.zipfile.info import ZipInfo
+from zipctl.zipfile.path.glob import Translator
+
+__all__ = ["MEMBER_HELP", "glob_matcher", "select", "select_infos"]
+
+MEMBER_HELP = (
+    "Member names or glob patterns; a directory name picks everything under it. "
+    "Default: every member"
+)
+
+_TRANSLATOR = Translator(seps="/")
+_GLOB_CHARS = frozenset("*?[")
+
+
+def glob_matcher(pattern: str) -> Callable[[str], bool]:
+    """Return a predicate for member names matching *pattern*.
+
+    ``*`` and ``?`` stay within one path segment, ``**`` crosses ``/``, and
+    ``[...]`` is a character class.  A name equal to *pattern* always matches,
+    so an archive member whose name contains glob characters can still be
+    named literally.  Raises :class:`CliError` for a malformed pattern.
+    """
+    try:
+        compiled = re.compile(_TRANSLATOR.translate(pattern))
+    except (ValueError, re.error) as exc:
+        raise UsageError(
+            f"invalid pattern {printable(pattern)!r}: {printable(str(exc))}",
+        ) from None
+    return lambda name: name == pattern or compiled.match(name) is not None
+
+
+def _under(ordered: list[str], directory: str) -> list[str]:
+    """The names in sorted *ordered* that start with *directory* (ending in ``/``)."""
+    start = end = bisect_left(ordered, directory)
+    while end < len(ordered) and ordered[end].startswith(directory):
+        end += 1
+    return ordered[start:end]
+
+
+def select(names: Sequence[str], patterns: Sequence[str], what: str) -> set[str]:
+    """The *names* picked by *patterns* (all of them without any).
+
+    A plain name (no glob characters) that names a directory also picks
+    everything under it, as ``7z x`` and ``tar x`` do; ``unzip`` needs ``dir/*``.
+    A pattern that matches nothing is a usage error, raised before any work
+    is done: a typo must not leave members untouched while the command
+    reports success.
+    """
+    if not patterns:
+        return set(names)
+    present = set(names)
+    ordered = sorted(present)
+    chosen: set[str] = set()
+    dead: list[str] = []
+    for pattern in patterns:
+        if _GLOB_CHARS.isdisjoint(pattern):
+            # A plain name matches itself, its directory entry and what is under
+            # it: a lookup instead of scanning every name.  A malformed pattern
+            # needs a glob character, so nothing is left unchecked.
+            hit = present & {pattern, pattern + "/"}
+            if directory := pattern.rstrip("/"):
+                hit.update(_under(ordered, directory + "/"))
+        else:
+            matches = glob_matcher(pattern)
+            hit = {name for name in names if matches(name)}
+        if not hit:
+            dead.append(pattern)
+        chosen |= hit
+    if dead:
+        listed = ", ".join(
+            f"'{printable(pattern)}'"
+            + ("" if _GLOB_CHARS & set(pattern) else did_you_mean(pattern, ordered))
+            for pattern in dead
+        )
+        raise UsageError(f"no {what} matches {listed}")
+    return chosen
+
+
+def select_infos(
+    infos: Sequence[ZipInfo], patterns: Sequence[str], what: str = "member"
+) -> list[ZipInfo]:
+    """The *infos* whose names *patterns* pick, in archive order."""
+    if not patterns:
+        return list(infos)
+    chosen = select([info.filename for info in infos], patterns, what)
+    return [info for info in infos if info.filename in chosen]

zipctl/cli/commands/helpers/sources.py

@@ -0,0 +1,32 @@
+"""Reading text that a command line option names by file or standard input."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from zipctl.cli.context import Context
+from zipctl.cli.errors import UsageError, os_error_text
+from zipctl.cli.output import printable
+
+__all__ = ["read_text_source"]
+
+
+def read_text_source(source: str, ctx: Context, what: str) -> str:
+    """The UTF-8 text of file *source* (``-`` is standard input), BOM removed.
+
+    *what* names the file in messages, e.g. ``"policy file"``.
+    """
+    if source == "-":
+        named = f"the {what} on standard input"
+        ctx.claim_stdin(f"the {what}")
+        data = ctx.read_stdin()
+    else:
+        named = f"{what} {printable(source)}"
+        try:
+            data = Path(source).read_bytes()
+        except OSError as exc:
+            raise UsageError(f"cannot read {named}: {os_error_text(exc)}") from None
+    try:
+        return data.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        raise UsageError(f"{named} is not valid UTF-8") from None

zipctl/cli/commands/inspect.py

@@ -0,0 +1,145 @@
+"""The ``inspect`` command."""
+
+from __future__ import annotations
+
+import os
+from typing import Protocol
+
+from zipctl.cli.archive import open_archive
+from zipctl.cli.commands.helpers.command import Subparsers, add_command
+from zipctl.cli.commands.helpers.output_options import OutputArgs, add_output_options
+from zipctl.cli.commands.helpers.policy_options import (
+    PolicyArgs,
+    add_policy_options,
+    load_policy,
+)
+from zipctl.cli.context import Context
+from zipctl.cli.errors import EXIT_FAILURE, EXIT_OK
+from zipctl.cli.output import (
+    count,
+    human_size,
+    printable,
+    write_json,
+)
+from zipctl.zipfile.extract import (
+    ViolationAction,
+)
+from zipctl.zipfile.inspection import InspectionResult
+
+
+class InspectArgs(OutputArgs, PolicyArgs, Protocol):
+    archive: str
+    destination: str | None
+
+
+def _section(title: str, names: tuple[str, ...], flagged: set[str]) -> list[str]:
+    """A titled list of *names*, leaving out those a violation already reports."""
+    names = tuple(name for name in names if name not in flagged)
+    if not names:
+        return []
+    return [f"  {title} ({len(names)}):", *(f"    {printable(name)}" for name in names)]
+
+
+def _violations(result: InspectionResult) -> list[str]:
+    if not result.violations:
+        return []
+    return [
+        f"  Violations ({len(result.violations)}):",
+        *(
+            f"    {violation.action.value.upper():<7}{printable(violation.member)}: "
+            f"{printable(violation.message)}"
+            for violation in result.violations
+        ),
+    ]
+
+
+def _render_inspection(
+    ctx: Context,
+    archive: str,
+    destination: str,
+    result: InspectionResult,
+    errors: int,
+    quiet: bool,
+) -> None:
+    if quiet:
+        if errors:  # the details alone: no verdict, as when nothing is wrong
+            ctx.out("Details:")
+            for line in _violations(result):
+                ctx.out(line)
+        return
+    verdict = "OK" if not errors else f"FAILED ({count(errors, 'error')})"
+    directories = sum(1 for member in result.members if member.is_directory)
+    flagged = {violation.member for violation in result.violations}
+    ctx.out(f"Archive:      {printable(archive)}")
+    ctx.out(f"Destination:  {printable(destination)}")
+    ctx.out(
+        f"Members:      {result.total_entries} "
+        f"({count(directories, 'directory', 'directories')})"
+    )
+    ctx.out(
+        f"Size:         {human_size(result.total_uncompressed_size)} "
+        f"({human_size(result.total_compressed_size)} compressed)"
+    )
+    details = [
+        *_violations(result),
+        *_section("Encrypted members", result.encrypted_members, flagged),
+        *_section("Suspicious paths", result.suspicious_paths, flagged),
+        *_section("Symlinks", result.symlinks, flagged),
+        *_section("Special files", result.special_files, flagged),
+        *_section("Large members", result.large_members, flagged),
+        *_section("Ratio outliers", result.compress_ratio_outliers, flagged),
+        *_section("Duplicate names", result.duplicate_member_names, flagged),
+    ]
+    ctx.out(f"Result:       {verdict}")
+    if details:
+        ctx.out("")
+        ctx.out("Details:")
+        for line in details:
+            ctx.out(line)
+
+
+def cmd_inspect(args: InspectArgs, ctx: Context) -> int:
+    policy = load_policy(args, ctx)
+    destination = os.path.abspath(args.destination or os.getcwd())
+    with open_archive(args.archive) as zf:
+        result = zf.inspect(destination, policy)
+    errors = sum(v.action is ViolationAction.ERROR for v in result.violations)
+    if args.json:
+        write_json(
+            ctx.stdout,
+            {
+                "ok": errors == 0,
+                "archive": args.archive,
+                "destination": destination,
+                "inspection": result,
+            },
+        )
+    else:
+        _render_inspection(
+            ctx,
+            args.archive,
+            destination,
+            result,
+            errors,
+            args.quiet,
+        )
+    return EXIT_FAILURE if errors else EXIT_OK
+
+
+def register(subparsers: Subparsers) -> None:
+    parser = add_command(
+        subparsers,
+        "inspect",
+        cmd_inspect,
+        "report what extracting would flag, without reading any data",
+    )
+    parser.add_argument("archive", metavar="ARCHIVE", help="the archive to inspect")
+    parser.add_argument(
+        "-d",
+        "--destination",
+        metavar="DIR",
+        help="directory the members would be extracted into, used to detect "
+        "overwrites. Default: the current directory",
+    )
+    add_output_options(parser, quiet=True)
+    add_policy_options(parser)

zipctl/cli/commands/list.py

@@ -0,0 +1,166 @@
+"""The ``list`` command."""
+
+from __future__ import annotations
+
+import stat
+from dataclasses import dataclass
+from typing import Protocol
+
+from zipctl.cli.archive import open_archive
+from zipctl.cli.commands.helpers.command import Subparsers, add_command
+from zipctl.cli.commands.helpers.output_options import OutputArgs, add_output_options
+from zipctl.cli.commands.helpers.selection import MEMBER_HELP, select_infos
+from zipctl.cli.context import Context
+from zipctl.cli.errors import EXIT_OK
+from zipctl.cli.methods import compression_label, method_of
+from zipctl.cli.output import (
+    count,
+    format_table,
+    human_size,
+    printable,
+    write_json,
+)
+from zipctl.zipfile.file import ZipFile
+from zipctl.zipfile.info import ZipInfo
+
+
+class ListArgs(OutputArgs, Protocol):
+    archive: str
+    members: list[str]
+    long: bool
+
+
+def _timestamp(info: ZipInfo, separator: str) -> str:
+    year, month, day, hour, minute, second = info.date_time
+    date = f"{year:04d}-{month:02d}-{day:02d}"
+    return f"{date}{separator}{hour:02d}:{minute:02d}:{second:02d}"
+
+
+def _mode_text(info: ZipInfo) -> str:
+    mode = info.unix_mode
+    if not mode:
+        return "-"
+    if not stat.S_IFMT(mode):  # permissions only: the kind comes from the name
+        mode |= stat.S_IFDIR if info.is_dir() else stat.S_IFREG
+    return stat.filemode(mode)
+
+
+def _saved(compressed: int, original: int) -> str:
+    """The share of *original* that compression saved (``-`` for no data).
+
+    Like ``unzip -v``: stored data saves ``0%``, data that grew is negative.
+    """
+    return f"{round(100 * (1 - compressed / original))}%" if original else "-"
+
+
+@dataclass(frozen=True)
+class Listed:
+    """One member as ``list --json`` reports it."""
+
+    name: str
+    directory: bool
+    is_symlink: bool
+    size: int
+    compressed_size: int
+    modified: str
+    compression: str
+    encryption: str
+    crc32: str
+    mode: str | None
+    comment: str
+
+
+def _listed(info: ZipInfo) -> Listed:
+    mode = info.unix_mode
+    return Listed(
+        info.filename,
+        info.is_dir(),
+        info.is_symlink(),
+        info.file_size,
+        info.compress_size,
+        _timestamp(info, "T"),
+        compression_label(info),
+        method_of(info).label,
+        f"{info.CRC:08x}",
+        f"{mode:o}" if mode else None,
+        info.comment.decode("utf-8", "replace"),
+    )
+
+
+def _list_json(ctx: Context, zf: ZipFile, archive: str, infos: list[ZipInfo]) -> None:
+    write_json(
+        ctx.stdout,
+        {
+            "ok": True,
+            "archive": archive,
+            "comment": zf.comment.decode("utf-8", "replace"),
+            "member_count": len(infos),
+            "members": [_listed(info) for info in infos],
+        },
+    )
+
+
+def _crc_text(info: ZipInfo) -> str:
+    """The CRC-32, or ``-`` where AE-2 does not store it."""
+    return f"{info.CRC:08x}" if info.stores_crc else "-"
+
+
+def _list_long(ctx: Context, zf: ZipFile, infos: list[ZipInfo]) -> None:
+    encrypted = any(method_of(info).is_encrypted for info in infos)
+    headers = ["Mode", "Length", "Compressed", "Saved", "Modified", "Method"]
+    if encrypted:
+        headers.append("Encryption")
+    headers += ["CRC-32", "Name"]
+    rows: list[list[str]] = []
+    for info in infos:
+        row = [
+            _mode_text(info),
+            str(info.file_size),
+            str(info.compress_size),
+            _saved(info.compress_size, info.file_size),
+            _timestamp(info, " "),
+            compression_label(info),
+        ]
+        if encrypted:
+            row.append(method_of(info).label)
+        rows.append([*row, _crc_text(info), printable(info.filename)])
+    for line in format_table(headers, rows, right_aligned=(1, 2, 3)):
+        ctx.out(line)
+    total = sum(info.file_size for info in infos)
+    stored = sum(info.compress_size for info in infos)
+    footer = f"{count(len(infos), 'member')}, {human_size(total)}"
+    if total:
+        footer += f" ({human_size(stored)} compressed, {_saved(stored, total)} saved)"
+    ctx.out("")
+    ctx.out(footer)
+    if zf.comment:
+        ctx.out("")
+        ctx.out("Comment:")
+        for line in zf.comment.decode("utf-8", "replace").splitlines():
+            ctx.out(f"  {printable(line)}")
+
+
+def cmd_list(args: ListArgs, ctx: Context) -> int:
+    with open_archive(args.archive) as zf:
+        infos = select_infos(zf.infolist(), args.members)
+        if args.json:
+            _list_json(ctx, zf, args.archive, infos)
+        elif args.long:
+            _list_long(ctx, zf, infos)
+        else:
+            for info in infos:
+                ctx.out(printable(info.filename))
+    return EXIT_OK
+
+
+def register(subparsers: Subparsers) -> None:
+    parser = add_command(subparsers, "list", cmd_list, "list archive members")
+    parser.add_argument("archive", metavar="ARCHIVE", help="the archive to list")
+    parser.add_argument("members", nargs="*", metavar="MEMBER", help=MEMBER_HELP)
+    parser.add_argument(
+        "-l",
+        "--long",
+        action="store_true",
+        help="show modes, sizes, space saved, dates, methods and the archive comment",
+    )
+    add_output_options(parser)