zipctl 0.1.0__py3-none-any.whl

zipctl/cli/commands/helpers/copying/copy.py

@@ -0,0 +1,211 @@
+"""Reading the input archive and writing every member into the new one."""
+
+from __future__ import annotations
+
+import zlib
+from collections.abc import Sequence
+
+from zipctl.cli.archive import CHUNK
+from zipctl.cli.atomic import replacing
+from zipctl.cli.commands.helpers.copying.options import CopyJob
+from zipctl.cli.commands.helpers.copying.report import report_copy
+from zipctl.cli.commands.helpers.copying.targets import (
+    Copied,
+    Target,
+    describe_failure,
+)
+from zipctl.cli.commands.helpers.copying.verify import verify_copy
+from zipctl.cli.commands.helpers.passwords.pool import PasswordPool, PasswordProblem
+from zipctl.cli.context import Context
+from zipctl.cli.errors import EXIT_OK, CliError
+from zipctl.cli.methods import (
+    ENCRYPTION_METHODS,
+    NO_ENCRYPTION,
+    compression_label,
+    method_of,
+)
+from zipctl.cli.output import printable
+from zipctl.cryptography import wz_aes_stores_crc
+from zipctl.zipfile.file import ZipFile
+from zipctl.zipfile.info import ZipInfo
+
+
+def read_passwords(
+    zf: ZipFile, infos: Sequence[ZipInfo], pool: PasswordPool
+) -> list[bytes | None]:
+    """The password of every encrypted member (``None`` for plain ones).
+
+    Stops at the first member that cannot be unlocked: the output would be
+    incomplete, so there is no point asking for the rest.
+    """
+    passwords: list[bytes | None] = []
+    for info in infos:
+        if not info.is_encrypted:
+            passwords.append(None)
+            continue
+        password = pool.resolve(zf, info)
+        if isinstance(password, PasswordProblem):
+            raise CliError(f"cannot read {printable(info.filename)}: {password.text}")
+        passwords.append(password)
+    return passwords
+
+
+def _target_keeps_crc(target: Target) -> bool:
+    """Whether a member written for *target* carries its CRC-32 (WZ-AES 2 does not)."""
+    if not target.method.is_aes:
+        return True
+    return wz_aes_stores_crc(
+        target.extra.force_wz_aes_version if target.extra else None
+    )
+
+
+def _can_copy_raw(info: ZipInfo, target: Target) -> bool:
+    """Whether *info* can go to *target* without decompressing and compressing it.
+
+    When the CRC-32 is lost on the way (from or to WZ-AES 2) the data is read once
+    more to compute it, which a ZipCrypto member, being slow to decrypt, is not
+    worth: that copy takes the ordinary path.
+    """
+    if info.stores_crc and _target_keeps_crc(target):
+        return True
+    return method_of(info) is not ENCRYPTION_METHODS["zipcrypto"]
+
+
+def _checksum(src: ZipFile, info: ZipInfo, password: bytes | None) -> tuple[int, int]:
+    """The CRC-32 and length of the data of *info*, read (and so checked) in full."""
+    crc = size = 0
+    with src.open(info, pwd=password) as reader:
+        while chunk := reader.read(CHUNK):
+            crc = zlib.crc32(chunk, crc)
+            size += len(chunk)
+    return crc, size
+
+
+def _copy_compressed(
+    src: ZipFile,
+    dst: ZipFile,
+    info: ZipInfo,
+    new: ZipInfo,
+    password: bytes | None,
+    target: Target,
+) -> tuple[int, int]:
+    """Copy *info* with its compressed data as it is; returns its CRC-32 and size."""
+    if info.stores_crc and _target_keeps_crc(target):
+        crc, size = info.CRC, info.file_size  # not read: --verify checks them
+    else:
+        crc, size = _checksum(src, info, password)
+    dst._copy_raw(  # noqa: SLF001  # pyright: ignore[reportPrivateUsage]  # the library hook made for this copy
+        src,
+        info,
+        new,
+        crc=crc,
+        size=size,
+        pwd=password,
+        encryption=target.method.scheme,
+        password=target.password,
+        extra=target.extra,
+    )
+    return crc, size
+
+
+def _recompress(
+    src: ZipFile,
+    dst: ZipFile,
+    info: ZipInfo,
+    new: ZipInfo,
+    password: bytes | None,
+    target: Target,
+) -> tuple[int, int]:
+    """Copy *info* by reading its data and compressing it again."""
+    crc = size = 0
+    with (
+        src.open(info, pwd=password) as reader,
+        dst.open(
+            new,
+            "w",
+            encryption=target.method.scheme,
+            password=target.password,
+            extra=target.extra,
+        ) as writer,
+    ):
+        while chunk := reader.read(CHUNK):
+            crc = zlib.crc32(chunk, crc)
+            size += len(chunk)
+            writer.write(chunk)
+    return crc, size
+
+
+def _copy(
+    src: ZipFile,
+    dst: ZipFile,
+    infos: Sequence[ZipInfo],
+    passwords: Sequence[bytes | None],
+    targets: Sequence[Target],
+    compress_type: int | None,
+    level: int | None,
+) -> list[Copied]:
+    copied: list[Copied] = []
+    for info, password, target in zip(infos, passwords, targets, strict=True):
+        new = ZipInfo(info.filename, info.date_time)
+        new.comment = info.comment
+        new.external_attr = info.external_attr
+        new.internal_attr = info.internal_attr
+        new.create_system = info.create_system
+        new.extra = info.carried_extra
+        if info.is_dir():
+            dst.mkdir(new)
+            copied.append(
+                Copied(info.filename, True, 0, 0, None, NO_ENCRYPTION, NO_ENCRYPTION)
+            )
+            continue
+        new.compress_type = (
+            info.compress_type if compress_type is None else compress_type
+        )
+        new.compress_level = level
+        new.file_size = info.file_size
+        raw = compress_type is None and _can_copy_raw(info, target)
+        try:
+            if raw:
+                crc, size = _copy_compressed(src, dst, info, new, password, target)
+            else:
+                crc, size = _recompress(src, dst, info, new, password, target)
+        except Exception as exc:
+            raise CliError(
+                f"cannot copy {printable(info.filename)}: {describe_failure(exc)}"
+            ) from None
+        copied.append(
+            Copied(
+                info.filename,
+                False,
+                size,
+                crc,
+                compression_label(new),
+                method_of(info),
+                target.method,
+                raw,
+            )
+        )
+    return copied
+
+
+def run_copy(
+    ctx: Context,
+    src: ZipFile,
+    passwords: Sequence[bytes | None],
+    targets: Sequence[Target],
+    job: CopyJob,
+    *,
+    verb: str,
+    compress_type: int | None = None,
+    level: int | None = None,
+) -> int:
+    """Copy every member of *src* per *targets* into ``job.output``, then report."""
+    infos = src.infolist()
+    with replacing(job.output, overwrite=job.force) as scratch:
+        with ZipFile(scratch, "w", strict_timestamps=False) as dst:
+            dst.comment = src.comment
+            copied = _copy(src, dst, infos, passwords, targets, compress_type, level)
+        if job.verify:
+            verify_copy(scratch, src, infos, targets, copied)
+    report_copy(ctx, job, verb, copied)
+    return EXIT_OK

zipctl/cli/commands/helpers/copying/options.py

@@ -0,0 +1,80 @@
+"""The command-line side of the copying commands: arguments and path checks."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from dataclasses import dataclass
+from typing import Protocol
+
+from zipctl.cli.commands.helpers.output_options import (
+    OutputArgs,
+    OutputOptions,
+    add_output_options,
+)
+from zipctl.cli.errors import CliError, UsageError
+from zipctl.cli.output import printable
+
+
+class CopyArgs(OutputArgs, Protocol):
+    """What :func:`add_copy_options` leaves on the parsed arguments."""
+
+    input: str
+    output: str
+    force: bool
+    no_verify: bool
+
+
+@dataclass(frozen=True)
+class CopyJob:
+    """What :func:`add_copy_options` collects: the files and how to report."""
+
+    input: str
+    output: str
+    force: bool
+    verify: bool
+    report: OutputOptions
+
+
+def copy_job(args: CopyArgs) -> CopyJob:
+    return CopyJob(
+        args.input,
+        args.output,
+        args.force,
+        not args.no_verify,
+        OutputOptions.from_args(args),
+    )
+
+
+def check_paths(job: CopyJob) -> None:
+    """Refuse an output that is the input, or that exists without ``--force``."""
+    try:
+        same = os.path.samefile(job.input, job.output)
+    except OSError:
+        same = False
+    if same:
+        raise UsageError(
+            f"{printable(job.output)} is the input archive; zipctl never rewrites "
+            "an archive in place (write a new file, then move it over the original)",
+        )
+    if os.path.lexists(job.output) and not job.force:
+        raise CliError(
+            f"{printable(job.output)} already exists (use --force to replace it)"
+        )
+
+
+def add_copy_options(parser: argparse.ArgumentParser) -> None:
+    """The arguments every copying command takes."""
+    parser.add_argument("input", metavar="INPUT", help="the archive to read")
+    parser.add_argument(
+        "output", metavar="OUTPUT", help="the new archive (never the input itself)"
+    )
+    parser.add_argument(
+        "--force", action="store_true", help="replace OUTPUT if it exists"
+    )
+    parser.add_argument(
+        "--no-verify",
+        action="store_true",
+        help="skip reading OUTPUT back before moving it into place",
+    )
+    add_output_options(parser, verbose_help="list every member", quiet=True)

zipctl/cli/commands/helpers/copying/report.py

@@ -0,0 +1,63 @@
+"""The report the copy commands print after a copy."""
+
+from __future__ import annotations
+
+from zipctl.cli.commands.helpers.copying.options import CopyJob
+from zipctl.cli.commands.helpers.copying.targets import Copied
+from zipctl.cli.context import Context
+from zipctl.cli.output import JsonValue, count, printable, write_json
+
+
+def report_copy(ctx: Context, job: CopyJob, verb: str, copied: list[Copied]) -> None:
+    files = [c for c in copied if not c.directory]
+    encrypted = sum(c.after.is_encrypted for c in files)
+    if job.report.json:
+        members: list[dict[str, JsonValue]] = [
+            {
+                "name": c.name,
+                "directory": c.directory,
+                "size": c.size,
+                "compression": c.compression,
+                "encryption_before": c.before.label,
+                "encryption_after": c.after.label,
+            }
+            for c in copied
+        ]
+        write_json(
+            ctx.stdout,
+            {
+                "ok": True,
+                "input": job.input,
+                "output": job.output,
+                "verified": job.verify,
+                "file_count": len(files),
+                "directory_count": len(copied) - len(files),
+                "encrypted_count": encrypted,
+                "members": members,
+            },
+        )
+        return
+    if job.report.quiet:
+        return
+    if job.report.verbose:
+        for c in copied:
+            if c.directory:
+                ctx.out(f"Copying: {printable(c.name)} (directory)")
+                continue
+            change = c.after.label
+            if c.before != c.after:
+                change = f"{c.before.label} -> {change}"
+            ctx.out(f"Copying: {printable(c.name)} ({c.compression}, {change})")
+    if ctx.warned or (job.report.verbose and copied):
+        ctx.out()
+    parts = [
+        count(len(files), "file"),
+        count(len(copied) - len(files), "directory", "directories"),
+    ]
+    if encrypted:
+        parts.append(f"{encrypted} encrypted")
+    verified = " (verified)" if job.verify else ""
+    ctx.out(
+        f"{verb} {printable(job.input)} into {printable(job.output)}: "
+        f"{', '.join(parts)}{verified}"
+    )

zipctl/cli/commands/helpers/copying/targets.py

@@ -0,0 +1,76 @@
+"""How each member of a copy is written, and the record of what was written."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from zipctl.cli.errors import CliError
+from zipctl.cli.methods import (
+    NO_ENCRYPTION,
+    UNKNOWN_AES,
+    EncryptionMethod,
+    extra_for,
+    method_of,
+)
+from zipctl.cli.output import printable
+from zipctl.zipfile.file import ZipFileExtra
+from zipctl.zipfile.info import ZipInfo
+
+
+@dataclass(frozen=True)
+class Target:
+    """How one member is written: its encryption method, password and AES settings."""
+
+    method: EncryptionMethod = NO_ENCRYPTION
+    password: bytes | None = None
+    extra: ZipFileExtra | None = None
+
+
+PLAIN = Target()
+
+
+def target_for_method(
+    method: EncryptionMethod, password: bytes | None, aes_version: int | None
+) -> Target:
+    """The target that writes with *method*, its *password* and AES version."""
+    return Target(
+        method,
+        password if method.is_encrypted else None,
+        extra_for(method, aes_version),
+    )
+
+
+def keep_target(info: ZipInfo, password: bytes | None) -> Target:
+    """The target that writes *info* again with the protection it already has."""
+    method = method_of(info)
+    if not method.is_encrypted:
+        return PLAIN
+    if method is UNKNOWN_AES:
+        raise CliError(
+            f"cannot keep the encryption of {printable(info.filename)}: unknown "
+            f"AES strength {info.aes_extra.wz_aes_strength}"
+        )
+    if not method.is_aes:
+        return Target(method, password)
+    extra = ZipFileExtra(
+        force_wz_aes_version=info.aes_extra.wz_aes_version, wz_aes_nbits=method.aes_bits
+    )
+    return Target(method, password, extra)
+
+
+@dataclass
+class Copied:
+    """What was written for one member."""
+
+    name: str
+    directory: bool
+    size: int
+    crc: int
+    compression: str | None
+    before: EncryptionMethod
+    after: EncryptionMethod
+    raw: bool = False  # the compressed data was copied as it is
+
+
+def describe_failure(exc: Exception) -> str:
+    return printable(str(exc) or type(exc).__name__)

zipctl/cli/commands/helpers/copying/verify.py

@@ -0,0 +1,111 @@
+"""Reading a new archive back to prove it holds what was meant to be written."""
+
+from __future__ import annotations
+
+import zlib
+from collections.abc import Sequence
+
+from zipctl.cli.archive import CHUNK
+from zipctl.cli.commands.helpers.copying.targets import (
+    Copied,
+    Target,
+    describe_failure,
+)
+from zipctl.cli.errors import EXIT_FAILURE, CliError
+from zipctl.cli.methods import method_of
+from zipctl.cli.output import printable
+from zipctl.zipfile.file import ZipFile
+from zipctl.zipfile.info import ZipInfo
+from zipctl.zipfile.shared import MASK_COMPRESS_OPTIONS
+
+_SHOWN = 10  # problems listed before "and N more"
+
+
+def verify_copy(
+    path: str,
+    src: ZipFile,
+    infos: Sequence[ZipInfo],
+    targets: Sequence[Target],
+    copied: Sequence[Copied],
+) -> None:
+    """Read the new archive back; raise if it differs from what was intended."""
+    try:
+        out = ZipFile(path)
+    except Exception as exc:
+        raise CliError(
+            f"the new archive cannot be read back: {describe_failure(exc)}"
+        ) from None
+    problems: list[str] = []
+    with out:
+        new_infos = out.infolist()
+        same_count = len(new_infos) == len(infos)
+        if not same_count:
+            problems.append(f"expected {len(infos)} members but found {len(new_infos)}")
+        if out.comment != src.comment:
+            problems.append("the archive comment differs")
+        if same_count:
+            for old, new, target, record in zip(
+                infos, new_infos, targets, copied, strict=True
+            ):
+                problems.extend(_member_problems(out, old, new, target, record))
+    if problems:
+        shown = problems[:_SHOWN]
+        if len(problems) > _SHOWN:
+            shown.append(f"and {len(problems) - _SHOWN} more")
+        raise CliError(
+            "verification of the new archive failed; nothing was written",
+            EXIT_FAILURE,
+            tuple(shown),
+        )
+
+
+def _member_problems(
+    out: ZipFile, old: ZipInfo, new: ZipInfo, target: Target, record: Copied
+) -> list[str]:
+    name = printable(old.filename)
+    if (
+        new.filename,
+        new.date_time,
+        new.external_attr,
+        new.internal_attr,
+        new.create_system,
+        new.comment,
+        new.carried_extra,
+    ) != (
+        old.filename,
+        old.date_time,
+        old.external_attr,
+        old.internal_attr,
+        old.create_system,
+        old.comment,
+        old.carried_extra,
+    ):
+        return [f"{name}: name, date, mode, comment or extra fields differ"]
+    if record.raw and (
+        new.compress_type,
+        new.flag_bits & MASK_COMPRESS_OPTIONS,
+        new.file_size,
+    ) != (old.compress_type, old.flag_bits & MASK_COMPRESS_OPTIONS, old.file_size):
+        return [f"{name}: the compressed data was not copied as it was"]
+    found = method_of(new)
+    if found != target.method:
+        return [f"{name}: is {found.label}, not {target.method.label}"]
+    if old.is_dir():
+        return []
+    return _data_problems(out, new, target, record, name)
+
+
+def _data_problems(
+    out: ZipFile, new: ZipInfo, target: Target, record: Copied, name: str
+) -> list[str]:
+    crc = size = 0
+    try:
+        with out.open(new, pwd=target.password) as stream:
+            while chunk := stream.read(CHUNK):
+                crc = zlib.crc32(chunk, crc)
+                size += len(chunk)
+    except Exception as exc:
+        return [f"{name}: {describe_failure(exc)}"]
+    if size != record.size or crc != record.crc:
+        return [f"{name}: the data read back differs from the data copied"]
+    return []

zipctl/cli/commands/helpers/create_report.py

@@ -0,0 +1,154 @@
+"""What ``create`` prints, as text or as JSON."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from zipctl.cli.commands.helpers.output_options import OutputOptions
+from zipctl.cli.context import Context
+from zipctl.cli.methods import NO_ENCRYPTION
+from zipctl.cli.output import JsonValue, count, human_size, printable, write_json
+
+__all__ = ["Added", "Planned", "report_created", "report_dry_run"]
+
+
+@dataclass(frozen=True)
+class Added:
+    """A member as written to the archive."""
+
+    name: str
+    directory: bool
+    is_symlink: bool
+    size: int
+    compressed_size: int
+    compression: str | None
+    encryption: str
+
+
+@dataclass(frozen=True)
+class Planned:
+    """A member ``--dry-run`` says would be written; *encryption* is a label."""
+
+    name: str
+    directory: bool
+    is_symlink: bool
+    compression: str | None
+    encryption: str
+
+
+def _detail(member: Added | Planned) -> str:
+    """What a listing says about *member*: its kind, compression and protection."""
+    if member.directory:
+        return "directory"
+    plain = member.encryption == NO_ENCRYPTION.label
+    if member.is_symlink:
+        return "symlink" if plain else f"symlink, {member.encryption}"
+    return f"{member.compression}, {member.encryption}"
+
+
+def _warn_skipped(
+    ctx: Context, skipped: list[tuple[str, str]], unused: list[str]
+) -> None:
+    for path, reason in skipped:
+        ctx.warn(f"skipping {printable(path)}: {reason}")
+    for pattern in unused:
+        ctx.warn(f"--exclude {printable(pattern)!r} matched nothing")
+
+
+def _skipped_json(
+    skipped: list[tuple[str, str]], unused: list[str]
+) -> dict[str, JsonValue]:
+    return {
+        "skipped": [{"path": p, "reason": r} for p, r in skipped],
+        "unused_excludes": unused,
+    }
+
+
+def report_dry_run(
+    ctx: Context,
+    archive: str,
+    members: list[Planned],
+    skipped: list[tuple[str, str]],
+    unused: list[str],
+    *,
+    appending: bool,
+    output: OutputOptions,
+) -> None:
+    files = sum(not m.directory for m in members)
+    directories = len(members) - files
+    if output.json:
+        write_json(
+            ctx.stdout,
+            {
+                "ok": True,
+                "archive": archive,
+                "dry_run": True,
+                "appended": appending,
+                "file_count": files,
+                "directory_count": directories,
+                "members": members,
+                **_skipped_json(skipped, unused),
+            },
+        )
+        return
+    if output.quiet:
+        return
+    _warn_skipped(ctx, skipped, unused)
+    for member in members:
+        ctx.out(f"Would add: {printable(member.name)} ({_detail(member)})")
+    if members or ctx.warned:
+        ctx.out()
+    parts = f"{count(files, 'file')}, {count(directories, 'directory', 'directories')}"
+    name = printable(archive)
+    verb = f"add to {name}" if appending else f"create {name}"
+    ctx.out(f"Dry run: would {verb}: {parts}")
+
+
+def report_created(
+    ctx: Context,
+    archive: str,
+    members: list[Added],
+    skipped: list[tuple[str, str]],
+    unused: list[str],
+    *,
+    appended: bool,
+    output: OutputOptions,
+) -> None:
+    files = [m for m in members if not m.directory]
+    directories = len(members) - len(files)
+    original = sum(m.size for m in files)
+    stored = sum(m.compressed_size for m in files)
+    if output.json:
+        write_json(
+            ctx.stdout,
+            {
+                "ok": True,
+                "archive": archive,
+                "appended": appended,
+                "file_count": len(files),
+                "directory_count": directories,
+                "bytes_in": original,
+                "bytes_out": stored,
+                "members": members,
+                **_skipped_json(skipped, unused),
+            },
+        )
+        return
+    if output.quiet:
+        return
+    _warn_skipped(ctx, skipped, unused)
+    if output.verbose:
+        for member in members:
+            ctx.out(f"Adding: {printable(member.name)} ({_detail(member)})")
+    if ctx.warned or (output.verbose and members):
+        ctx.out()
+    encrypted = sum(m.encryption != NO_ENCRYPTION.label for m in files)
+    parts = [count(len(files), "file"), count(directories, "directory", "directories")]
+    if encrypted:
+        parts.append(f"{encrypted} encrypted")
+    size = f"{human_size(original)}, {human_size(stored)} compressed"
+    name = printable(archive)
+    if appended:
+        ctx.out(f"Added {', '.join(parts)} ({size}) to {name}")
+    else:
+        ctx.out(f"Created {name}: {', '.join(parts)} ({size})")

zipctl/cli/commands/helpers/encryption/__init__.py

@@ -0,0 +1 @@
+"""Encryption rules: the plan, the JSON spec and the command-line options."""