aprf-reader 0.4.0__tar.gz → 0.5.0__tar.gz

{aprf_reader-0.4.0/src/aprf_reader.egg-info → aprf_reader-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.4.0
+Version: 0.5.0
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -24,7 +24,7 @@ Two ways to use it, with the same results:
   your reports on the page. They are read on your computer and never uploaded.
 - **On the command line:** `pipx install aprf-reader`.
 
-**Status: 0.4, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.5, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -59,7 +59,7 @@ the changes worth a look. Piped or written to a file with `--md`, the output
 is Markdown with the full daily tables.
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+mailbox exports (`.mbox`), `.zip` files of any of those, or folders of them. The easiest way to collect reports
 is a mail filter that moves them into one folder, then an export of that folder:
 Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
 label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
@@ -78,7 +78,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml`, `.mbox` and `.zip`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/README.md

@@ -11,7 +11,7 @@ Two ways to use it, with the same results:
   your reports on the page. They are read on your computer and never uploaded.
 - **On the command line:** `pipx install aprf-reader`.
 
-**Status: 0.4, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.5, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -46,7 +46,7 @@ the changes worth a look. Piped or written to a file with `--md`, the output
 is Markdown with the full daily tables.
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+mailbox exports (`.mbox`), `.zip` files of any of those, or folders of them. The easiest way to collect reports
 is a mail filter that moves them into one folder, then an export of that folder:
 Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
 label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
@@ -65,7 +65,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml`, `.mbox` and `.zip`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aprf-reader"
-version = "0.4.0"
+version = "0.5.0"
 description = "Summarise APRF (Aggregate Performance Reporting) inbox placement reports."
 readme = "README.md"
 license = { text = "MIT" }

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/src/aprf_reader/__init__.py

@@ -1,3 +1,3 @@
 """Read APRF aggregate performance reports."""
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/src/aprf_reader/cli.py

@@ -17,7 +17,7 @@ from .summary import to_dict
 
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser(prog="aprf-reader", description="Summarise APRF placement reports.")
-    p.add_argument("paths", nargs="+", type=Path, help="report files (.json, .json.gz, .eml, .mbox) or folders")
+    p.add_argument("paths", nargs="+", type=Path, help="report files (.json, .json.gz, .eml, .mbox, .zip) or folders")
     p.add_argument("--md", type=Path, help="write the summary here instead of printing it")
     p.add_argument("--html", type=Path, help="also write a self-contained HTML page here")
     p.add_argument("--markdown", action="store_true", help="print Markdown with daily tables, even in a terminal")

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/src/aprf_reader/load.py

@@ -7,6 +7,7 @@ import hashlib
 import io
 import mailbox
 import re
+import zipfile
 import zlib
 from collections.abc import Iterable, Iterator
 from dataclasses import dataclass, field
@@ -17,7 +18,9 @@ from pathlib import Path
 
 from .report import Report, ReportError, parse_items
 
-SUFFIXES = (".json", ".json.gz", ".eml", ".mbox")
+SUFFIXES = (".json", ".json.gz", ".eml", ".mbox", ".zip")
+# A zip of a year of daily reports for a few selectors is a few thousand files
+MAX_ZIP_ENTRIES = 20_000
 # A daily report is a few KB; this stops a stray huge file or gzip bomb
 MAX_BYTES = 10 * 1024 * 1024
 
@@ -97,20 +100,50 @@ def _attachments(msg: Message, label: str) -> Iterator[tuple[str, bytes, Mail]]:
             yield where, data, mail(False)
 
 
-def _payloads(path: Path) -> Iterator[tuple[str, bytes, Mail | None]]:
-    """Yield (label, report JSON bytes, how it arrived) for each report in a file."""
-    data = path.read_bytes()
+def _payloads(label: str, data: bytes) -> Iterator[tuple[str, bytes, Mail | None]]:
+    """Yield (label, report JSON bytes, how it arrived) for each report in one file's bytes."""
     if len(data) > MAX_BYTES:
         raise ReportError("larger than 10 MB")
-    if path.name.lower().endswith(".eml"):
-        found = list(_attachments(BytesParser(policy=policy.default).parsebytes(data), str(path)))
+    if label.lower().endswith(".eml"):
+        found = list(_attachments(BytesParser(policy=policy.default).parsebytes(data), label))
         if not found:
             raise ReportError("no report attachment, which section 8 calls for")
         yield from found
     elif data[:2] == b"\x1f\x8b":
-        yield str(path), _gunzip(data), None
+        yield label, _gunzip(data), None
     else:
-        yield str(path), data, None
+        yield label, data, None
+
+
+def _zip(path: Path, out: Loaded) -> Iterator[tuple[str, bytes]]:
+    """Yield (label, bytes) for each report file in a zip, in name order; a bad member is skipped, not fatal."""
+    try:
+        z = zipfile.ZipFile(path)
+    except zipfile.BadZipFile:
+        raise ReportError("not a readable zip file") from None
+    with z:
+        # __MACOSX holds Finder metadata copies named like the real files
+        members = sorted((i for i in z.infolist() if not i.is_dir() and not i.filename.startswith("__MACOSX/")
+                          and i.filename.lower().endswith(SUFFIXES)), key=lambda i: i.filename)
+        if len(members) > MAX_ZIP_ENTRIES:
+            raise ReportError(f"more than {MAX_ZIP_ENTRIES:,} report files in one zip")
+        for i in members:
+            label = f"{path}/{i.filename}"
+            name = i.filename.lower()
+            if name.endswith((".zip", ".mbox")):
+                out.skipped.append((label, "a zip or mailbox inside a zip is not read; unzip it first"))
+            elif i.flag_bits & 1:
+                out.skipped.append((label, "encrypted in the zip"))
+            elif i.file_size > MAX_BYTES:
+                out.skipped.append((label, "larger than 10 MB"))
+            else:
+                try:
+                    with z.open(i) as f:
+                        data = f.read(MAX_BYTES + 1)
+                except (zipfile.BadZipFile, NotImplementedError, zlib.error, EOFError, OSError):
+                    out.skipped.append((label, "could not be unpacked from the zip"))
+                    continue
+                yield label, data
 
 
 def _mbox(path: Path, out: Loaded) -> Iterator[tuple[str, bytes, Mail | None]]:
@@ -232,8 +265,16 @@ def load(paths: Iterable[Path]) -> Loaded:
 
     for path in _files(paths):
         try:
+            if path.name.lower().endswith(".zip"):
+                for label, data in _zip(path, out):
+                    try:
+                        for where, d, mail in list(_payloads(label, data)):
+                            add(where, d, mail)
+                    except (OSError, EOFError, zlib.error, ReportError) as e:
+                        out.skipped.append((label, str(e)))
+                continue
             # An mbox can be large, so it is read one message at a time
-            payloads = _mbox(path, out) if _is_mbox(path) else list(_payloads(path))
+            payloads = _mbox(path, out) if _is_mbox(path) else list(_payloads(str(path), path.read_bytes()))
             for where, data, mail in payloads:
                 add(where, data, mail)
         except (OSError, EOFError, zlib.error, ReportError) as e:

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/src/aprf_reader/report.py

@@ -115,7 +115,7 @@ def _counts(value: object, where: str, section: str) -> dict[str, int]:
     for name, n in value.items():
         k = _int(n)
         if k is None or k < 0 or k > MAX_COUNT:
-            raise ReportError(f"{where}.{name} is not a count ({section}): {json.dumps(n, ensure_ascii=False, separators=(",", ":"))}")
+            raise ReportError(f"{where}.{name} is not a count ({section}): {json.dumps(n, ensure_ascii=False, separators=(',', ':'))}")
         out[str(name)] = k
     return out
 
@@ -194,7 +194,7 @@ def _warnings(h: dict, segments: tuple[Segment, ...]) -> tuple[tuple[str, str],
         raw = h["sdi_used"]
         if not (raw in ("N/A", "N/F") or (isinstance(raw, str) and SDI_FORMAT.fullmatch(raw))):
             out.append(("sdi_used is not N/A, N/F, or the sdi value as published in DNS (section 5.2.1)",
-                        json.dumps(raw, ensure_ascii=False, separators=(",", ":"))))
+                        json.dumps(raw, ensure_ascii=False, separators=(',', ':'))))
     keys = lambda attr: sorted({k for s in segments for k in getattr(s, attr)})
     for k in keys("classification"):
         if k not in NAMED_CLASSIFICATION:

{aprf_reader-0.4.0 → aprf_reader-0.5.0/src/aprf_reader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.4.0
+Version: 0.5.0
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -24,7 +24,7 @@ Two ways to use it, with the same results:
   your reports on the page. They are read on your computer and never uploaded.
 - **On the command line:** `pipx install aprf-reader`.
 
-**Status: 0.4, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.5, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -59,7 +59,7 @@ the changes worth a look. Piped or written to a file with `--md`, the output
 is Markdown with the full daily tables.
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+mailbox exports (`.mbox`), `.zip` files of any of those, or folders of them. The easiest way to collect reports
 is a mail filter that moves them into one folder, then an export of that folder:
 Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
 label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
@@ -78,7 +78,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml`, `.mbox` and `.zip`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.4.0 → aprf_reader-0.5.0}/tests/test_web_parity.py

@@ -3,6 +3,7 @@
 import gzip
 import json
 import mailbox
+import re
 import shutil
 import subprocess
 from email.message import EmailMessage
@@ -27,11 +28,18 @@ def web(mode: str, paths: list[Path]) -> str:
                           capture_output=True, text=True, env={"APRF_VERSION": __version__}).stdout
 
 
+def _engine(doc: dict) -> dict:
+    """Python and browsers word JSON parse errors differently; compare only our part."""
+    cut = lambda r: r.split(": ", 1)[0] if r.startswith("not valid JSON") else r
+    return {**doc, "skipped": [{**k, "reason": cut(k["reason"])} for k in doc["skipped"]]}
+
+
 def same(paths: list[Path]) -> None:
     loaded = load(paths)
-    assert json.loads(web("--json", paths)) == to_dict(loaded)
-    assert web("--md", paths) == to_markdown(loaded)
-    assert web("--html", paths) == to_html(loaded)
+    assert _engine(json.loads(web("--json", paths))) == _engine(to_dict(loaded))
+    text = lambda t: re.sub(r"(not valid JSON, which section 5 requires): [^\n<]*", r"\1", t)
+    assert text(web("--md", paths)) == text(to_markdown(loaded))
+    assert text(web("--html", paths)) == text(to_html(loaded))
 
 
 def report(day: int, selector: str = "s1", inbox: int = 100) -> dict:
@@ -128,3 +136,35 @@ def test_section_8_checks_the_same(tmp_path):
     assert "attachment name stands in for a character a file name cannot hold, such as / or *" in by_file["safe.eml"]
     assert {"attachment name does not follow section 8", "subject does not follow section 8",
             "attachment Content-Type is not application/json, or application/gzip when compressed"} <= by_file["wrong.eml"]
+
+
+def test_zips_read_the_same(tmp_path):
+    import zipfile
+    raw = lambda day, **kw: json.dumps(report(day, **kw)).encode()
+    z = tmp_path / "reports.zip"
+    with zipfile.ZipFile(z, "w") as f:
+        f.writestr("b/20261002.json", raw(1), compress_type=zipfile.ZIP_DEFLATED)
+        f.writestr("a/20261001.json", raw(0), compress_type=zipfile.ZIP_STORED)
+        f.writestr("c/20261003.json.gz", gzip.compress(raw(2)))
+        f.writestr("d/20261004.eml", email(raw(3), "json", "x.json").as_bytes(), compress_type=zipfile.ZIP_DEFLATED)
+        f.writestr("__MACOSX/a/._20261001.json", b"\x00\x05\x16\x07 Finder junk")
+        f.writestr("e/inner.zip", b"PK nested")
+        f.writestr("e/box.mbox", b"From x\n\nhi\n")
+        f.writestr("f/huge.json", b" " * (11 * 1024 * 1024), compress_type=zipfile.ZIP_DEFLATED)
+        f.writestr("g/notes.txt", b"not a report suffix")
+        f.writestr("h/broken.json", b"{oops", compress_type=zipfile.ZIP_DEFLATED)
+        f.writestr("i/crc.json", raw(5), compress_type=zipfile.ZIP_STORED)
+    # Flip one byte of the stored crc.json so its CRC no longer matches
+    data = bytearray(z.read_bytes())
+    at = data.index(b'"dkim_selector"', data.index(b"i/crc.json") + 10)
+    data[at + 2] ^= 0x01
+    z.write_bytes(bytes(data))
+    (tmp_path / "fake.zip").write_bytes(b"not a zip at all")
+    same([z, tmp_path / "fake.zip"])
+    loaded = load([z, tmp_path / "fake.zip"])
+    assert sorted(r.day.day for r in loaded.reports) == [1, 2, 3, 4]
+    reasons = {w.rsplit("/", 1)[-1]: r for w, r in loaded.skipped}
+    assert reasons["crc.json"] == "could not be unpacked from the zip"
+    assert reasons["inner.zip"] == reasons["box.mbox"] == "a zip or mailbox inside a zip is not read; unzip it first"
+    assert reasons["huge.json"] == "larger than 10 MB" and reasons["fake.zip"] == "not a readable zip file"
+    assert "broken.json" in reasons and not any("MACOSX" in w for w, _ in loaded.skipped)