aprf-reader 0.1.1__tar.gz → 0.2.0__tar.gz

{aprf_reader-0.1.1/src/aprf_reader.egg-info → aprf_reader-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.1.1
+Version: 0.2.0
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -17,7 +17,7 @@ Reads APRF (Aggregate Performance Reporting) reports and tells you where the
 mail you signed landed: how much reached the inbox, how much was treated as
 unwanted, and how recipients reacted, day by day and per DKIM selector.
 
-**Status: 0.1, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.2, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -41,10 +41,16 @@ environment, or with pip (`pip3` on some systems) inside a virtual environment.
 pipx install aprf-reader
 aprf-reader reports/                      # Markdown summary to the terminal
 aprf-reader reports/ --html summary.html  # and a page you can open or share
+aprf-reader APRF.mbox                     # a mail folder exported as mbox
 ```
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-or folders of them. Folders are searched recursively. Files it skips are listed
+mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+is a mail filter that moves them into one folder, then an export of that folder:
+Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
+label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
+Mailbox > Export Mailbox. Comcast's reports come from
+`aprf-noreply@alerts.xfinity.com` with subjects starting `APRF:`. Folders are searched recursively. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per
@@ -54,7 +60,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz` and `.eml`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.1.1 → aprf_reader-0.2.0}/README.md

@@ -4,7 +4,7 @@ Reads APRF (Aggregate Performance Reporting) reports and tells you where the
 mail you signed landed: how much reached the inbox, how much was treated as
 unwanted, and how recipients reacted, day by day and per DKIM selector.
 
-**Status: 0.1, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.2, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -28,10 +28,16 @@ environment, or with pip (`pip3` on some systems) inside a virtual environment.
 pipx install aprf-reader
 aprf-reader reports/                      # Markdown summary to the terminal
 aprf-reader reports/ --html summary.html  # and a page you can open or share
+aprf-reader APRF.mbox                     # a mail folder exported as mbox
 ```
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-or folders of them. Folders are searched recursively. Files it skips are listed
+mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+is a mail filter that moves them into one folder, then an export of that folder:
+Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
+label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
+Mailbox > Export Mailbox. Comcast's reports come from
+`aprf-noreply@alerts.xfinity.com` with subjects starting `APRF:`. Folders are searched recursively. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per
@@ -41,7 +47,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz` and `.eml`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.1.1 → aprf_reader-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aprf-reader"
-version = "0.1.1"
+version = "0.2.0"
 description = "Summarise APRF (Aggregate Performance Reporting) inbox placement reports."
 readme = "README.md"
 license = { text = "MIT" }

{aprf_reader-0.1.1 → aprf_reader-0.2.0}/src/aprf_reader/__init__.py

@@ -1,3 +1,3 @@
 """Read APRF aggregate performance reports."""
 
-__version__ = "0.1.1"
+__version__ = "0.2.0"

{aprf_reader-0.1.1 → aprf_reader-0.2.0}/src/aprf_reader/cli.py

@@ -13,7 +13,7 @@ from .render import to_html, to_markdown
 
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser(prog="aprf-reader", description="Summarise APRF placement reports.")
-    p.add_argument("paths", nargs="+", type=Path, help="report files (.json, .json.gz, .eml) or folders")
+    p.add_argument("paths", nargs="+", type=Path, help="report files (.json, .json.gz, .eml, .mbox) or folders")
     p.add_argument("--md", type=Path, help="write the Markdown summary here instead of printing it")
     p.add_argument("--html", type=Path, help="also write a self-contained HTML page here")
     p.add_argument("--version", action="version", version=f"aprf-reader {__version__}")

aprf_reader-0.2.0/src/aprf_reader/load.py

@@ -0,0 +1,139 @@
+"""Find report files, open them, and drop duplicates."""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import io
+import mailbox
+import zlib
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from pathlib import Path
+
+from .report import Report, ReportError, parse
+
+SUFFIXES = (".json", ".json.gz", ".eml", ".mbox")
+# A daily report is a few KB; this stops a stray huge file or gzip bomb
+MAX_BYTES = 10 * 1024 * 1024
+
+
+@dataclass
+class Loaded:
+    reports: list[Report] = field(default_factory=list)
+    skipped: list[tuple[str, str]] = field(default_factory=list)  # (where, reason)
+
+
+def _files(paths: Iterable[Path]) -> Iterator[Path]:
+    for p in paths:
+        if p.is_dir():
+            yield from sorted(f for f in p.rglob("*") if f.is_file() and (f.name.lower().endswith(SUFFIXES) or _is_mbox(f)))
+        else:
+            yield p
+
+
+def _is_mbox(path: Path) -> bool:
+    # Apple Mail exports a folder "Name.mbox" holding a file named just "mbox"
+    return path.name.lower() == "mbox" or path.name.lower().endswith(".mbox")
+
+
+def _gunzip(data: bytes) -> bytes:
+    with gzip.GzipFile(fileobj=io.BytesIO(data)) as g:
+        out = g.read(MAX_BYTES + 1)
+    if len(out) > MAX_BYTES:
+        raise ReportError("larger than 10 MB unpacked")
+    return out
+
+
+def _attachments(msg: Message, label: str) -> Iterator[tuple[str, bytes]]:
+    """Yield (label, report JSON bytes) for each report attached to one email."""
+    for part in msg.walk():
+        if part.is_multipart():
+            continue
+        fname = (part.get_filename() or "").lower()
+        ctype = part.get_content_type()
+        data = part.get_payload(decode=True) or b""
+        where = f"{label} ({fname or ctype})"
+        if len(data) > MAX_BYTES:
+            raise ReportError(f"{where} is larger than 10 MB")
+        if fname.endswith(".gz") or ctype in ("application/gzip", "application/x-gzip") or data[:2] == b"\x1f\x8b":
+            yield where, _gunzip(data)
+        # By name or type, or an attachment of unhelpful type that is a JSON array or object
+        elif fname.endswith(".json") or ctype == "application/json" or (
+                part.get_content_disposition() == "attachment" and data.lstrip()[:1] in (b"[", b"{")):
+            yield where, data
+
+
+def _payloads(path: Path) -> Iterator[tuple[str, bytes]]:
+    """Yield (label, report JSON bytes) for each report in a file."""
+    data = path.read_bytes()
+    if len(data) > MAX_BYTES:
+        raise ReportError("larger than 10 MB")
+    if path.name.lower().endswith(".eml"):
+        found = list(_attachments(BytesParser(policy=policy.default).parsebytes(data), str(path)))
+        if not found:
+            raise ReportError("no report attachment")
+        yield from found
+    elif data[:2] == b"\x1f\x8b":
+        yield str(path), _gunzip(data)
+    else:
+        yield str(path), data
+
+
+def _mbox(path: Path, out: Loaded) -> Iterator[tuple[str, bytes]]:
+    """Yield reports from a mailbox export; one skip line for all mail with none attached."""
+    box = mailbox.mbox(path, factory=lambda f: BytesParser(policy=policy.default).parse(f), create=False)
+    without = 0
+    try:
+        for i, msg in enumerate(box, 1):
+            label = f"{path} message {i}"
+            try:
+                found = list(_attachments(msg, label))
+            # OSError covers gzip.BadGzipFile; one bad message must not end the mailbox
+            except (OSError, EOFError, zlib.error, ReportError) as e:
+                out.skipped.append((label, str(e)))
+                continue
+            without += not found
+            yield from found
+    finally:
+        box.close()
+    if without:
+        out.skipped.append((str(path), f"{without} message{'s' if without != 1 else ''} with no report attached"))
+
+
+def load(paths: Iterable[Path]) -> Loaded:
+    out = Loaded()
+    seen_bytes: set[str] = set()
+    seen_keys: dict[tuple, str] = {}
+
+    def add(where: str, data: bytes) -> None:
+        digest = hashlib.sha256(data).hexdigest()
+        if digest in seen_bytes:
+            out.skipped.append((where, "duplicate of a file already read"))
+            return
+        seen_bytes.add(digest)
+        try:
+            reports = parse(data)
+        except ReportError as e:
+            out.skipped.append((where, str(e)))
+            return
+        for r in reports:
+            # The draft does not yet say whether a resend replaces the first; keep the first
+            if r.key in seen_keys:
+                out.skipped.append((where, f"same day already reported in {seen_keys[r.key]}; kept that one"))
+                continue
+            seen_keys[r.key] = where
+            out.reports.append(r)
+
+    for path in _files(paths):
+        try:
+            # An mbox can be large, so it is read one message at a time
+            payloads = _mbox(path, out) if _is_mbox(path) else list(_payloads(path))
+            for where, data in payloads:
+                add(where, data)
+        except (OSError, EOFError, zlib.error, ReportError) as e:
+            out.skipped.append((str(path), str(e)))
+    return out

{aprf_reader-0.1.1 → aprf_reader-0.2.0/src/aprf_reader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.1.1
+Version: 0.2.0
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -17,7 +17,7 @@ Reads APRF (Aggregate Performance Reporting) reports and tells you where the
 mail you signed landed: how much reached the inbox, how much was treated as
 unwanted, and how recipients reacted, day by day and per DKIM selector.
 
-**Status: 0.1, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
+**Status: 0.2, beta.** Reads reports in the format of draft-ietf-mailmaint-aprf-00.
 
 ## What APRF is
 
@@ -41,10 +41,16 @@ environment, or with pip (`pip3` on some systems) inside a virtual environment.
 pipx install aprf-reader
 aprf-reader reports/                      # Markdown summary to the terminal
 aprf-reader reports/ --html summary.html  # and a page you can open or share
+aprf-reader APRF.mbox                     # a mail folder exported as mbox
 ```
 
 Point it at report files (`.json`, `.json.gz`), saved report emails (`.eml`),
-or folders of them. Folders are searched recursively. Files it skips are listed
+mailbox exports (`.mbox`), or folders of them. The easiest way to collect reports
+is a mail filter that moves them into one folder, then an export of that folder:
+Gmail through [Google Takeout](https://takeout.google.com/) (pick Mail and the
+label), Thunderbird with the ImportExportTools NG add-on, or Apple Mail's
+Mailbox > Export Mailbox. Comcast's reports come from
+`aprf-noreply@alerts.xfinity.com` with subjects starting `APRF:`. Folders are searched recursively. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per
@@ -54,7 +60,7 @@ to mark a change in what a category counts.
 
 ## What it does
 
-- Read report files (`.json`, `.json.gz` and `.eml`) or a folder of them, and skip
+- Read report files (`.json`, `.json.gz`, `.eml` and `.mbox`) or a folder of them, and skip
   duplicates and files it cannot read, with a reason for each.
 - Summarise placement per signing domain and selector: inbox, unwanted and
   recipient reactions, by day and as a trend.

{aprf_reader-0.1.1 → aprf_reader-0.2.0}/tests/test_reader.py

@@ -1,5 +1,6 @@
 import gzip
 import json
+import mailbox
 from datetime import date
 from email.message import EmailMessage
 from pathlib import Path
@@ -91,6 +92,45 @@ def test_load_gz_eml_folder_and_skips(tmp_path):
     assert "duplicate" in reasons["copy.json"] and "same day" in reasons["resend.json"]
 
 
+def report_email(day: int, payload: bytes, subtype: str, filename: str | None) -> EmailMessage:
+    msg = EmailMessage()
+    msg["From"] = "aprf-noreply@alerts.xfinity.com"
+    msg["Subject"] = f"APRF: day {day}"
+    msg.set_content("report attached")
+    msg.add_attachment(payload, maintype="application", subtype=subtype, filename=filename)
+    return msg
+
+
+def test_attachment_found_by_content(tmp_path):
+    # A provider may label the file octet-stream or give it no useful name
+    raw = json.dumps(report(0)).encode()
+    (tmp_path / "plain.eml").write_bytes(report_email(0, raw, "octet-stream", "20261001_example.com_s1_Comcast").as_bytes())
+    gz = gzip.compress(json.dumps(report(1)).encode())
+    (tmp_path / "gz.eml").write_bytes(report_email(1, gz, "octet-stream", None).as_bytes())
+    loaded = load([tmp_path])
+    assert sorted(r.day.day for r in loaded.reports) == [1, 2] and not loaded.skipped
+
+
+def test_mbox_export(tmp_path):
+    (tmp_path / "Reports.mbox").mkdir()  # Apple Mail's export layout
+    box = mailbox.mbox(tmp_path / "Reports.mbox" / "mbox")
+    box.add(report_email(0, gzip.compress(json.dumps(report(0)).encode()), "gzip", "a.json.gz"))
+    box.add(report_email(1, json.dumps(report(1)).encode(), "json", "b.json"))
+    note = EmailMessage()
+    note["Subject"] = "not a report"
+    note.set_content("hello")
+    box.add(note)
+    box.add(note)
+    box.add(report_email(2, b"\x1f\x8b broken", "gzip", "c.json.gz"))
+    box.add(report_email(3, b"not gzip at all", "gzip", "d.json.gz"))
+    box.add(report_email(4, json.dumps(report(4)).encode(), "json", "e.json"))
+    box.close()
+    loaded = load([tmp_path])
+    assert sorted(r.day.day for r in loaded.reports) == [1, 2, 5]  # read on past bad messages
+    reasons = [r for _, r in loaded.skipped]
+    assert len(reasons) == 3 and "2 messages with no report attached" in reasons
+
+
 def test_hostile_files_skipped_not_crashed(tmp_path):
     (tmp_path / "deep.json").write_text("[" * 100_000 + "]" * 100_000)
     write(tmp_path / "far.json", report(0, report_end=10**20))

aprf_reader-0.1.1/src/aprf_reader/load.py

@@ -1,98 +0,0 @@
-"""Find report files, open them, and drop duplicates."""
-
-from __future__ import annotations
-
-import gzip
-import hashlib
-import io
-import zlib
-from collections.abc import Iterable, Iterator
-from dataclasses import dataclass, field
-from email import policy
-from email.parser import BytesParser
-from pathlib import Path
-
-from .report import Report, ReportError, parse
-
-SUFFIXES = (".json", ".json.gz", ".eml")
-# A daily report is a few KB; this stops a stray huge file or gzip bomb
-MAX_BYTES = 10 * 1024 * 1024
-
-
-@dataclass
-class Loaded:
-    reports: list[Report] = field(default_factory=list)
-    skipped: list[tuple[str, str]] = field(default_factory=list)  # (where, reason)
-
-
-def _files(paths: Iterable[Path]) -> Iterator[Path]:
-    for p in paths:
-        if p.is_dir():
-            yield from sorted(f for f in p.rglob("*") if f.is_file() and f.name.lower().endswith(SUFFIXES))
-        else:
-            yield p
-
-
-def _gunzip(data: bytes) -> bytes:
-    with gzip.GzipFile(fileobj=io.BytesIO(data)) as g:
-        out = g.read(MAX_BYTES + 1)
-    if len(out) > MAX_BYTES:
-        raise ReportError("larger than 10 MB unpacked")
-    return out
-
-
-def _payloads(path: Path) -> Iterator[tuple[str, bytes]]:
-    """Yield (label, report JSON bytes) for each report in a file."""
-    data = path.read_bytes()
-    if len(data) > MAX_BYTES:
-        raise ReportError("larger than 10 MB")
-    name = path.name.lower()
-    if name.endswith(".eml"):
-        msg = BytesParser(policy=policy.default).parsebytes(data)
-        found = False
-        for part in msg.walk():
-            fname = (part.get_filename() or "").lower()
-            ctype = part.get_content_type()
-            if fname.endswith(".json.gz") or ctype == "application/gzip":
-                found = True
-                yield f"{path} ({fname or ctype})", _gunzip(part.get_payload(decode=True) or b"")
-            elif fname.endswith(".json") or ctype == "application/json":
-                found = True
-                yield f"{path} ({fname or ctype})", part.get_payload(decode=True) or b""
-        if not found:
-            raise ReportError("no .json or .json.gz attachment")
-    elif data[:2] == b"\x1f\x8b":
-        yield str(path), _gunzip(data)
-    else:
-        yield str(path), data
-
-
-def load(paths: Iterable[Path]) -> Loaded:
-    out = Loaded()
-    seen_bytes: set[str] = set()
-    seen_keys: dict[tuple, str] = {}
-    for path in _files(paths):
-        try:
-            payloads = list(_payloads(path))
-        except (OSError, EOFError, zlib.error, ReportError) as e:
-            out.skipped.append((str(path), str(e)))
-            continue
-        for where, data in payloads:
-            digest = hashlib.sha256(data).hexdigest()
-            if digest in seen_bytes:
-                out.skipped.append((where, "duplicate of a file already read"))
-                continue
-            seen_bytes.add(digest)
-            try:
-                reports = parse(data)
-            except ReportError as e:
-                out.skipped.append((where, str(e)))
-                continue
-            for r in reports:
-                # The draft does not yet say whether a resend replaces the first; keep the first
-                if r.key in seen_keys:
-                    out.skipped.append((where, f"same day already reported in {seen_keys[r.key]}; kept that one"))
-                    continue
-                seen_keys[r.key] = where
-                out.reports.append(r)
-    return out