aprf-reader 0.6.0__tar.gz → 0.6.2__tar.gz

{aprf_reader-0.6.0/src/aprf_reader.egg-info → aprf_reader-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.6.0
+Version: 0.6.2
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -79,7 +79,7 @@ marked read, moved or deleted. It reads messages whose subject contains
 from the `APRF_IMAP_PASSWORD` environment variable, or it asks. Gmail and
 Outlook.com need an app password, not your account password, and a Gmail
 label is read as a folder of the same name. The connection uses TLS on port
-993; give another port as `--imap host:port`. Files it skips are listed
+993 and checks the server's certificate; give another port as `--imap host:port`. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/README.md

@@ -66,7 +66,7 @@ marked read, moved or deleted. It reads messages whose subject contains
 from the `APRF_IMAP_PASSWORD` environment variable, or it asks. Gmail and
 Outlook.com need an app password, not your account password, and a Gmail
 label is read as a folder of the same name. The connection uses TLS on port
-993; give another port as `--imap host:port`. Files it skips are listed
+993 and checks the server's certificate; give another port as `--imap host:port`. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aprf-reader"
-version = "0.6.0"
+version = "0.6.2"
 description = "Summarise APRF (Aggregate Performance Reporting) inbox placement reports."
 readme = "README.md"
 license = { text = "MIT" }

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/src/aprf_reader/__init__.py

@@ -1,3 +1,3 @@
 """Read APRF aggregate performance reports."""
 
-__version__ = "0.6.0"
+__version__ = "0.6.2"

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/src/aprf_reader/cli.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import argparse
 import getpass
+import imaplib
 import json
 import os
 import shutil
@@ -49,11 +50,20 @@ def main(argv: list[str] | None = None) -> int:
         host, _, port = args.imap.partition(":")
         if port and not port.isdigit():
             p.error(f"--imap port is not a number: {port}")
-        password = os.environ.get("APRF_IMAP_PASSWORD") or getpass.getpass(f"IMAP password for {args.user}: ")
+        password = os.environ.get("APRF_IMAP_PASSWORD")
+        if not password:
+            if pasted is not None:
+                p.error("stdin is the pasted report, so the password must come from APRF_IMAP_PASSWORD")
+            try:
+                password = getpass.getpass(f"IMAP password for {args.user}: ")
+            except (EOFError, KeyboardInterrupt):
+                print(file=sys.stderr)
+                p.error("no password: set APRF_IMAP_PASSWORD or run it in a terminal")
         try:
             # Fetched in full before reading, so a dropped connection leaves no half summary
             messages = list(fetch(host, args.user, password, args.folder, int(port or 993), args.since, args.all_mail))
-        except (ImapError, OSError) as e:
+        # imaplib.IMAP4.error and UnicodeError are a backstop; fetch() turns what it can into ImapError
+        except (ImapError, imaplib.IMAP4.error, OSError, UnicodeError) as e:
             print(f"aprf-reader: {args.imap}: {e}", file=sys.stderr)
             return 2
         print(f"read {len(messages)} message{'s' if len(messages) != 1 else ''} from {args.folder} on {host}", file=sys.stderr)

aprf_reader-0.6.2/src/aprf_reader/imap.py

@@ -0,0 +1,141 @@
+"""Read report emails straight from a mailbox over IMAP, without changing anything there."""
+
+from __future__ import annotations
+
+import base64
+import imaplib
+import re
+import ssl
+from collections.abc import Iterator
+from datetime import date
+
+from .load import MAX_BYTES
+
+# Comcast's subjects start "APRF:"; IMAP SUBJECT matches a substring, without case
+SUBJECT = "APRF:"
+# Messages fetched per round trip
+BATCH = 50
+# A few years of daily reports for a few selectors; more is a wrong folder
+MAX_MESSAGES = 20_000
+
+
+class ImapError(Exception):
+    pass
+
+
+def _check(typ: str, data: list, what: str) -> list:
+    if typ != "OK":
+        detail = b" ".join(d for d in data if isinstance(d, bytes)).decode(errors="replace")
+        raise ImapError(f"{what} failed: {detail or typ}")
+    return data
+
+
+def _text(e: Exception) -> str:
+    # imaplib puts the server's reply bytes in the exception as they came
+    a = e.args[0] if e.args else ""
+    return a.decode(errors="replace") if isinstance(a, bytes) else str(e)
+
+
+def utf7(name: str) -> str:
+    """A folder name in IMAP's modified UTF-7 (RFC 3501 5.1.3), as Exchange and others name
+    folders such as "Gelöschte Elemente"; plain ASCII is unchanged."""
+    out, run = [], []
+
+    def flush():
+        if run:
+            b = base64.b64encode("".join(run).encode("utf-16-be")).decode().rstrip("=").replace("/", ",")
+            out.append(f"&{b}-")
+            run.clear()
+
+    for ch in name:
+        if 0x20 <= ord(ch) <= 0x7E:
+            flush()
+            out.append("&-" if ch == "&" else ch)
+        else:
+            run.append(ch)
+    flush()
+    return "".join(out)
+
+
+def _login(m, user: str, password: str) -> None:
+    if user.isascii() and password.isascii():
+        _check(*m.login(user, password), "login")
+    else:
+        # LOGIN only carries ASCII; SASL PLAIN carries UTF-8 (RFC 4616)
+        if "AUTH=PLAIN" not in getattr(m, "capabilities", ("AUTH=PLAIN",)):
+            raise ImapError("login failed: the user name or password has non-ASCII characters, "
+                            "and this server does not offer AUTH=PLAIN, which is needed to send them")
+        creds = b"\0" + user.encode() + b"\0" + password.encode()
+        _check(*m.authenticate("PLAIN", lambda _: creds), "login")
+
+
+def _quote(s: str) -> str:
+    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def criteria(since: date | None = None, every: bool = False) -> list[str]:
+    out = [] if every else ["SUBJECT", _quote(SUBJECT)]
+    if since:
+        out += ["SINCE", since.strftime("%d-%b-%Y")]
+    return out or ["ALL"]
+
+
+def _messages(m, host: str, folder: str, since: date | None, every: bool) -> Iterator[tuple[str, bytes | None]]:
+    (ids,) = _check(*m.uid("SEARCH", *criteria(since, every)), "search")
+    uids = (ids or b"").split()
+    if len(uids) > MAX_MESSAGES:
+        raise ImapError(f"{len(uids):,} messages match, more than {MAX_MESSAGES:,}; "
+                        "pick the folder the reports are filed in, or use --since")
+    for i in range(0, len(uids), BATCH):
+        chunk = b",".join(uids[i:i + BATCH]).decode()
+        # Sizes first, so a huge message is never downloaded
+        sizes = {}
+        for item in _check(*m.uid("FETCH", chunk, "(UID RFC822.SIZE)"), "fetch"):
+            line = item[0] if isinstance(item, tuple) else item
+            u, n = re.search(rb"UID (\d+)", line or b""), re.search(rb"RFC822\.SIZE (\d+)", line or b"")
+            if u and n:
+                sizes[u.group(1)] = int(n.group(1))
+        big = [u for u in uids[i:i + BATCH] if sizes.get(u, 0) > MAX_BYTES]
+        for u in big:
+            yield f"imap://{host}/{folder} uid {u.decode()}", None
+        small = [u for u in uids[i:i + BATCH] if u not in big]
+        if not small:
+            continue
+        data = _check(*m.uid("FETCH", b",".join(small).decode(), "(UID BODY.PEEK[])"), "fetch")
+        for item in data:
+            if not isinstance(item, tuple):
+                continue  # the closing ")" of each response
+            head, body = item
+            # Most servers put UID before the body; one that puts it after still has a sequence number
+            uid = re.search(rb"UID (\d+)", head)
+            which = f"uid {uid.group(1).decode()}" if uid else f"message {head.split(b' ', 1)[0].decode(errors='replace')}"
+            yield f"imap://{host}/{folder} {which}", body
+
+
+def fetch(host: str, user: str, password: str, folder: str = "INBOX", port: int = 993,
+          since: date | None = None, every: bool = False, conn=None) -> Iterator[tuple[str, bytes | None]]:
+    """Yield (label, raw message bytes) for each matching message in one folder; the
+    bytes are None for a message over MAX_BYTES, which is not downloaded.
+
+    The folder is opened read-only and bodies are fetched with BODY.PEEK, so no
+    message is marked read, moved or deleted."""
+    # imaplib's own default context checks neither the certificate nor the host name
+    m = conn or imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=60)
+    try:
+        try:
+            _login(m, user, password)
+        except imaplib.IMAP4.error as e:
+            raise ImapError(f"login failed: {_text(e)}") from None
+        try:
+            _check(*m.select(_quote(utf7(folder)), readonly=True), f"opening folder {folder}")
+        except imaplib.IMAP4.error as e:
+            raise ImapError(f"opening folder {folder} failed: {_text(e)}") from None
+        try:
+            yield from _messages(m, host, folder, since, every)
+        except imaplib.IMAP4.error as e:  # includes abort: the server hung up mid-run
+            raise ImapError(f"connection lost: {_text(e)}") from None
+    finally:
+        try:
+            m.logout()
+        except (imaplib.IMAP4.error, OSError):
+            pass

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/src/aprf_reader/load.py

@@ -232,15 +232,16 @@ def pasted_name(data: bytes) -> str:
     return "pasted.json.gz" if data[:2] == b"\x1f\x8b" else "pasted.eml"
 
 
-def _message(label: str, data: bytes) -> list[tuple[str, bytes, Mail]]:
-    """Reports in one raw email fetched from a mailbox; mail with none is counted, not skipped."""
-    if len(data) > MAX_BYTES:
+def _message(label: str, data: bytes | None) -> list[tuple[str, bytes, Mail]]:
+    """Reports in one raw email fetched from a mailbox; mail with none is counted, not skipped.
+    None is a message too large to have been downloaded."""
+    if data is None or len(data) > MAX_BYTES:
         raise ReportError("larger than 10 MB")
     return list(_attachments(BytesParser(policy=policy.default).parsebytes(data), label))
 
 
 def load(paths: Iterable[Path], pasted: bytes | None = None,
-         messages: Iterable[tuple[str, bytes]] = (), source: str = "mailbox") -> Loaded:
+         messages: Iterable[tuple[str, bytes | None]] = (), source: str = "mailbox") -> Loaded:
     out = Loaded()
     seen_keys: dict[tuple, tuple[str, Report]] = {}
 

{aprf_reader-0.6.0 → aprf_reader-0.6.2/src/aprf_reader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.6.0
+Version: 0.6.2
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11
@@ -79,7 +79,7 @@ marked read, moved or deleted. It reads messages whose subject contains
 from the `APRF_IMAP_PASSWORD` environment variable, or it asks. Gmail and
 Outlook.com need an app password, not your account password, and a Gmail
 label is read as a folder of the same name. The connection uses TLS on port
-993; give another port as `--imap host:port`. Files it skips are listed
+993 and checks the server's certificate; give another port as `--imap host:port`. Files it skips are listed
 on stderr and at the end of the summary, each with a reason.
 
 Each stream (provider, signing domain, selector) gets a table with one row per

{aprf_reader-0.6.0 → aprf_reader-0.6.2}/tests/test_imap.py

@@ -1,6 +1,7 @@
 import gzip
 import imaplib
 import json
+import ssl
 from datetime import date
 
 import pytest
@@ -31,6 +32,10 @@ class FakeImap:
             return "NO", [b"[NONEXISTENT] Unknown Mailbox"]
         return "OK", [str(len(self.messages)).encode()]
 
+    def authenticate(self, mech, cb):
+        self.calls.append(("authenticate", mech, cb(b"")))
+        return "OK", [b"logged in"]
+
     def uid(self, command, *args):
         self.calls.append((command, *args))
         if command == "SEARCH":
@@ -38,7 +43,10 @@ class FakeImap:
         out = []
         for u in args[0].split(","):
             body = self.messages[int(u)]
-            out += [(f"{u} (UID {u} RFC822.SIZE {len(body)} BODY[] {{{len(body)}}}".encode(), body), b")"]
+            if "BODY" not in args[1]:
+                out.append(f"{u} (UID {u} RFC822.SIZE {len(body)})".encode())
+            else:
+                out += [(f"{u} (UID {u} BODY[] {{{len(body)}}}".encode(), body), b")"]
         return "OK", out
 
     def logout(self):
@@ -59,7 +67,7 @@ def test_fetch_is_read_only_and_batched(monkeypatch):
     got = list(fetch("imap.example.com", "u", "right", "APRF", conn=fake))
     assert [label for label, _ in got] == [f"imap://imap.example.com/APRF uid {u}" for u in range(100, 105)]
     assert ("select", '"APRF"', True) in fake.calls  # EXAMINE, not SELECT
-    fetches = [c for c in fake.calls if c[0] == "FETCH"]
+    fetches = [c for c in fake.calls if c[0] == "FETCH" and "BODY" in c[2]]
     assert len(fetches) == 3 and all("BODY.PEEK[]" in c[2] for c in fetches)  # PEEK leaves \\Seen alone
     assert fake.calls[-1] == ("logout",)
 
@@ -87,7 +95,7 @@ def test_fetched_mail_read_like_an_mbox():
 
 def test_cli_imap(monkeypatch, capsys):
     fake = FakeImap(mails(2))
-    monkeypatch.setattr(imaplib, "IMAP4_SSL", lambda host, port, timeout: fake)
+    monkeypatch.setattr(imaplib, "IMAP4_SSL", lambda host, port, ssl_context, timeout: fake)
     monkeypatch.setenv("APRF_IMAP_PASSWORD", "right")
     assert cli.main(["--imap", "imap.example.com", "--user", "u", "--folder", "APRF", "--json"]) == 0
     out, err = capsys.readouterr()
@@ -96,10 +104,71 @@ def test_cli_imap(monkeypatch, capsys):
 
 
 def test_cli_imap_failure_and_usage(monkeypatch, capsys):
-    monkeypatch.setattr(imaplib, "IMAP4_SSL", lambda host, port, timeout: FakeImap({}))
+    monkeypatch.setattr(imaplib, "IMAP4_SSL", lambda host, port, ssl_context, timeout: FakeImap({}))
     monkeypatch.setenv("APRF_IMAP_PASSWORD", "wrong")
     assert cli.main(["--imap", "h", "--user", "u"]) == 2
     assert "login failed" in capsys.readouterr().err
     for argv in ([], ["--imap", "h"], ["--imap", "h:x", "--user", "u"]):
         with pytest.raises(SystemExit):
             cli.main(argv)
+
+
+def test_certificate_is_checked(monkeypatch):
+    seen = {}
+
+    def connect(host, port, ssl_context, timeout):
+        seen["ctx"] = ssl_context
+        return FakeImap({})
+
+    monkeypatch.setattr(imaplib, "IMAP4_SSL", connect)
+    list(fetch("h", "u", "right"))
+    assert seen["ctx"].verify_mode == ssl.CERT_REQUIRED and seen["ctx"].check_hostname
+
+
+def test_huge_message_not_downloaded(monkeypatch):
+    monkeypatch.setattr(imap, "MAX_BYTES", 1000)
+    msgs = mails(1)
+    msgs[200] = b"x" * 2000
+    fake = FakeImap(msgs)
+    got = dict(fetch("h", "u", "right", conn=fake))
+    assert got["imap://h/INBOX uid 200"] is None and got["imap://h/INBOX uid 100"]
+    assert all("200" not in c[1] for c in fake.calls if c[0] == "FETCH" and "BODY" in c[2])
+    loaded = load([], None, got.items())
+    assert loaded.skipped == [("imap://h/INBOX uid 200", "larger than 10 MB")]
+
+
+def test_connection_lost_mid_run_is_a_clean_error(monkeypatch, capsys):
+    fake = FakeImap(mails(2))
+
+    def hang_up(*a):
+        raise imaplib.IMAP4.abort("socket error: EOF")
+
+    fake.uid = hang_up
+    with pytest.raises(ImapError, match="connection lost: socket error: EOF"):
+        list(fetch("h", "u", "right", conn=fake))
+    monkeypatch.setattr(imaplib, "IMAP4_SSL", lambda host, port, ssl_context, timeout: fake)
+    monkeypatch.setenv("APRF_IMAP_PASSWORD", "right")
+    assert cli.main(["--imap", "h", "--user", "u"]) == 2
+    assert "connection lost" in capsys.readouterr().err
+
+
+def test_non_ascii_folder_and_password():
+    assert imap.utf7("Gelöschte Elemente") == "Gel&APY-schte Elemente"
+    assert imap.utf7("日本語") == "&ZeVnLIqe-"  # the RFC 3501 example
+    assert imap.utf7("A&B") == "A&-B"
+    fake = FakeImap({}, folders=("Gel&APY-schte Elemente",))
+    list(fetch("h", "u", "sécret", "Gelöschte Elemente", conn=fake))
+    assert ("authenticate", "PLAIN", "\0u\0sécret".encode()) in fake.calls
+    assert ("select", '"Gel&APY-schte Elemente"', True) in fake.calls
+
+
+def test_no_terminal_for_the_password(monkeypatch, capsys):
+    monkeypatch.delenv("APRF_IMAP_PASSWORD", raising=False)
+
+    def no_tty(prompt):
+        raise EOFError
+
+    monkeypatch.setattr(cli.getpass, "getpass", no_tty)
+    with pytest.raises(SystemExit):
+        cli.main(["--imap", "h", "--user", "u"])
+    assert "set APRF_IMAP_PASSWORD" in capsys.readouterr().err

aprf_reader-0.6.0/src/aprf_reader/imap.py

@@ -1,77 +0,0 @@
-"""Read report emails straight from a mailbox over IMAP, without changing anything there."""
-
-from __future__ import annotations
-
-import imaplib
-import re
-from collections.abc import Iterator
-from datetime import date
-
-# Comcast's subjects start "APRF:"; IMAP SUBJECT matches a substring, without case
-SUBJECT = "APRF:"
-# Messages fetched per round trip
-BATCH = 50
-# A few years of daily reports for a few selectors; more is a wrong folder
-MAX_MESSAGES = 20_000
-
-
-class ImapError(Exception):
-    pass
-
-
-def _check(typ: str, data: list, what: str) -> list:
-    if typ != "OK":
-        detail = b" ".join(d for d in data if isinstance(d, bytes)).decode(errors="replace")
-        raise ImapError(f"{what} failed: {detail or typ}")
-    return data
-
-
-def _quote(s: str) -> str:
-    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
-
-
-def criteria(since: date | None = None, every: bool = False) -> list[str]:
-    out = [] if every else ["SUBJECT", _quote(SUBJECT)]
-    if since:
-        out += ["SINCE", since.strftime("%d-%b-%Y")]
-    return out or ["ALL"]
-
-
-def fetch(host: str, user: str, password: str, folder: str = "INBOX", port: int = 993,
-          since: date | None = None, every: bool = False, conn=None) -> Iterator[tuple[str, bytes]]:
-    """Yield (label, raw message bytes) for each matching message in one folder.
-
-    The folder is opened read-only and bodies are fetched with BODY.PEEK, so no
-    message is marked read, moved or deleted."""
-    m = conn or imaplib.IMAP4_SSL(host, port, timeout=60)
-    try:
-        try:
-            _check(*m.login(user, password), "login")
-        except imaplib.IMAP4.error as e:
-            raise ImapError(f"login failed: {e}") from None
-        try:
-            _check(*m.select(_quote(folder), readonly=True), f"opening folder {folder}")
-        except imaplib.IMAP4.error as e:
-            raise ImapError(f"opening folder {folder} failed: {e}") from None
-        (ids,) = _check(*m.uid("SEARCH", *criteria(since, every)), "search")
-        uids = (ids or b"").split()
-        if len(uids) > MAX_MESSAGES:
-            raise ImapError(f"{len(uids):,} messages match, more than {MAX_MESSAGES:,}; "
-                            "pick the folder the reports are filed in, or use --since")
-        for i in range(0, len(uids), BATCH):
-            chunk = b",".join(uids[i:i + BATCH]).decode()
-            data = _check(*m.uid("FETCH", chunk, "(UID RFC822.SIZE BODY.PEEK[])"), "fetch")
-            for item in data:
-                if not isinstance(item, tuple):
-                    continue  # the closing ")" of each response
-                head, body = item
-                # Most servers put UID before the body; one that puts it after still has a sequence number
-                uid = re.search(rb"UID (\d+)", head)
-                which = f"uid {uid.group(1).decode()}" if uid else f"message {head.split(b' ', 1)[0].decode(errors='replace')}"
-                label = f"imap://{host}/{folder} {which}"
-                yield label, body
-    finally:
-        try:
-            m.logout()
-        except (imaplib.IMAP4.error, OSError):
-            pass