aprf-reader 0.3.0__tar.gz → 0.3.1__tar.gz

{aprf_reader-0.3.0/src/aprf_reader.egg-info → aprf_reader-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.3.0
+Version: 0.3.1
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aprf-reader"
-version = "0.3.0"
+version = "0.3.1"
 description = "Summarise APRF (Aggregate Performance Reporting) inbox placement reports."
 readme = "README.md"
 license = { text = "MIT" }

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/src/aprf_reader/__init__.py

@@ -1,3 +1,3 @@
 """Read APRF aggregate performance reports."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/src/aprf_reader/load.py

@@ -14,7 +14,7 @@ from email.message import Message
 from email.parser import BytesParser
 from pathlib import Path
 
-from .report import Report, ReportError, parse
+from .report import Report, ReportError, parse_items
 
 SUFFIXES = (".json", ".json.gz", ".eml", ".mbox")
 # A daily report is a few KB; this stops a stray huge file or gzip bomb
@@ -116,16 +116,20 @@ def load(paths: Iterable[Path]) -> Loaded:
             return
         seen_bytes.add(digest)
         try:
-            reports = parse(data)
+            items = parse_items(data)
         except ReportError as e:
             out.skipped.append((where, str(e)))
             return
-        for r in reports:
+        for i, r in enumerate(items, 1):
+            label = f"{where} report {i}" if len(items) > 1 else where
+            if isinstance(r, ReportError):
+                out.skipped.append((label, str(r)))
+                continue
             # The draft does not yet say whether a resend replaces the first; keep the first
             if r.key in seen_keys:
-                out.skipped.append((where, f"same day already reported in {seen_keys[r.key]}; kept that one"))
+                out.skipped.append((label, f"same day already reported in {seen_keys[r.key]}; kept that one"))
                 continue
-            seen_keys[r.key] = where
+            seen_keys[r.key] = label
             out.reports.append(r)
 
     for path in _files(paths):

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/src/aprf_reader/render.py

@@ -11,6 +11,8 @@ from .load import Loaded
 from .report import DRAFT
 from .summary import Row, Stream, streams
 
+NO_COUNTS = "These reports carry no counts."
+
 CAVEAT = (
     "Counts are bucket upper bounds chosen by each provider: a ceiling, not an exact figure. "
     "Placement counts mail received that day; engagement counts actions taken that day on mail "
@@ -50,6 +52,9 @@ def to_markdown(loaded: Loaded) -> str:
     for s in ss:
         lines += [f"## {_md(s.source)}: {_md(s.domain)}, selector {_md(s.selector)}", "", _span(s), ""]
         cols = [k for _, k in s.columns]
+        if not cols:
+            lines += [NO_COUNTS, ""]
+            continue
         lines.append("| Day | Segment | " + " | ".join(_md(c) for c in cols) + " |")
         lines.append("|---|---|" + "---:|" * len(cols))
         for row in s.rows:
@@ -112,6 +117,9 @@ def to_html(loaded: Loaded) -> str:
     for s in ss:
         cols = [k for _, k in s.columns]
         parts.append(f"<h2>{e(s.source)}: {e(s.domain)}, selector {e(s.selector)}</h2>")
+        if not cols:
+            parts.append(f"<p class=muted>{e(_span(s))}</p><p>{NO_COUNTS}</p>")
+            continue
         parts.append(f"<p class=muted>{e(_span(s))}</p><div class=wrap><table><thead><tr><th>Day<th>Segment")
         parts.append("".join(f"<th>{e(c)}" for c in cols) + "</thead><tbody>")
         parts.append("<tr class=trend><td>Trend<td class=muted>sum per day")
@@ -166,6 +174,13 @@ def _bars(points: list[tuple[int, int]], width: int) -> str:
     return "".join(" " if v is None else BARS[min(7, v * 7 // top)] if v else BARS[0] for v in cells)
 
 
+def _spread(points: list[tuple[int, int]]) -> str:
+    lo, hi = min(v for _, v in points), max(v for _, v in points)
+    if lo == hi:
+        return f"{lo:,}" if len(points) == 1 else f"{lo:,} every day"
+    return f"{lo:,} to {hi:,}"
+
+
 def _date(d) -> str:
     return f"{d:%b} {d.day}"
 
@@ -195,10 +210,13 @@ def to_terminal(loaded: Loaded, color: bool = True, width: int = 100) -> str:
             line += st.dim(" · ") + st.warn(f"{_plural(s.missing, 'day')} with no report")
         out += [line, ""]
         names = [k for _, k in s.columns]
+        if not names:
+            out += ["  " + st.dim(NO_COUNTS), ""]
+            continue
         nw = max([len(n) for n in names] + [5])
         trends = [s.trend(i) for i in range(len(names))]
         latest = [f"{t[-1][1]:,}" for t in trends]
-        ranges = [f"{min(v for _, v in t):,} to {max(v for _, v in t):,}" for t in trends]
+        ranges = [_spread(t) for t in trends]
         lw, rw = max(len(x) for x in latest + ["latest"]), max(len(x) for x in ranges + ["range"])
         bw = max(10, min(60, width - (2 + nw + 2 + lw + 2 + rw + 2)))
         out.append(st.dim(f"  {'':<{nw}}  {'latest':>{lw}}  {'range':<{rw}}  trend"))
@@ -216,7 +234,8 @@ def to_terminal(loaded: Loaded, color: bool = True, width: int = 100) -> str:
         out.append("")
     if loaded.skipped:
         out.append(st.warn(f"Skipped {len(loaded.skipped)}"))
-        out += [st.dim(f"  {w}: {r}") for w, r in loaded.skipped]
+        for w, r in loaded.skipped:
+            out += [st.dim(line) for line in _wrap(f"{w}: {r}", width - 2, "  ")]
         out.append("")
     out.append(st.dim(f"aprf-reader {__version__}, {DRAFT}. Daily tables: --markdown, or --html for a page."))
     return "\n".join(out) + "\n"

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/src/aprf_reader/report.py

@@ -15,10 +15,16 @@ DRAFT = "draft-ietf-mailmaint-aprf-00"
 # Names the draft defines, in display order; providers may add others
 PLACEMENT_NAMES = ("inbox", "unwanted")
 ENGAGEMENT_NAMES = ("positive", "neutral", "negative")
-# sdi_used values (first word) that mean no SDI was applied (section 5.2.1)
+# sdi_used values that mean no SDI was applied (section 5.2.1), compared on
+# the first word without case or trailing punctuation
 NO_SDI = ("", "N/A", "N/F")
 
 
+def uses_sdi(value: str) -> bool:
+    first = value.split(" ", 1)[0].rstrip(";,.").upper()
+    return first not in NO_SDI
+
+
 class ReportError(ValueError):
     """The data is not an APRF report this module can read."""
 
@@ -50,15 +56,45 @@ class Report:
         # Section 8: the report's date is its ending date, in UTC
         return _day(self.report_end)
 
+    @property
+    def stream_key(self) -> tuple[str, str, str]:
+        return (self.source, name_key(self.dkim_domain), self.dkim_selector.lower())
+
     @property
     def key(self) -> tuple[str, str, str, date]:
-        return (self.source, self.dkim_domain.lower(), self.dkim_selector, self.day)
+        return (*self.stream_key, self.day)
+
+
+def name_key(domain: str) -> str:
+    """DNS names compare without case, and bücher.example is xn--bcher-kva.example."""
+    d = domain.lower()
+    # Only plain letters, digits, dots and hyphens; the web page converts the same set
+    # shortcut: Python uses IDNA 2003 and browsers UTS 46, which differ on a few letters such as ß
+    if d.isascii() or not all(c.isalnum() or c in ".-" for c in d):
+        return d
+    try:
+        return d.encode("idna").decode("ascii")
+    except UnicodeError:
+        return d
 
 
 def _day(epoch: int) -> date:
     return datetime.fromtimestamp(epoch, UTC).date()
 
 
+# JavaScript numbers are exact only up to here; the web page reads the same files
+MAX_COUNT = 2**53 - 1
+
+
+def _int(v: object) -> int | None:
+    """An integer, or a float with nothing after the point (100.0); otherwise None."""
+    if isinstance(v, bool):
+        return None
+    if isinstance(v, float) and v.is_integer():
+        v = int(v)
+    return v if isinstance(v, int) else None
+
+
 def _counts(value: object, where: str) -> dict[str, int]:
     if value is None:
         return {}
@@ -66,9 +102,10 @@ def _counts(value: object, where: str) -> dict[str, int]:
         raise ReportError(f"{where} is not an object")
     out = {}
     for name, n in value.items():
-        if isinstance(n, bool) or not isinstance(n, int) or n < 0:
-            raise ReportError(f"{where}.{name} is not a count: {n!r}")
-        out[str(name)] = n
+        k = _int(n)
+        if k is None or k < 0 or k > MAX_COUNT:
+            raise ReportError(f"{where}.{name} is not a count: {json.dumps(n, ensure_ascii=False)}")
+        out[str(name)] = k
     return out
 
 
@@ -103,13 +140,19 @@ def _report(obj: object) -> Report:
     for name in ("dkim_domain", "dkim_selector"):
         if not isinstance(h.get(name), str) or not h[name]:
             raise ReportError(f"header.{name} is missing")
-    for name in ("report_start", "report_end"):
-        if isinstance(h.get(name), bool) or not isinstance(h.get(name), int):
+    start, end = _int(h.get("report_start")), _int(h.get("report_end"))
+    for name, v in (("report_start", start), ("report_end", end)):
+        if v is None:
             raise ReportError(f"header.{name} is not an epoch time")
+        # Seconds past 1e11 are beyond the year 5138; up to 1e14 that is milliseconds
+        if 10**11 <= abs(v) < 10**14:
+            raise ReportError(f"header.{name} looks like milliseconds, not seconds: {v}")
+    if end < start:
+        raise ReportError("header.report_end is before report_start")
     try:
-        _day(h["report_end"])
+        _day(end)
     except (OverflowError, OSError, ValueError):
-        raise ReportError(f"header.report_end is out of range: {h['report_end']}") from None
+        raise ReportError(f"header.report_end is out of range: {end}") from None
     body = obj.get("body", [])
     if not isinstance(body, list):
         raise ReportError("body is not a list")
@@ -117,8 +160,8 @@ def _report(obj: object) -> Report:
         source=_text(h.get("source")) or "unknown",
         dkim_domain=h["dkim_domain"],
         dkim_selector=h["dkim_selector"],
-        report_start=h["report_start"],
-        report_end=h["report_end"],
+        report_start=start,
+        report_end=end,
         version=_text(h.get("version")),
         # Comcast's beta sends " - N/A - Beta - Not in use"
         sdi_used=_text(h.get("sdi_used")).strip(" -"),
@@ -128,8 +171,9 @@ def _report(obj: object) -> Report:
     )
 
 
-def parse(data: bytes) -> list[Report]:
-    """Parse one report document. The draft's samples wrap reports in a list."""
+def parse_items(data: bytes) -> list[Report | ReportError]:
+    """Each report in a document, or the reason it cannot be read, so one bad
+    report does not cost the rest of the file."""
     try:
         doc = json.loads(data)
     except (UnicodeDecodeError, json.JSONDecodeError) as e:
@@ -139,4 +183,19 @@ def parse(data: bytes) -> list[Report]:
     items = doc if isinstance(doc, list) else [doc]
     if not items:
         raise ReportError("empty list")
-    return [_report(o) for o in items]
+    out: list[Report | ReportError] = []
+    for o in items:
+        try:
+            out.append(_report(o))
+        except ReportError as e:
+            out.append(e)
+    return out
+
+
+def parse(data: bytes) -> list[Report]:
+    """Parse one report document. The draft's samples wrap reports in a list."""
+    items = parse_items(data)
+    for r in items:
+        if isinstance(r, ReportError):
+            raise r
+    return items  # type: ignore[return-value]

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/src/aprf_reader/summary.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass
 from datetime import date, timedelta
 from functools import cached_property
 
-from .report import ENGAGEMENT_NAMES, NO_SDI, PLACEMENT_NAMES, Report
+from .report import ENGAGEMENT_NAMES, PLACEMENT_NAMES, Report, uses_sdi
 
 
 # Counts where a jump up is bad news, or a drop is
@@ -64,7 +64,7 @@ class Stream:
                 out.append(Row(prev + timedelta(1), d - timedelta(1), None))
             for s in self.by_day[d].segments:
                 vals = tuple(getattr(s, a).get(k) for a, k in self.columns)
-                out.append(Row(d, d, " / ".join(s.path) or "all mail", vals))
+                out.append(Row(d, d, " / ".join(p or "(blank)" for p in s.path) or "all mail", vals))
             prev = d
         return out
 
@@ -106,8 +106,7 @@ class Stream:
             prev = r.version
         # "TBD" is the draft samples' placeholder
         out += [f"Provider note: {t}" for t in sorted({r.extra_info for r in self.by_day.values()} - {"", "TBD"})]
-        sdi = sorted({r.sdi_used for r in self.by_day.values()
-                      if r.sdi_used.split(" ", 1)[0] not in NO_SDI})
+        sdi = sorted({r.sdi_used for r in self.by_day.values() if uses_sdi(r.sdi_used)})
         if sdi:
             out.append("SDI used: " + ", ".join(sdi))
         return out
@@ -116,7 +115,7 @@ class Stream:
 def streams(reports: list[Report]) -> list[Stream]:
     groups: dict[tuple[str, str, str], dict[date, Report]] = defaultdict(dict)
     for r in reports:
-        groups[(r.source, r.dkim_domain.lower(), r.dkim_selector)][r.day] = r
+        groups[r.stream_key][r.day] = r
     return [Stream(src, dom, sel, days) for (src, dom, sel), days in sorted(groups.items())]
 
 

{aprf_reader-0.3.0 → aprf_reader-0.3.1/src/aprf_reader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aprf-reader
-Version: 0.3.0
+Version: 0.3.1
 Summary: Summarise APRF (Aggregate Performance Reporting) inbox placement reports.
 License: MIT
 Requires-Python: >=3.11

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/tests/test_reader.py

@@ -234,3 +234,24 @@ def test_cli_terminal_when_interactive(monkeypatch, capsys):
     assert out.startswith("APRF placement summary") and "\033[" not in out
     assert main([str(FIX / "sample_six_weeks.json"), "--markdown"]) == 0
     assert capsys.readouterr().out.startswith("# APRF placement summary")
+
+
+def test_stress_set_one_bad_report_does_not_cost_the_file():
+    loaded = load([FIX / "stress_100.json"])
+    assert len(loaded.reports) == 88
+    reasons = {w.rsplit(" ", 1)[-1]: r for w, r in loaded.skipped}
+    assert reasons["73"] == 'body[0].engagement.positive is not a count: "100"'
+    assert reasons["79"] == "header.report_end is before report_start"
+    assert reasons["81"].startswith("header.report_start looks like milliseconds")
+    assert "report 63" in reasons["64"]  # bücher.example is xn--bcher-kva.example
+    assert "report 3" in reasons["70"]  # DNS-AUDIT.COM / SELECTOR1 is dns-audit.com / selector1
+    ss = {(s.source, s.domain, s.selector): s for s in streams(loaded.reports)}
+    obj = report(0)
+    obj["body"][0]["classification"]["unwanted"] = 100.0
+    assert parse(json.dumps(obj).encode())[0].segments[0].classification["unwanted"] == 100
+    for sel in ("sdi0", "sdi1", "sdi2", "sdi3", "sdi4"):  # n/a, "N/A ", "", null, N/F;
+        assert not ss[("Comcast/Xfinity", "dns-audit.com", sel)].notes, sel
+    blank = ss[("Comcast/Xfinity", "dns-audit.com", "s13")]
+    assert "(blank)" in [r.segment for r in blank.rows]
+    assert not ss[("Comcast/Xfinity", "dns-audit.com", "s8")].columns
+    assert "These reports carry no counts." in to_markdown(loaded)

{aprf_reader-0.3.0 → aprf_reader-0.3.1}/tests/test_web_parity.py

@@ -52,7 +52,8 @@ def email(payload: bytes, subtype: str, filename: str | None, cte: str | None =
     return msg
 
 
-@pytest.mark.parametrize("name", ["draft_sample_1.json", "draft_sample_2.json", "comcast_beta.json", "sample_six_weeks.json"])
+@pytest.mark.parametrize("name", ["draft_sample_1.json", "draft_sample_2.json", "comcast_beta.json", "sample_six_weeks.json",
+                                  "stress_100.json"])
 def test_fixtures(name):
     same([FIX / name])