sendermap 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sendermap-0.1.3 → sendermap-0.1.4}/CHANGELOG.md +9 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/PKG-INFO +2 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/README.md +1 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/engine.py +4 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/render/html.py +56 -30
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/render/markdown.py +2 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/render/text.py +2 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/render/view.py +104 -13
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/setup.py +1 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/ct.py +2 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/networks.py +8 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/spf.py +4 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/spf_checks.py +2 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/subdomains.py +6 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/web.py +3 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/static/app.js +96 -41
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/static/style.css +29 -27
- {sendermap-0.1.3 → sendermap-0.1.4}/server.json +3 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/allbirds.com/expected.json +4 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/casper.com/expected.json +5 -5
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/glossier.com/expected.json +3 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/mills-reeve.com/expected.json +2 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/allbirds.com/markdown-evidence.md +5 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/allbirds.com/markdown.md +5 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/allbirds.com/page.html +127 -94
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/allbirds.com/text-evidence.txt +7 -7
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/allbirds.com/text.txt +7 -7
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/casper.com/markdown-evidence.md +6 -5
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/casper.com/markdown.md +6 -5
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/casper.com/page.html +119 -84
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/casper.com/text-evidence.txt +9 -10
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/casper.com/text.txt +9 -10
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/example.com/markdown-evidence.md +2 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/example.com/markdown.md +2 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/example.com/page.html +7 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/example.com/text-evidence.txt +1 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/example.com/text.txt +1 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/glossier.com/markdown-evidence.md +3 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/glossier.com/markdown.md +3 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/glossier.com/page.html +80 -55
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/glossier.com/text-evidence.txt +3 -6
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/glossier.com/text.txt +3 -6
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/mills-reeve.com/markdown-evidence.md +4 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/mills-reeve.com/markdown.md +4 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/mills-reeve.com/page.html +47 -30
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/mills-reeve.com/text-evidence.txt +5 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/snapshots/mills-reeve.com/text.txt +5 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_caps.py +7 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_html.py +18 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_setup.py +4 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_spf_checks.py +4 -1
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_story.py +14 -4
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_view.py +42 -2
- {sendermap-0.1.3 → sendermap-0.1.4}/uv.lock +3 -3
- {sendermap-0.1.3 → sendermap-0.1.4}/.github/workflows/publish.yml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/.github/workflows/recheck-corpus.yml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/.github/workflows/test.yml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/.github/workflows/verify-proofs.yml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/.gitignore +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/ARCHITECTURE.md +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/CLAUDE.md +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/LICENSE +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/data/hunt_domains.txt +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/pyproject.toml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/__init__.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/catalog.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/cli.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/data/schema/sendermap-1.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/data/vendors.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/data/vendors_pending.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/dns.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/infer.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/mcp_server.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/model.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/proofs.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/py.typed +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/reconcile.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/render/__init__.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/selectors.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/__init__.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/bimi.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/dkim.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/labels.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/mta_sts.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/mx.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/reports.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/spf_eval.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/sources/tokens.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/story.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/testing.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/__init__.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/app.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/static/index.html +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/sendermap/web/static/llms.txt +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/skills/sendermap/SKILL.md +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/conftest.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/README.md +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/allbirds.com/case.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/allbirds.com/dns.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/allbirds.com/http.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/casper.com/case.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/casper.com/dns.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/casper.com/http.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/example.com/case.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/example.com/dns.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/example.com/expected.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/example.com/http.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/glossier.com/case.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/glossier.com/dns.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/glossier.com/http.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/mills-reeve.com/case.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/mills-reeve.com/dns.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/corpus/mills-reeve.com/http.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/fixtures/crtsh_allbirds.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/fixtures/mcp-server-2025-12-11.schema.json +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_alias_subdomains.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_backed_only.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_catalog.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_checked.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_cli.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_corpus.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_ct.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_delegation.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_determinism.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_dkim.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_dkim_denial.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_engine.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_evidence.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_gap_kinds.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_gaps.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_hunt_proofs.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_labels_say_what_was_seen.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_listing.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_mcp.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_networks.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_one_selector_list.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_package.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_pending_vendors.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_publish.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_recheck_corpus.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_reconcile.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_record_replay.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_resolver.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_schema.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_selectors.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_skill.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_snapshots.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_sources.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_spf.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_spf_eval.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_subprocessor_names.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_supporting_evidence.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_vendor_proofs.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_verify_proofs.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_web.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_web_guard.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_web_mcp.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/test_wildcard_subdomains.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/vectors/rfc7208-tests.LICENSE +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tests/vectors/rfc7208-tests.yml +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tools/fill_corpus.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tools/hunt_proofs.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tools/record_corpus.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tools/release_notes.py +0 -0
- {sendermap-0.1.3 → sendermap-0.1.4}/tools/verify_proofs.py +0 -0
|
@@ -11,6 +11,15 @@ fails before anything is published.
|
|
|
11
11
|
|
|
12
12
|
## Unreleased
|
|
13
13
|
|
|
14
|
+
## 0.1.4 (2026-10-24)
|
|
15
|
+
|
|
16
|
+
- The site leads with the story and the ring map side by side, above Needs attention, so the findings sit above the fold. The story prints one sentence per paragraph, its Not checked sentence muted. From 900px up the map takes a fixed 340px column beside the story; below that it stacks under the story and grows to 560px.
|
|
17
|
+
- Every service name and sector label fits on the ring map. The rings stop inside the box to leave an outer band for names; each label sits outward along its own angle, at 10px (12px past 600px rendered), and a label that would collide is tried one line out, then one line up or down, before it is dropped. glossier.com shows 9 of 9 names and 4 of 4 sectors at 1280, 899 and 390px, where before it showed 3 and 1. The map draws after the result is shown and redraws on resize.
|
|
18
|
+
- The map centre is a plain hub at half its old size. The domain no longer prints in it, since it fit only sometimes and heads the page already.
|
|
19
|
+
- Services on the site are rows, not cards: name, category and where, tier badge, then the deciding record with the others under it, the setup clause and All records. Four columns from 900px up, two lines below. Each row keeps its `service-<slug>` id, so a tap on the map still lands on it.
|
|
20
|
+
- Not checked and What was asked say each reason once in all three views. Gaps of one question, kind, cap and outcome are one group listing the names together ("DKIM selectors at marketing, receipts and send.glossier.com: 82 of 109 not asked at each"), with the cap said last. What was asked leads with the lookups and names in all.
|
|
21
|
+
- certifi in `uv.lock` moves to 2026.10.23, past the emSign root CAs whose intermediate keys were compromised. It comes in through httpx; no code changes.
|
|
22
|
+
|
|
14
23
|
## 0.1.3 (2026-10-23)
|
|
15
24
|
|
|
16
25
|
- The terminal and Markdown views render one view model (`render/view.py`) and say each thing once: findings come first, each service shows one evidence line per kind ("DKIM keys: selector1, selector2 (both 2048 bit, by CNAME)") and its setup in one clause ("setup complete", or only what is missing), an item a finding names is said in the setup by its state alone, and What was asked gives counts. Full records and the names asked print with `--evidence`, the names in Markdown as a last section, Appendix: names asked. The Markdown senders table gains a Setup column, and its Setups section goes. Allbirds' Markdown view falls from 111 lines to 76.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sendermap
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Type a domain, see every service that sends email as it, with the public evidence for each.
|
|
5
5
|
Project-URL: Homepage, https://github.com/marmot7775/sendermap
|
|
6
6
|
Author: Neil Anuskiewicz
|
|
@@ -35,7 +35,7 @@ Description-Content-Type: text/markdown
|
|
|
35
35
|
|
|
36
36
|
Type a domain, see every service that sends email as it, and the public evidence for each one.
|
|
37
37
|
|
|
38
|
-
**Status: early release (0.1.
|
|
38
|
+
**Status: early release (0.1.4).** The command-line tool works, and the website runs at [sendermap.dev](https://sendermap.dev).
|
|
39
39
|
Needs Python 3.12 or later.
|
|
40
40
|
|
|
41
41
|
## What it shows
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
Type a domain, see every service that sends email as it, and the public evidence for each one.
|
|
9
9
|
|
|
10
|
-
**Status: early release (0.1.
|
|
10
|
+
**Status: early release (0.1.4).** The command-line tool works, and the website runs at [sendermap.dev](https://sendermap.dev).
|
|
11
11
|
Needs Python 3.12 or later.
|
|
12
12
|
|
|
13
13
|
## What it shows
|
|
@@ -210,8 +210,8 @@ def _capped(sending: tuple[str, ...]) -> tuple[tuple[str, ...], Observed]:
|
|
|
210
210
|
if not (dropped := len(sending) - len(picked)):
|
|
211
211
|
return picked, Observed()
|
|
212
212
|
happened = (
|
|
213
|
-
f"{dropped} of {len(sending)} not checked
|
|
214
|
-
"
|
|
213
|
+
f"{dropped} of {len(sending)} not checked. The first {MAX_SUBDOMAINS} in alphabetical "
|
|
214
|
+
f"order were checked (cap {MAX_SUBDOMAINS})."
|
|
215
215
|
)
|
|
216
216
|
gap = Gap("sending subdomains", happened, GapKind.SUBDOMAINS, dropped, MAX_SUBDOMAINS)
|
|
217
217
|
return picked, Observed(gaps=(gap,))
|
|
@@ -254,8 +254,8 @@ async def _subdomain(
|
|
|
254
254
|
swept = not any(a.happened.startswith(dkim.NO_SELECTORS) for a in keys.absences)
|
|
255
255
|
if swept and (unasked := len(set(order) - set(probed))):
|
|
256
256
|
happened = (
|
|
257
|
-
f"{unasked} of {len(order)} not asked
|
|
258
|
-
"
|
|
257
|
+
f"{unasked} of {len(order)} not asked. The first {SUBDOMAIN_PROBES} were asked, plus "
|
|
258
|
+
f"any the name's own SPF, MX or delegation points to (cap {SUBDOMAIN_PROBES})."
|
|
259
259
|
)
|
|
260
260
|
gap = Gap(f"DKIM selectors at {name}", happened, GapKind.DKIM, unasked, SUBDOMAIN_PROBES)
|
|
261
261
|
budget = Observed(gaps=(gap,))
|
|
@@ -17,7 +17,7 @@ from .view import (
|
|
|
17
17
|
AskedView,
|
|
18
18
|
ClaimedPart,
|
|
19
19
|
FindingView,
|
|
20
|
-
|
|
20
|
+
GapGroup,
|
|
21
21
|
Head,
|
|
22
22
|
OffView,
|
|
23
23
|
SenderMap,
|
|
@@ -25,6 +25,7 @@ from .view import (
|
|
|
25
25
|
TierView,
|
|
26
26
|
UnknownView,
|
|
27
27
|
View,
|
|
28
|
+
listed,
|
|
28
29
|
view,
|
|
29
30
|
)
|
|
30
31
|
|
|
@@ -49,13 +50,13 @@ def show(shown: View) -> str:
|
|
|
49
50
|
_list("Certificate transparency", "ct", shown.ct),
|
|
50
51
|
_absences(shown.absences),
|
|
51
52
|
_gaps(shown.gaps, shown.off),
|
|
52
|
-
_asked(shown.asked),
|
|
53
|
+
_asked(shown.asked, shown.asked_total),
|
|
53
54
|
]
|
|
54
55
|
return "".join(parts)
|
|
55
56
|
|
|
56
57
|
|
|
57
58
|
def slug(name: str) -> str:
|
|
58
|
-
"""A service name as a
|
|
59
|
+
"""A service name as a row's id suffix: lowercased, anything outside a to z and 0 to 9 a
|
|
59
60
|
hyphen. "Customer.io" is "customer-io"."""
|
|
60
61
|
return _NOT_SLUG.sub("-", name.lower())
|
|
61
62
|
|
|
@@ -107,10 +108,21 @@ def _head(head: Head) -> str:
|
|
|
107
108
|
"</header>",
|
|
108
109
|
]
|
|
109
110
|
if head.story:
|
|
110
|
-
lines
|
|
111
|
+
lines += ['<div class="lead">', *_story(head), "</div>"]
|
|
111
112
|
return "\n".join(lines) + "\n"
|
|
112
113
|
|
|
113
114
|
|
|
115
|
+
def _story(head: Head) -> list[str]:
|
|
116
|
+
"""The story one paragraph per sentence, the Not checked one muted. A story the rules did not
|
|
117
|
+
write comes as one string and stays one paragraph."""
|
|
118
|
+
said = head.sentences or (head.story,)
|
|
119
|
+
paragraphs = [
|
|
120
|
+
f'<p class="story-muted">{_e(s)}</p>' if s.startswith("Not checked") else f"<p>{_e(s)}</p>"
|
|
121
|
+
for s in said
|
|
122
|
+
]
|
|
123
|
+
return ['<div class="story">', *paragraphs, "</div>"]
|
|
124
|
+
|
|
125
|
+
|
|
114
126
|
def _findings(findings: tuple[FindingView, ...]) -> str:
|
|
115
127
|
if not findings:
|
|
116
128
|
return ""
|
|
@@ -128,36 +140,43 @@ def _findings(findings: tuple[FindingView, ...]) -> str:
|
|
|
128
140
|
|
|
129
141
|
|
|
130
142
|
def _tier(tier: TierView) -> str:
|
|
131
|
-
|
|
132
|
-
return _section(tier.tier, "tier", ['<div class="
|
|
143
|
+
rows = [line for service in tier.services for line in _row(service)]
|
|
144
|
+
return _section(tier.tier, "tier", ['<div class="rows">', *rows, "</div>"])
|
|
133
145
|
|
|
134
146
|
|
|
135
|
-
def
|
|
136
|
-
"""One service
|
|
137
|
-
|
|
147
|
+
def _row(service: ServiceView) -> list[str]:
|
|
148
|
+
"""One service as a row: name, category and where, tier badge, then the deciding kind with
|
|
149
|
+
the other kinds and the setup in one clause under it, and every record, with the setup item
|
|
150
|
+
by item, inside All records."""
|
|
138
151
|
badge = f"b-{slug(service.tier)}"
|
|
139
152
|
meta = service.category + (f", at {service.where}" if service.where else "")
|
|
153
|
+
first, rest = service.kinds[:1], service.kinds[1:]
|
|
140
154
|
lines = [
|
|
141
|
-
f'<article class="
|
|
142
|
-
f
|
|
143
|
-
f'<span class="badge {_e(badge)}">{_e(service.tier)}</span></div>',
|
|
155
|
+
f'<article class="row" id="service-{_e(slug(service.name))}">',
|
|
156
|
+
f"<h4>{_e(service.name)}</h4>",
|
|
144
157
|
f'<p class="meta">{_e(meta)}</p>',
|
|
145
|
-
'<
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
f": {_e(k.summary)}</li>"
|
|
149
|
-
for i, k in enumerate(service.kinds)
|
|
150
|
-
),
|
|
151
|
-
"</ul>",
|
|
158
|
+
f'<span class="badge {_e(badge)}">{_e(service.tier)}</span>',
|
|
159
|
+
'<div class="row-body">',
|
|
160
|
+
*(f"<p{_DECIDER}>{_kind_line(k.label, k.summary)}</p>" for k in first),
|
|
152
161
|
]
|
|
162
|
+
if rest:
|
|
163
|
+
lines += [
|
|
164
|
+
'<ul class="claims">',
|
|
165
|
+
*(f"<li>{_kind_line(k.label, k.summary)}</li>" for k in rest),
|
|
166
|
+
"</ul>",
|
|
167
|
+
]
|
|
153
168
|
if service.brief:
|
|
154
|
-
lines.append(f'<p class="
|
|
169
|
+
lines.append(f'<p class="setup">{_e(service.brief)}</p>')
|
|
155
170
|
full: list[str] = []
|
|
156
171
|
for kind in service.kinds:
|
|
157
172
|
full += [f'<p class="kind">{_e(kind.label)}</p>', *_records(kind.records)]
|
|
158
173
|
if service.setup and service.setup != service.brief:
|
|
159
|
-
full.append(f'<p class="
|
|
160
|
-
return [*lines, *_details("All records", "records", full), "</article>"]
|
|
174
|
+
full.append(f'<p class="setup">{_e(service.setup)}</p>')
|
|
175
|
+
return [*lines, *_details("All records", "records", full), "</div>", "</article>"]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _kind_line(label: str, summary: str) -> str:
|
|
179
|
+
return f'<span class="kind">{_e(label)}</span>: {_e(summary)}'
|
|
161
180
|
|
|
162
181
|
|
|
163
182
|
def _reporting(names: tuple[str, ...]) -> str:
|
|
@@ -208,29 +227,36 @@ def _absences(absences: tuple[AbsenceView, ...]) -> str:
|
|
|
208
227
|
return _section("Checked and not there", "absences", body)
|
|
209
228
|
|
|
210
229
|
|
|
211
|
-
def _gaps(gaps: tuple[
|
|
230
|
+
def _gaps(gaps: tuple[GapGroup, ...], off: tuple[OffView, ...]) -> str:
|
|
212
231
|
if not gaps and not off:
|
|
213
232
|
return ""
|
|
214
|
-
items = [f
|
|
233
|
+
items = [f"<li>{_gap(g)}</li>" for g in gaps]
|
|
215
234
|
items += [f'<li><span class="mono">{_e(o.check)}</span>: {_e(o.how)}</li>' for o in off]
|
|
216
235
|
return _section("Not checked", "not-checked", ['<ul class="gaps">', *items, "</ul>"])
|
|
217
236
|
|
|
218
237
|
|
|
219
|
-
def
|
|
220
|
-
"""
|
|
238
|
+
def _gap(group: GapGroup) -> str:
|
|
239
|
+
"""One group as a line of prose with only the names in mono."""
|
|
240
|
+
names = listed(f'<span class="mono">{_e(n)}</span>' for n in group.places())
|
|
241
|
+
where = f" at {names}" if group.named else ""
|
|
242
|
+
return f"{_e(group.asked)}{where}: {_e(group.outcome())}"
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _asked(asked: tuple[AskedView, ...], total: str) -> str:
|
|
246
|
+
"""What was asked in all, then at each name, counted, with every name inside Names asked."""
|
|
221
247
|
if not asked:
|
|
222
248
|
return ""
|
|
223
249
|
counts = [f'<li><span class="mono">{_e(a.where)}</span>: {_e(a.counts)}</li>' for a in asked]
|
|
224
|
-
body = ['<ul class="gaps">', *counts, "</ul>"]
|
|
250
|
+
body = [f'<p class="asked-total">{_e(total)}</p>', '<ul class="gaps">', *counts, "</ul>"]
|
|
225
251
|
names: list[str] = []
|
|
226
252
|
for a in asked:
|
|
227
253
|
if not a.names:
|
|
228
254
|
continue
|
|
229
255
|
names += [f'<h4 class="mono">{_e(a.where)}</h4>', "<ul>"]
|
|
230
|
-
for heading,
|
|
231
|
-
joined = _e(", ".join(
|
|
256
|
+
for heading, shown in a.names:
|
|
257
|
+
joined = _e(", ".join(shown))
|
|
232
258
|
names.append(
|
|
233
|
-
f'<li>{_e(heading)} ({len(
|
|
259
|
+
f'<li>{_e(heading)} ({len(shown)}): <span class="mono">{joined}</span></li>'
|
|
234
260
|
)
|
|
235
261
|
names.append("</ul>")
|
|
236
262
|
if names:
|
|
@@ -44,9 +44,9 @@ def _checks(shown: View) -> list[str]:
|
|
|
44
44
|
lines.append(f"- {a.asked}: {a.happened}")
|
|
45
45
|
lines += [f" - `{answer}`" for answer in a.answers]
|
|
46
46
|
if shown.gaps:
|
|
47
|
-
lines += [*_section("Not checked"), *(f"- {g
|
|
47
|
+
lines += [*_section("Not checked"), *(f"- {g}" for g in shown.gaps)]
|
|
48
48
|
if shown.asked:
|
|
49
|
-
lines += _section("What was asked")
|
|
49
|
+
lines += [*_section("What was asked"), shown.asked_total, ""]
|
|
50
50
|
lines += [f"- **{record.where}**: {record.counts}" for record in shown.asked]
|
|
51
51
|
if shown.off:
|
|
52
52
|
lines += [*_section("Not run"), *(f"- {o.check}: {o.how}" for o in shown.off)]
|
|
@@ -57,9 +57,10 @@ def _checks(shown: View, console: Console) -> None:
|
|
|
57
57
|
_line(f"{a.asked}: {a.happened}", console, 2)
|
|
58
58
|
for answer in a.answers:
|
|
59
59
|
_line(answer, console, 4, _DIM)
|
|
60
|
-
_dim_section("NOT CHECKED", [
|
|
60
|
+
_dim_section("NOT CHECKED", [str(g) for g in shown.gaps], console)
|
|
61
61
|
if shown.asked:
|
|
62
62
|
_heading("WHAT WAS ASKED", console)
|
|
63
|
+
_line(shown.asked_total, console, 2, _DIM)
|
|
63
64
|
for record in shown.asked:
|
|
64
65
|
_line(f"{record.where}: {record.counts}", console, 2, _DIM)
|
|
65
66
|
for heading, names in record.names:
|
|
@@ -46,7 +46,7 @@ __all__ = [
|
|
|
46
46
|
"AskedView",
|
|
47
47
|
"ClaimedPart",
|
|
48
48
|
"FindingView",
|
|
49
|
-
"
|
|
49
|
+
"GapGroup",
|
|
50
50
|
"Head",
|
|
51
51
|
"KindLine",
|
|
52
52
|
"NetworkGroup",
|
|
@@ -60,7 +60,9 @@ __all__ = [
|
|
|
60
60
|
"asked_in_full",
|
|
61
61
|
"claimed_parts",
|
|
62
62
|
"ct_lines",
|
|
63
|
+
"grouped_gaps",
|
|
63
64
|
"grouped_unknowns",
|
|
65
|
+
"listed",
|
|
64
66
|
"resolver_words",
|
|
65
67
|
"view",
|
|
66
68
|
]
|
|
@@ -185,9 +187,41 @@ class AbsenceView:
|
|
|
185
187
|
|
|
186
188
|
|
|
187
189
|
@dataclass(frozen=True, slots=True)
|
|
188
|
-
class
|
|
190
|
+
class GapGroup:
|
|
191
|
+
"""The gaps of one kind, cap and outcome, said once: what was asked without the name, the
|
|
192
|
+
names it applies to in map order, the outcome, the total count and the cap. `named` is
|
|
193
|
+
false for a gap with no name of its own, which holds the domain as its one name."""
|
|
194
|
+
|
|
189
195
|
asked: str
|
|
196
|
+
names: tuple[str, ...]
|
|
190
197
|
happened: str
|
|
198
|
+
kind: GapKind
|
|
199
|
+
count: int
|
|
200
|
+
cap: int
|
|
201
|
+
named: bool = True
|
|
202
|
+
|
|
203
|
+
def places(self) -> tuple[str, ...]:
|
|
204
|
+
"""The names as a line shows them: those that share a parent lead with their own labels
|
|
205
|
+
and the last carries the parent, as in "marketing", "receipts", "send.glossier.com"."""
|
|
206
|
+
if len(self.names) < 2:
|
|
207
|
+
return self.names
|
|
208
|
+
parent = _shared_parent(self.names)
|
|
209
|
+
if not parent:
|
|
210
|
+
return self.names
|
|
211
|
+
cut = len(parent) + 1
|
|
212
|
+
return (*(n[:-cut] for n in self.names[:-1]), self.names[-1])
|
|
213
|
+
|
|
214
|
+
def outcome(self) -> str:
|
|
215
|
+
"""What happened, with "at each" after its first sentence when it applies to several
|
|
216
|
+
names: "82 of 109 not asked at each. The first 27 were asked, ..."."""
|
|
217
|
+
if len(self.names) < 2:
|
|
218
|
+
return self.happened
|
|
219
|
+
first, stop, rest = self.happened.partition(". ")
|
|
220
|
+
return f"{first} at each{stop}{rest}"
|
|
221
|
+
|
|
222
|
+
def __str__(self) -> str:
|
|
223
|
+
where = f" at {listed(self.places())}" if self.named else ""
|
|
224
|
+
return f"{self.asked}{where}: {self.outcome()}"
|
|
191
225
|
|
|
192
226
|
|
|
193
227
|
@dataclass(frozen=True, slots=True)
|
|
@@ -220,10 +254,12 @@ class View:
|
|
|
220
254
|
unknowns: tuple[UnknownView, ...]
|
|
221
255
|
ct: tuple[str, ...]
|
|
222
256
|
absences: tuple[AbsenceView, ...]
|
|
223
|
-
gaps: tuple[
|
|
257
|
+
gaps: tuple[GapGroup, ...]
|
|
224
258
|
asked: tuple[AskedView, ...]
|
|
225
259
|
evidence: bool
|
|
226
260
|
off: tuple[OffView, ...] = ()
|
|
261
|
+
# What was asked in all, as one line: "739 lookups at 4 names".
|
|
262
|
+
asked_total: str = ""
|
|
227
263
|
|
|
228
264
|
|
|
229
265
|
def view(result: SenderMap, *, evidence: bool) -> View:
|
|
@@ -253,13 +289,14 @@ def view(result: SenderMap, *, evidence: bool) -> View:
|
|
|
253
289
|
absences=tuple(
|
|
254
290
|
AbsenceView(a.asked, a.happened, tuple(map(str, a.evidence))) for a in result.absences
|
|
255
291
|
),
|
|
256
|
-
gaps=tuple(
|
|
292
|
+
gaps=tuple(grouped_gaps(result.gaps, domain)),
|
|
257
293
|
asked=tuple(
|
|
258
294
|
AskedView(c.where, asked(c), tuple(asked_in_full(c)) if evidence else ())
|
|
259
295
|
for c in result.checked
|
|
260
296
|
),
|
|
261
297
|
evidence=evidence,
|
|
262
298
|
off=tuple(_off(g) for g in result.gaps if g.kind is GapKind.OFF),
|
|
299
|
+
asked_total=_asked_total(result),
|
|
263
300
|
)
|
|
264
301
|
|
|
265
302
|
|
|
@@ -326,7 +363,7 @@ def _findings(findings: list[Finding], evidence: bool) -> list[FindingView]:
|
|
|
326
363
|
shown.append(_finding(members[0][1], evidence))
|
|
327
364
|
continue
|
|
328
365
|
where, bits, detail = item
|
|
329
|
-
selectors =
|
|
366
|
+
selectors = listed(selector for selector, _ in members)
|
|
330
367
|
title = f"{_count_words(len(members))} DKIM keys at {where} are {bits}-bit: {selectors}"
|
|
331
368
|
records = tuple(str(e) for _, f in members for e in f.evidence) if evidence else ()
|
|
332
369
|
shown.append(FindingView(members[0][1].severity.value, title, detail, records))
|
|
@@ -432,7 +469,7 @@ def _kind_line(kind: Kind, claims: list[Claim], base: str, evidence: bool) -> Ki
|
|
|
432
469
|
summary += f" ({', '.join(notes)})"
|
|
433
470
|
names = list(dict.fromkeys(c.where for c in claims))
|
|
434
471
|
if names != [base]:
|
|
435
|
-
summary += f" at {
|
|
472
|
+
summary += f" at {listed(names)}"
|
|
436
473
|
records = tuple(_record(c) for c in claims) if evidence else ()
|
|
437
474
|
shown = one if len({c.evidence for c in claims}) == 1 else several
|
|
438
475
|
return KindLine(shown, summary, records)
|
|
@@ -459,7 +496,7 @@ def _bits(claims: list[Claim]) -> str:
|
|
|
459
496
|
if not distinct:
|
|
460
497
|
return ""
|
|
461
498
|
if len(distinct) > 1:
|
|
462
|
-
return f"{
|
|
499
|
+
return f"{listed([str(n) for n in distinct])} bit"
|
|
463
500
|
prefix = {1: "", 2: "both "}.get(len(sizes), "all ")
|
|
464
501
|
return f"{prefix}{distinct[0]} bit"
|
|
465
502
|
|
|
@@ -510,7 +547,7 @@ def _setup_clause(
|
|
|
510
547
|
if not said:
|
|
511
548
|
return head
|
|
512
549
|
one_name = {s.where for s in service.setups} == {base}
|
|
513
|
-
parts = [text if one_name else f"{text} at {
|
|
550
|
+
parts = [text if one_name else f"{text} at {listed(names)}" for text, names in said.items()]
|
|
514
551
|
return f"{head}: {', '.join(parts)}"
|
|
515
552
|
|
|
516
553
|
|
|
@@ -561,8 +598,8 @@ class NetworkGroup:
|
|
|
561
598
|
parts = [f"{ranges} range{'s' if ranges != 1 else ''}"] if ranges else []
|
|
562
599
|
if addresses:
|
|
563
600
|
parts.append(f"{addresses} address{'es' if addresses != 1 else ''}")
|
|
564
|
-
|
|
565
|
-
return f"{' and '.join(parts)} in {self.asn} ({self.name}){
|
|
601
|
+
by = f"; listed by {self.listed}" if self.listed else ""
|
|
602
|
+
return f"{' and '.join(parts)} in {self.asn} ({self.name}){by}"
|
|
566
603
|
|
|
567
604
|
|
|
568
605
|
def grouped_unknowns(unknowns: tuple[Unknown, ...]) -> list[Unknown | NetworkGroup]:
|
|
@@ -577,8 +614,8 @@ def grouped_unknowns(unknowns: tuple[Unknown, ...]) -> list[Unknown | NetworkGro
|
|
|
577
614
|
if match is None:
|
|
578
615
|
order.append(u)
|
|
579
616
|
continue
|
|
580
|
-
cidr, asn, name,
|
|
581
|
-
key = (asn, name,
|
|
617
|
+
cidr, asn, name, publishers = match.groups()
|
|
618
|
+
key = (asn, name, publishers or "", u.where)
|
|
582
619
|
if key not in keyed:
|
|
583
620
|
keyed[key] = []
|
|
584
621
|
order.append(key)
|
|
@@ -606,6 +643,51 @@ def _unknown(item: Unknown | NetworkGroup, domain: str, evidence: bool) -> Unkno
|
|
|
606
643
|
return UnknownView(item.heading(), values, where, True, tuple(shown), len(records) - len(shown))
|
|
607
644
|
|
|
608
645
|
|
|
646
|
+
# ---------------------------------------------------------------- not checked
|
|
647
|
+
|
|
648
|
+
# A gap's question that ends at the name it was asked at: "DKIM selectors at send.glossier.com".
|
|
649
|
+
_AT_NAME = re.compile(r"^(.+) at ([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+)$")
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def grouped_gaps(gaps: Iterable[Gap], domain: str) -> list[GapGroup]:
|
|
653
|
+
"""The gaps that are not turned-off checks, one group per question, kind, cap and outcome,
|
|
654
|
+
where the first of them stood, the names in map order and the counts added up. A gap whose
|
|
655
|
+
question names no name of its own carries the domain."""
|
|
656
|
+
groups: dict[tuple[str, str, GapKind, int, bool], list[tuple[str, Gap]]] = {}
|
|
657
|
+
for gap in gaps:
|
|
658
|
+
if gap.kind is GapKind.OFF:
|
|
659
|
+
continue
|
|
660
|
+
match = _AT_NAME.match(gap.asked)
|
|
661
|
+
question, name = match.groups() if match else (gap.asked, domain)
|
|
662
|
+
key = (question, gap.happened, gap.kind, gap.cap, match is not None)
|
|
663
|
+
groups.setdefault(key, []).append((name, gap))
|
|
664
|
+
return [
|
|
665
|
+
GapGroup(
|
|
666
|
+
question,
|
|
667
|
+
tuple(dict.fromkeys(name for name, _ in members)),
|
|
668
|
+
happened,
|
|
669
|
+
kind,
|
|
670
|
+
sum(g.count for _, g in members),
|
|
671
|
+
cap,
|
|
672
|
+
named,
|
|
673
|
+
)
|
|
674
|
+
for (question, happened, kind, cap, named), members in groups.items()
|
|
675
|
+
]
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _shared_parent(names: Iterable[str]) -> str:
|
|
679
|
+
"""The longest parent of two labels or more that every name sits under, or empty."""
|
|
680
|
+
split = [n.split(".") for n in names]
|
|
681
|
+
shared: list[str] = []
|
|
682
|
+
for labels in zip(*(reversed(s) for s in split), strict=False):
|
|
683
|
+
if len(set(labels)) != 1:
|
|
684
|
+
break
|
|
685
|
+
shared.insert(0, labels[0])
|
|
686
|
+
while shared and any(len(s) <= len(shared) for s in split):
|
|
687
|
+
shared.pop(0)
|
|
688
|
+
return ".".join(shared) if len(shared) >= 2 else ""
|
|
689
|
+
|
|
690
|
+
|
|
609
691
|
# ---------------------------------------------------------------- claimed, CT, asked
|
|
610
692
|
|
|
611
693
|
|
|
@@ -630,6 +712,14 @@ def _reconciled(item: Reconciled, part: str) -> str:
|
|
|
630
712
|
return line
|
|
631
713
|
|
|
632
714
|
|
|
715
|
+
def _asked_total(result: SenderMap) -> str:
|
|
716
|
+
if not result.checked:
|
|
717
|
+
return ""
|
|
718
|
+
lookups, names = result.run.lookups, len(result.checked)
|
|
719
|
+
plural = ("s" if lookups != 1 else "", "s" if names != 1 else "")
|
|
720
|
+
return f"{lookups} lookup{plural[0]} at {names} name{plural[1]}"
|
|
721
|
+
|
|
722
|
+
|
|
633
723
|
def asked(checked: Checked) -> str:
|
|
634
724
|
"""What was asked at one name, counted: "172 DKIM selectors, 23 CNAME labels, 2 pages"."""
|
|
635
725
|
counts = (
|
|
@@ -686,6 +776,7 @@ def ct_lines(result: SenderMap) -> list[str]:
|
|
|
686
776
|
return lines
|
|
687
777
|
|
|
688
778
|
|
|
689
|
-
def
|
|
779
|
+
def listed(items: Iterable[str]) -> str:
|
|
780
|
+
"""The items as prose: a; a and b; a, b and c."""
|
|
690
781
|
names = list(items)
|
|
691
782
|
return names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"
|
|
@@ -294,7 +294,7 @@ def _unfound(rule: SetupRule, where: str, question: str, asked: Asked) -> SetupI
|
|
|
294
294
|
unasked = [s for s in current if s not in reads]
|
|
295
295
|
if unasked and not (absent and absent.happened.startswith(NO_SELECTORS)):
|
|
296
296
|
cap = next((g for g in asked.gaps if g.asked == f"DKIM selectors at {where}"), None)
|
|
297
|
-
why = f"{cap.asked}: {cap.happened}" if cap else "not asked"
|
|
297
|
+
why = f"{cap.asked}: {cap.happened.rstrip('.')}" if cap else "not asked"
|
|
298
298
|
return _not_checked(rule, question, f"{_listed(unasked)} not asked ({why})")
|
|
299
299
|
none = [r for r in reads.values() if r]
|
|
300
300
|
evidence = tuple(e for r in none for e in r.evidence)
|
|
@@ -211,8 +211,8 @@ async def collect(ctx: Context, logs: Logs) -> tuple[Observed, CtSummary]:
|
|
|
211
211
|
observed = Observed()
|
|
212
212
|
if dropped := len(names) - len(picked):
|
|
213
213
|
happened = (
|
|
214
|
-
f"{dropped} of {len(names)} not resolved
|
|
215
|
-
"
|
|
214
|
+
f"{dropped} of {len(names)} not resolved. The first {MAX_NAMES} in alphabetical "
|
|
215
|
+
f"order were resolved (cap {MAX_NAMES})."
|
|
216
216
|
)
|
|
217
217
|
gap = Gap(
|
|
218
218
|
"names from certificate logs", happened, GapKind.CERTIFICATE_LOG, dropped, MAX_NAMES
|
|
@@ -40,7 +40,10 @@ RANGE_KINDS = frozenset({"IP address in SPF", "IP range in SPF"})
|
|
|
40
40
|
PUBLISHED = "published ranges: SPF of "
|
|
41
41
|
# How far below a service's own SPF record its published ranges are read.
|
|
42
42
|
MAX_PUBLISHED_DEPTH = 3
|
|
43
|
-
TOO_DEEP =
|
|
43
|
+
TOO_DEEP = (
|
|
44
|
+
f"includes more than {MAX_PUBLISHED_DEPTH} levels deep not read. Includes up to "
|
|
45
|
+
f"{MAX_PUBLISHED_DEPTH} levels deep were read (cap {MAX_PUBLISHED_DEPTH} levels)."
|
|
46
|
+
)
|
|
44
47
|
|
|
45
48
|
Network = ipaddress.IPv4Network | ipaddress.IPv6Network
|
|
46
49
|
# {network: {service: the term in that service's own SPF record that publishes it}}
|
|
@@ -93,7 +96,10 @@ async def collect(ctx: Context, where: str, ranges: list[tuple[str, Evidence]])
|
|
|
93
96
|
picked = rest[:MAX_RANGES]
|
|
94
97
|
if dropped := len(rest) - len(picked):
|
|
95
98
|
asked = f"network owners of the IP ranges in SPF at {where}"
|
|
96
|
-
happened =
|
|
99
|
+
happened = (
|
|
100
|
+
f"{dropped} of {len(rest)} not looked up. The first {MAX_RANGES} were looked up "
|
|
101
|
+
f"(cap {MAX_RANGES})."
|
|
102
|
+
)
|
|
97
103
|
gaps += (Gap(asked, happened, GapKind.NETWORKS, dropped, MAX_RANGES),)
|
|
98
104
|
owners = await asyncio.gather(*(range_owner(ctx, n) for _, _, n, _ in picked))
|
|
99
105
|
unknowns: list[Unknown] = []
|
|
@@ -130,7 +130,10 @@ async def collect(ctx: Context, where: str) -> tuple[Observed, SpfTree]:
|
|
|
130
130
|
walked = await _Walk(ctx, where).visit(where, record, depth=0, inside=None, authorized=True)
|
|
131
131
|
if too_deep := list(dict.fromkeys(walked.too_deep)):
|
|
132
132
|
asked = f"SPF includes more than {MAX_DEPTH} levels below {where}: {', '.join(too_deep)}"
|
|
133
|
-
happened =
|
|
133
|
+
happened = (
|
|
134
|
+
f"{len(too_deep)} not walked. Includes up to {MAX_DEPTH} levels deep were walked "
|
|
135
|
+
f"(cap {MAX_DEPTH} levels)."
|
|
136
|
+
)
|
|
134
137
|
gap = Gap(asked, happened, GapKind.SPF, len(too_deep), MAX_DEPTH)
|
|
135
138
|
walked += _Walked(observed=Observed(gaps=(gap,)))
|
|
136
139
|
tree = SpfTree(
|
|
@@ -157,7 +157,8 @@ def _left(asks: Sequence[_Ask]) -> Observed:
|
|
|
157
157
|
gaps = tuple(
|
|
158
158
|
Gap(
|
|
159
159
|
f"{EVALUATION} at {where} for {', '.join(vendors)}",
|
|
160
|
-
f"not evaluated
|
|
160
|
+
f"not evaluated. Evaluations ran until the budget of {BUDGET} lookups per map was "
|
|
161
|
+
f"spent (cap {BUDGET} lookups).",
|
|
161
162
|
GapKind.SPF,
|
|
162
163
|
len(vendors),
|
|
163
164
|
BUDGET,
|
|
@@ -143,13 +143,16 @@ def _from_certificates(ctx: Context, logged: Iterable[str]) -> tuple[set[str], t
|
|
|
143
143
|
(deep if too_deep else names).add(name)
|
|
144
144
|
gaps: tuple[Gap, ...] = ()
|
|
145
145
|
if deep:
|
|
146
|
-
happened =
|
|
146
|
+
happened = (
|
|
147
|
+
f"{len(deep)} not asked. Names up to {MAX_CT_DEPTH} labels below the domain were "
|
|
148
|
+
f"asked (cap {MAX_CT_DEPTH} labels)."
|
|
149
|
+
)
|
|
147
150
|
asked = f"certificate log names more than {MAX_CT_DEPTH} labels deep"
|
|
148
151
|
gaps += (Gap(asked, happened, GapKind.CERTIFICATE_LOG, len(deep), MAX_CT_DEPTH),)
|
|
149
152
|
if len(names) > MAX_FROM_CT:
|
|
150
153
|
happened = (
|
|
151
|
-
f"{len(names) - MAX_FROM_CT} of {len(names)} not asked
|
|
152
|
-
"
|
|
154
|
+
f"{len(names) - MAX_FROM_CT} of {len(names)} not asked. The first {MAX_FROM_CT} in "
|
|
155
|
+
f"alphabetical order were asked (cap {MAX_FROM_CT})."
|
|
153
156
|
)
|
|
154
157
|
dropped = len(names) - MAX_FROM_CT
|
|
155
158
|
gaps += (
|
|
@@ -97,7 +97,9 @@ def _page(ctx: Context, url: str) -> tuple[Checked, ...]:
|
|
|
97
97
|
|
|
98
98
|
|
|
99
99
|
def _cut(asked: str, cut: bool, kind: GapKind) -> tuple[Gap, ...]:
|
|
100
|
-
happened =
|
|
100
|
+
happened = (
|
|
101
|
+
f"the rest was not read. The first {MAX_BYTES:,} bytes were read (cap {MAX_BYTES:,} bytes)."
|
|
102
|
+
)
|
|
101
103
|
return (Gap(asked, happened, kind, cap=MAX_BYTES),) if cut else ()
|
|
102
104
|
|
|
103
105
|
|