citewatch 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. citewatch/__init__.py +57 -0
  2. citewatch/cards.py +557 -0
  3. citewatch/cli.py +863 -0
  4. citewatch/cost.py +262 -0
  5. citewatch/dates/__init__.py +34 -0
  6. citewatch/dates/candidates.py +408 -0
  7. citewatch/dates/pipeline.py +119 -0
  8. citewatch/dates/resolve.py +101 -0
  9. citewatch/dates/rules.py +80 -0
  10. citewatch/dates/sitemaps.py +135 -0
  11. citewatch/demo/__init__.py +7 -0
  12. citewatch/demo/component.py +138 -0
  13. citewatch/demo/guard.py +153 -0
  14. citewatch/demo/kv.py +212 -0
  15. citewatch/demo/live.py +273 -0
  16. citewatch/demo/logs.py +42 -0
  17. citewatch/demo/lookup.py +157 -0
  18. citewatch/demo/make_code_hash.py +49 -0
  19. citewatch/demo/markup.py +38 -0
  20. citewatch/demo/passkeys.py +386 -0
  21. citewatch/demo/settings.py +66 -0
  22. citewatch/env.py +48 -0
  23. citewatch/examples/uk-health-summary.json.gz +0 -0
  24. citewatch/extract/__init__.py +18 -0
  25. citewatch/extract/extractors.py +147 -0
  26. citewatch/extract/stage.py +189 -0
  27. citewatch/extract/urls.py +108 -0
  28. citewatch/fetch/__init__.py +28 -0
  29. citewatch/fetch/client.py +434 -0
  30. citewatch/fetch/history.py +132 -0
  31. citewatch/fetch/keys.py +125 -0
  32. citewatch/fetch/log.py +56 -0
  33. citewatch/fetch/pages.py +266 -0
  34. citewatch/fetch/rate.py +99 -0
  35. citewatch/fetch/robots.py +140 -0
  36. citewatch/handcheck.py +221 -0
  37. citewatch/ledger.py +75 -0
  38. citewatch/metrics/__init__.py +5 -0
  39. citewatch/metrics/compute.py +602 -0
  40. citewatch/metrics/data.py +166 -0
  41. citewatch/metrics/stats.py +120 -0
  42. citewatch/metrics/summary.py +231 -0
  43. citewatch/metrics/views.py +44 -0
  44. citewatch/prices.yaml +19 -0
  45. citewatch/py.typed +0 -0
  46. citewatch/questions.py +84 -0
  47. citewatch/report/__init__.py +5 -0
  48. citewatch/report/dashboard.py +762 -0
  49. citewatch/report/post.py +235 -0
  50. citewatch/runners/__init__.py +29 -0
  51. citewatch/runners/anthropic_runner.py +168 -0
  52. citewatch/runners/base.py +144 -0
  53. citewatch/runners/openai_runner.py +117 -0
  54. citewatch/runners/schedule.py +423 -0
  55. citewatch/runs.py +202 -0
  56. citewatch/schemas/__init__.py +61 -0
  57. citewatch/schemas/_base.py +160 -0
  58. citewatch/schemas/_iso.py +257 -0
  59. citewatch/schemas/cards.py +120 -0
  60. citewatch/schemas/config.py +232 -0
  61. citewatch/schemas/pages.py +92 -0
  62. citewatch/schemas/study.py +133 -0
  63. citewatch/schemas/summary.py +187 -0
  64. citewatch/schemas/thresholds.py +165 -0
  65. citewatch/store.py +127 -0
  66. citewatch/support/__init__.py +5 -0
  67. citewatch/support/calibrate.py +83 -0
  68. citewatch/support/judge.py +455 -0
  69. citewatch/support/report.py +191 -0
  70. citewatch/support/sample.py +84 -0
  71. citewatch/templates/RUBRIC.md +9 -0
  72. citewatch/templates/env.example +3 -0
  73. citewatch/templates/questions.jsonl +2 -0
  74. citewatch/templates/study.yaml +23 -0
  75. citewatch/thresholds.py +129 -0
  76. citewatch-0.1.0.dist-info/METADATA +210 -0
  77. citewatch-0.1.0.dist-info/RECORD +80 -0
  78. citewatch-0.1.0.dist-info/WHEEL +4 -0
  79. citewatch-0.1.0.dist-info/entry_points.txt +2 -0
  80. citewatch-0.1.0.dist-info/licenses/LICENSE +21 -0
citewatch/__init__.py ADDED
@@ -0,0 +1,57 @@
1
+ """citewatch: measure which web pages AI answers with web search cite, and how fresh they are.
2
+
3
+ Public API (semver, see docs/versioning.md): the results format in `citewatch.schemas`, and
4
+ `canonical_url`, `publisher_of`, `UrlError`, `lookup`, `read_cards` and `NotCited`, importable
5
+ from here or from `citewatch.extract.urls` and `citewatch.cards`. They load on first use, so
6
+ `import citewatch` (and the CLI) stays fast.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from importlib import import_module
12
+ from importlib.metadata import PackageNotFoundError, version
13
+ from typing import TYPE_CHECKING, Any
14
+
15
+ try:
16
+ __version__ = version("citewatch")
17
+ except PackageNotFoundError: # running from a source tree that was never installed
18
+ __version__ = "0.0.0"
19
+
20
+ if TYPE_CHECKING:
21
+ from citewatch import schemas
22
+ from citewatch.cards import NotCited, lookup, read_cards
23
+ from citewatch.extract.urls import UrlError, canonical_url, publisher_of
24
+
25
+ _LAZY = {
26
+ "canonical_url": "citewatch.extract.urls",
27
+ "publisher_of": "citewatch.extract.urls",
28
+ "UrlError": "citewatch.extract.urls",
29
+ "lookup": "citewatch.cards",
30
+ "read_cards": "citewatch.cards",
31
+ "NotCited": "citewatch.cards",
32
+ }
33
+
34
+ __all__ = [
35
+ "NotCited",
36
+ "UrlError",
37
+ "__version__",
38
+ "canonical_url",
39
+ "lookup",
40
+ "publisher_of",
41
+ "read_cards",
42
+ "schemas",
43
+ ]
44
+
45
+
46
+ def __getattr__(name: str) -> Any:
47
+ if name == "schemas":
48
+ return import_module("citewatch.schemas")
49
+ if name in _LAZY:
50
+ value = getattr(import_module(_LAZY[name]), name)
51
+ globals()[name] = value
52
+ return value
53
+ raise AttributeError(f"module 'citewatch' has no attribute {name!r}")
54
+
55
+
56
+ def __dir__() -> list[str]:
57
+ return sorted({*globals(), *__all__})
citewatch/cards.py ADDED
@@ -0,0 +1,557 @@
1
+ """Domain cards, lookup and the improvement report (node D5.1, Tech Spec §9, OWNER D-65).
2
+
3
+ One `DomainCard` per publisher cited in the scored runs, built from `summary.json` and the
4
+ frozen study data, so the demo answers "is my site cited, how often, how fresh, and what would
5
+ help?" without calling a model. The improvement rows describe sourcing, dates and readability
6
+ only; they never judge medical accuracy (NFR4), and they never state more than the data shows
7
+ about a publisher's site (D-69).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import datetime as dt
13
+ import json
14
+ from collections import Counter, defaultdict
15
+ from collections.abc import Iterable, Mapping, Sequence
16
+ from dataclasses import dataclass, field
17
+ from pathlib import Path
18
+ from typing import Any
19
+ from urllib.parse import urlsplit
20
+
21
+ from citewatch.extract.urls import INSTITUTIONAL, UrlError, publisher_of
22
+ from citewatch.metrics.data import StudyData
23
+ from citewatch.schemas import DomainCard, Improvement, StudySummary
24
+
25
+ CARDS_FILE = "domain_cards.json"
26
+ CITED_INSTEAD_TOP = 5
27
+ EXAMPLES = 3
28
+ # date sources a reader never sees on the page: the search tool's page age, sitemaps, headers
29
+ HIDDEN_SOURCES = frozenset({"sitemap", "http_header", "page_age"})
30
+ TOP_FOR_INTENT = 5
31
+ DOCS = ("is a document rather than a web page", "are documents rather than web pages")
32
+ TS_PAGES = "dated pages that loaded and were cited on time-sensitive questions"
33
+ # a missing intent is reported only for publishers cited broadly enough for the gap to mean
34
+ # something: on this many distinct questions, across at least two intents (D-69)
35
+ INTENT_MIN_QUESTIONS = 5
36
+ INTENT_MIN_INTENTS = 2
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class _Study:
41
+ """Everything the improvement rows read, loaded once."""
42
+
43
+ meta: Mapping[str, Mapping[str, Any]]
44
+ robots: Mapping[str, int | None] # host -> status of its robots.txt, if never readable
45
+ verdicts: Mapping[str, Mapping[str, str]]
46
+ hand: Mapping[str, str]
47
+ cite_pub: Mapping[str, str]
48
+ newest_ts: Mapping[str, dt.date] # url -> latest time-sensitive answer citing it
49
+ intent_leaders: Mapping[str, tuple[str, ...]]
50
+ stale_days: int
51
+ answers: Sequence[Any] = field(default_factory=tuple)
52
+
53
+
54
+ def _key(*parts: object) -> str:
55
+ return "/".join(str(p) for p in parts)
56
+
57
+
58
+ def _rows(path: Path) -> Iterable[dict[str, Any]]:
59
+ if path.is_file():
60
+ for x in path.read_text(encoding="utf-8").splitlines():
61
+ if x.strip():
62
+ yield json.loads(x)
63
+
64
+
65
+ def _unreadable_robots(base: Path) -> dict[str, int | None]:
66
+ """Hosts whose robots.txt never returned 200, with the last status seen (H1, D-69): the
67
+ fetcher treats an unreadable robots.txt as disallowing the host, which is not the same as
68
+ the site's robots.txt disallowing citewatch."""
69
+ seen: dict[str, list[int | None]] = defaultdict(list)
70
+ for r in _rows(base / "data" / "fetch_log.jsonl"):
71
+ if urlsplit(r["url"]).path == "/robots.txt":
72
+ seen[r["host"]].append(r.get("status"))
73
+
74
+ def readable(h: str) -> bool:
75
+ if 200 in seen[h]:
76
+ return True
77
+ # robots rules are per origin: x's robots.txt counts as read only when it redirected
78
+ # (3xx) to its www/non-www twin and that twin answered 200 (pass 2, L-d)
79
+ twin = h.removeprefix("www.") if h.startswith("www.") else f"www.{h}"
80
+ redirected = any(s is not None and 300 <= s < 400 for s in seen[h])
81
+ return redirected and 200 in seen.get(twin, [])
82
+
83
+ return {h: s[-1] for h, s in list(seen.items()) if not readable(h)}
84
+
85
+
86
+ def _verdicts(base: Path) -> dict[str, dict[str, str]]:
87
+ """citation_ref -> {judge: verdict} from the support check, if it ran."""
88
+ out: dict[str, dict[str, str]] = defaultdict(dict)
89
+ for v in _rows(base / "data" / "support_verdicts.jsonl"):
90
+ out[v["citation_ref"]][v["judge"]] = v["verdict"]
91
+ return out
92
+
93
+
94
+ def _citation_publishers(base: Path, runs: set[int]) -> dict[str, str]:
95
+ out = {}
96
+ for c in _rows(base / "data" / "citations.jsonl"):
97
+ if c["run"] in runs:
98
+ out[f"{c['run']}:{c['question_id']}:{c['model']}:{c['ordinal']}"] = c["publisher"]
99
+ return out
100
+
101
+
102
+ def _of(k: int, n: int, noun: str, verbs: tuple[str, str] = ("is", "are")) -> str:
103
+ """'1 of 3 cited pages is' / '2 of 3 cited pages are'."""
104
+ return f"{k} of {n} {noun} {verbs[0] if k == 1 else verbs[1]}"
105
+
106
+
107
+ def _robots_failure(status: int | None) -> str:
108
+ """What happened to a robots.txt request, in words."""
109
+ if status is None:
110
+ return "did not respond"
111
+ if 300 <= status < 400:
112
+ return f"redirected away from /robots.txt (HTTP {status})"
113
+ if status in (401, 403, 429):
114
+ return f"refused the request (HTTP {status})"
115
+ if status >= 500:
116
+ return f"returned a server error (HTTP {status})"
117
+ return f"could not be read (HTTP {status})"
118
+
119
+
120
+ def cited_instead(data: StudyData, publisher: str) -> tuple[str, ...]:
121
+ """§9: on questions where `publisher` was cited by one model or run but not another, the
122
+ publishers cited in the answers that left it out, by count then name."""
123
+ by_q: dict[str, list[Any]] = defaultdict(list)
124
+ for a in data.answers:
125
+ if a.valid:
126
+ by_q[a.qid].append(a)
127
+ counts: Counter[str] = Counter()
128
+ for answers in by_q.values():
129
+ has = [any(c.publisher == publisher for c in a.cites) for a in answers]
130
+ if any(has) and not all(has):
131
+ for a, h in zip(answers, has, strict=True):
132
+ if not h:
133
+ counts.update({c.publisher for c in a.cites} - {publisher})
134
+ ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
135
+ return tuple(p for p, _ in ranked[:CITED_INSTEAD_TOP])
136
+
137
+
138
+ def _intent_leaders(summary: StudySummary) -> dict[str, tuple[str, ...]]:
139
+ """intent -> publishers cited in most answers to that intent (presence k over model/run)."""
140
+ k: dict[str, Counter[str]] = defaultdict(Counter)
141
+ for r in summary.share:
142
+ if r.kind == "presence" and r.intent:
143
+ k[r.intent][r.publisher] += r.share.k or 0
144
+ return {
145
+ i: tuple(p for p, n in sorted(c.items(), key=lambda kv: (-kv[1], kv[0])) if n)[
146
+ :TOP_FOR_INTENT
147
+ ]
148
+ for i, c in k.items()
149
+ }
150
+
151
+
152
+ def _improvements(publisher: str, summary: StudySummary, st: _Study) -> list[Improvement]:
153
+ rows: list[Improvement] = []
154
+ answers = st.answers
155
+ urls = sorted({c.url for a in answers for c in a.cites if c.publisher == publisher})
156
+ meta = st.meta
157
+ outcome = {u: meta.get(u, {}).get("outcome", "missing") for u in urls}
158
+
159
+ # intent_absent: broad presence elsewhere, none on an intent (summary presence rows)
160
+ cited_intents = {
161
+ r.intent
162
+ for r in summary.share
163
+ if r.kind == "presence" and r.publisher == publisher and r.intent and (r.share.k or 0) > 0
164
+ }
165
+ intents = sorted({a.intent for a in answers})
166
+ questions = {a.qid for a in answers if any(c.publisher == publisher for c in a.cites)}
167
+ absent = [i for i in intents if i not in cited_intents]
168
+ if (
169
+ absent
170
+ and len(questions) >= INTENT_MIN_QUESTIONS
171
+ and len(cited_intents) >= INTENT_MIN_INTENTS
172
+ ):
173
+ names = ", ".join(i.replace("_", " ") for i in absent)
174
+ leaders = sorted(
175
+ {p for i in absent for p in st.intent_leaders.get(i, ())[:3]} - {publisher}
176
+ )
177
+ lead = f" Cited most on those questions: {', '.join(leaders)}." if leaders else ""
178
+ rows.append(
179
+ Improvement(
180
+ code="intent_absent",
181
+ finding=f"Cited on {len(questions)} questions across {len(cited_intents)} topics, "
182
+ f"but on no {names} question. This may simply be outside the site's remit.",
183
+ fix="If the site covers these topics, check that a page answers such questions "
184
+ f"directly, in plain text near the top, with a visible date.{lead}",
185
+ count=len(absent),
186
+ of=len(intents),
187
+ sources=("summary.json",),
188
+ examples=tuple(absent[:EXAMPLES]),
189
+ )
190
+ )
191
+
192
+ ok = [u for u in urls if outcome[u] == "ok"]
193
+ undated = [u for u in ok if not meta[u].get("effective_date")]
194
+ if undated:
195
+ rows.append(
196
+ Improvement(
197
+ code="undated",
198
+ finding=f"{_of(len(undated), len(ok), 'cited pages that loaded', ('has', 'have'))} "
199
+ "no date citewatch could read on the page.",
200
+ fix='Add a visible "last updated" date and schema.org dateModified to each page.',
201
+ count=len(undated),
202
+ of=len(ok),
203
+ sources=("page_meta.jsonl",),
204
+ examples=tuple(undated[:EXAMPLES]),
205
+ )
206
+ )
207
+ dated_ok = [u for u in ok if meta[u].get("effective_date")]
208
+ hidden = [
209
+ u
210
+ for u in dated_ok
211
+ if meta[u].get("effective_source") in HIDDEN_SOURCES
212
+ and not any( # any date shown on the page itself, even a published one (pass 2, M-a)
213
+ c.get("source") not in HIDDEN_SOURCES and not c.get("rejected")
214
+ for c in meta[u].get("candidates", ())
215
+ )
216
+ ]
217
+ if hidden:
218
+ rows.append(
219
+ Improvement(
220
+ code="hidden_date",
221
+ finding=f"{_of(len(hidden), len(dated_ok), 'dated pages that loaded')} dated "
222
+ "only by a sitemap, an HTTP header or the search tool's page age; citewatch found "
223
+ "no date on the page itself.",
224
+ fix="Show the last-updated date on the page and mark it up (schema.org "
225
+ "dateModified or a <time> element), so automated readers find the date a "
226
+ "person sees.",
227
+ count=len(hidden),
228
+ of=len(dated_ok),
229
+ sources=("page_meta.jsonl",),
230
+ examples=tuple(hidden[:EXAMPLES]),
231
+ )
232
+ )
233
+ # stale (§6.3 reading): pages that loaded, cited on time-sensitive questions, dated more
234
+ # than stale_days before the latest such answer
235
+ ts = [u for u in dated_ok if u in st.newest_ts]
236
+ stale = [
237
+ u
238
+ for u in ts
239
+ if (st.newest_ts[u] - dt.date.fromisoformat(meta[u]["effective_date"])).days > st.stale_days
240
+ ]
241
+ if stale:
242
+ rows.append(
243
+ Improvement(
244
+ code="stale",
245
+ finding=f"{_of(len(stale), len(ts), TS_PAGES)} dated more than "
246
+ f"{st.stale_days} days before the answers that cited them.",
247
+ fix="Review these pages: if the guidance has changed, update and re-date them; "
248
+ "if it has not, a visible reviewed date says so.",
249
+ count=len(stale),
250
+ of=len(ts),
251
+ sources=("page_meta.jsonl", "answers.jsonl", "questions.jsonl"),
252
+ examples=tuple(stale[:EXAMPLES]),
253
+ )
254
+ )
255
+ disallowed = [u for u in urls if outcome[u] == "robots_disallowed"]
256
+
257
+ def robots_host(u: str) -> str: # the host that refused: the final hop's, where known
258
+ final = urlsplit(meta.get(u, {}).get("final_url") or u).hostname or ""
259
+ return final if final in st.robots else urlsplit(u).hostname or ""
260
+
261
+ unreadable = [u for u in disallowed if robots_host(u) in st.robots]
262
+ real = [u for u in disallowed if u not in set(unreadable)]
263
+ if real:
264
+ rows.append(
265
+ Improvement(
266
+ code="robots_blocked",
267
+ finding=f"{_of(len(real), len(urls), 'cited pages')} disallowed by the site's "
268
+ "robots.txt for citewatch's reader, so the dates could not be checked.",
269
+ fix="Decide whether independent checkers may read these pages; AI search tools "
270
+ "use their own crawlers, so this does not by itself stop citation.",
271
+ count=len(real),
272
+ of=len(urls),
273
+ sources=("page_meta.jsonl", "fetch_log.jsonl"),
274
+ examples=tuple(real[:EXAMPLES]),
275
+ )
276
+ )
277
+ if unreadable:
278
+ statuses = {u: st.robots[robots_host(u)] for u in unreadable}
279
+ kinds = sorted({_robots_failure(v) for v in statuses.values()})
280
+ hosts = {robots_host(u) for u in unreadable}
281
+ elsewhere = sorted(h for h in hosts if all(h != urlsplit(u).hostname for u in unreadable))
282
+ one = len(unreadable) == 1
283
+ where = (
284
+ f"; the failure was at {', '.join(elsewhere)}, where "
285
+ f"{'the page redirects' if one else 'the pages redirect'}"
286
+ if elsewhere
287
+ else ""
288
+ )
289
+ stricter = any(v is not None and (v < 500) for v in statuses.values())
290
+ fixes = []
291
+ if elsewhere:
292
+ fixes.append("The failure was on the site these links redirect to, not on this one.")
293
+ if any(v in (401, 403, 429) for v in statuses.values()):
294
+ fixes.append(
295
+ "Bot protection that refuses robots.txt keeps the site's rules from careful "
296
+ "readers, which then stay away; serving robots.txt to them makes the rules "
297
+ "visible."
298
+ )
299
+ if any(v is None or v >= 500 for v in statuses.values()):
300
+ fixes.append(
301
+ "An unavailable robots.txt leaves well-behaved readers treating the site as off "
302
+ "limits (RFC 9309); keeping it available keeps the site's rules visible."
303
+ )
304
+ if any(v is not None and 300 <= v < 400 for v in statuses.values()):
305
+ fixes.append(
306
+ "citewatch follows a robots.txt redirect only to another /robots.txt; serving "
307
+ "robots.txt at its own address lets strict readers see the rules."
308
+ )
309
+ if not fixes:
310
+ fixes.append("Serving robots.txt with HTTP 200 keeps the site's rules visible.")
311
+ rule = (
312
+ "citewatch reads a page only after reading its site's robots.txt and treats one it "
313
+ "cannot read as off limits"
314
+ + (" (stricter than RFC 9309 for refusals and redirects)" if stricter else "")
315
+ )
316
+ rows.append(
317
+ Improvement(
318
+ code="robots_unreadable",
319
+ finding=f"{_of(len(unreadable), len(urls), 'cited pages')} on "
320
+ f"{'a host' if len(hosts) == 1 else 'hosts'} whose robots.txt "
321
+ f"{' or '.join(kinds)}{where}. {rule}, so it did not read "
322
+ f"{'it' if one else 'them'}.",
323
+ fix=" ".join(fixes),
324
+ count=len(unreadable),
325
+ of=len(urls),
326
+ sources=("page_meta.jsonl", "fetch_log.jsonl"),
327
+ examples=tuple(unreadable[:EXAMPLES]),
328
+ )
329
+ )
330
+ refused = [u for u in urls if outcome[u] in ("blocked", "gated")]
331
+ if refused:
332
+ rows.append(
333
+ Improvement(
334
+ code="blocked",
335
+ finding=f"{_of(len(refused), len(urls), 'cited pages')} refused to automated "
336
+ "readers or behind a sign-in when checked.",
337
+ fix="If these pages are public guidance, let well-behaved readers load them so "
338
+ "their dates can be checked.",
339
+ count=len(refused),
340
+ of=len(urls),
341
+ sources=("page_meta.jsonl",),
342
+ examples=tuple(refused[:EXAMPLES]),
343
+ )
344
+ )
345
+ files = [u for u in urls if outcome[u] in ("not_html", "too_large")]
346
+ if files:
347
+ rows.append(
348
+ Improvement(
349
+ code="not_html",
350
+ finding=f"{_of(len(files), len(urls), 'cited links', DOCS)} "
351
+ f"({'for example a PDF' if len(files) == 1 else 'for example PDFs'}); citewatch "
352
+ "reads dates from web pages only.",
353
+ fix="Publish key guidance as an HTML page with a visible date, and link the "
354
+ "document from it.",
355
+ count=len(files),
356
+ of=len(urls),
357
+ sources=("page_meta.jsonl",),
358
+ examples=tuple(files[:EXAMPLES]),
359
+ )
360
+ )
361
+ broken = [u for u in urls if outcome[u] == "fetch_error"]
362
+ if broken:
363
+ rows.append(
364
+ Improvement(
365
+ code="fetch_error",
366
+ finding=f"{_of(len(broken), len(urls), 'cited pages')} failing to load when "
367
+ "checked (for example a 404 or a timeout).",
368
+ fix="Fix or redirect these URLs; AI answers still link to them.",
369
+ count=len(broken),
370
+ of=len(urls),
371
+ sources=("page_meta.jsonl",),
372
+ examples=tuple(broken[:EXAMPLES]),
373
+ )
374
+ )
375
+ # unsupported: both AI judges say the claim is not on the page, and the owner's check did
376
+ # not find it supported either (D-69: never contradict the owner's labels)
377
+ mine = {ref: v for ref, v in st.verdicts.items() if st.cite_pub.get(ref) == publisher}
378
+ both_no = sorted(
379
+ r
380
+ for r, v in mine.items()
381
+ if len(v) >= 2 and "supported" not in v.values() and st.hand.get(r) != "supported"
382
+ )
383
+ if both_no:
384
+ rows.append(
385
+ Improvement(
386
+ code="unsupported",
387
+ finding=f"In the support sample, both AI judges found the cited claim not stated "
388
+ f"on the page for {len(both_no)} of {len(mine)} sampled citations, and the "
389
+ "owner's check did not find it supported.",
390
+ fix="State the key facts these answers rely on explicitly in the page text "
391
+ "(numbers, eligibility, steps), not only in images, tables or linked files.",
392
+ count=len(both_no),
393
+ of=len(mine),
394
+ sources=("support_verdicts.jsonl", "support_labels.jsonl"),
395
+ examples=tuple(both_no[:EXAMPLES]),
396
+ )
397
+ )
398
+ return rows
399
+
400
+
401
+ def build_cards(
402
+ base: Path, data: StudyData, summary: StudySummary, stale_days: int
403
+ ) -> list[DomainCard]:
404
+ from citewatch.support.report import load_hand_labels
405
+
406
+ answers = [a for a in data.answers if a.valid]
407
+ newest_ts: dict[str, dt.date] = {}
408
+ for a in answers:
409
+ if a.time_sensitive:
410
+ for c in a.cites:
411
+ d = a.answered_at.date()
412
+ if c.url not in newest_ts or d > newest_ts[c.url]:
413
+ newest_ts[c.url] = d
414
+ st = _Study(
415
+ meta={r["url"]: r for r in _rows(base / "data" / "page_meta.jsonl")},
416
+ robots=_unreadable_robots(base),
417
+ verdicts=_verdicts(base),
418
+ hand=load_hand_labels(base),
419
+ cite_pub=_citation_publishers(base, set(summary.runs)),
420
+ newest_ts=newest_ts,
421
+ intent_leaders=_intent_leaders(summary),
422
+ stale_days=stale_days,
423
+ answers=answers,
424
+ )
425
+ pooled = {r.publisher: r for r in summary.freshness if r.model is None}
426
+ presence = [r for r in summary.share if r.kind == "presence"]
427
+ citations: Counter[str] = Counter()
428
+ citing: dict[str, set[tuple[int, str, str]]] = defaultdict(set)
429
+ questions: dict[str, set[str]] = defaultdict(set)
430
+ for a in answers:
431
+ for c in a.cites:
432
+ citations[c.publisher] += 1
433
+ citing[c.publisher].add((a.run, a.qid, a.model))
434
+ questions[c.publisher].add(a.qid)
435
+ cards = []
436
+ for p in sorted(citations, key=lambda p: (-citations[p], p)):
437
+ fresh = pooled.get(p)
438
+ cards.append(
439
+ DomainCard(
440
+ publisher=p,
441
+ share_by_model={
442
+ _key(r.model, f"run-{r.run}"): r.share
443
+ for r in presence
444
+ if r.publisher == p and r.intent is None
445
+ },
446
+ share_by_intent={
447
+ _key(r.intent, r.model, f"run-{r.run}"): r.share
448
+ for r in presence
449
+ if r.publisher == p and r.intent is not None
450
+ },
451
+ citations=citations[p],
452
+ answers_citing=len(citing[p]),
453
+ median_freshness_lag_days=fresh.median_lag_days if fresh else None,
454
+ dated_pages=fresh.dated_pages if fresh else 0,
455
+ questions_citing=tuple(sorted(questions[p])),
456
+ cited_instead=cited_instead(data, p),
457
+ improvement=tuple(_improvements(p, summary, st)),
458
+ )
459
+ )
460
+ return cards
461
+
462
+
463
+ def write_cards(base: Path, cards: Iterable[DomainCard]) -> Path:
464
+ path = base / "results" / CARDS_FILE
465
+ path.parent.mkdir(parents=True, exist_ok=True)
466
+ payload = [c.model_dump(mode="json") for c in cards]
467
+ tmp = path.with_suffix(".json.tmp")
468
+ tmp.write_text(json.dumps(payload, indent=1, sort_keys=True) + "\n", encoding="utf-8")
469
+ tmp.replace(path)
470
+ return path
471
+
472
+
473
+ def read_cards(path: Path) -> list[DomainCard]:
474
+ return [DomainCard.model_validate(c) for c in json.loads(path.read_text(encoding="utf-8"))]
475
+
476
+
477
+ @dataclass(frozen=True)
478
+ class NotCited:
479
+ query: str
480
+ publisher: str | None
481
+ answers: int
482
+ message: str
483
+ top_for_intent: tuple[str, ...]
484
+
485
+
486
+ def normalise(query: str) -> str:
487
+ """Strip scheme, user info, `www.`, path, query and port; lower-case; IDN shown as Unicode."""
488
+ q = query.strip()
489
+ try:
490
+ host = urlsplit(q if "//" in q else f"//{q}").hostname or ""
491
+ except ValueError:
492
+ return ""
493
+ host = host.removeprefix("www.").rstrip(".").lower()
494
+ try:
495
+ return host.encode("ascii").decode("idna") if "xn--" in host else host
496
+ except UnicodeError:
497
+ return host
498
+
499
+
500
+ def _registrable(domain: str) -> bool:
501
+ from citewatch.extract.urls import _extractor
502
+
503
+ return bool(_extractor()(domain).domain)
504
+
505
+
506
+ def _intents(cards: Sequence[DomainCard]) -> set[str]:
507
+ return {k.split("/")[0] for c in cards for k in c.share_by_intent}
508
+
509
+
510
+ def lookup(
511
+ query: str,
512
+ cards: Sequence[DomainCard],
513
+ answers: int,
514
+ intent: str | None = None,
515
+ ) -> DomainCard | NotCited:
516
+ """The card for the publisher `query` maps to, through the extractor's own `publisher_of`;
517
+ else a not-cited result naming the study's `answers` and the top publishers for `intent`.
518
+ An intent the study does not have raises ValueError."""
519
+ if intent is not None and intent not in _intents(cards):
520
+ raise ValueError(f"unknown intent {intent!r}; this study has {sorted(_intents(cards))}")
521
+ host = normalise(query)
522
+ publisher = None
523
+ if host and "." in host and " " not in host and "\\" not in query:
524
+ try:
525
+ publisher = publisher_of(host)[0]
526
+ except UrlError:
527
+ publisher = None
528
+ if publisher and not _registrable(publisher) and publisher not in INSTITUTIONAL:
529
+ publisher = None # a public suffix such as co.uk is not a site (D-4 keeps nhs.uk)
530
+ by = {c.publisher: c for c in cards}
531
+ if publisher in by:
532
+ return by[publisher]
533
+ if intent:
534
+
535
+ def k_for(c: DomainCard) -> int:
536
+ return sum(
537
+ (s.k or 0) for key, s in c.share_by_intent.items() if key.startswith(f"{intent}/")
538
+ )
539
+
540
+ top = tuple(
541
+ c.publisher for c in sorted(cards, key=lambda c: (-k_for(c), c.publisher)) if k_for(c)
542
+ )
543
+ else:
544
+ top = tuple(c.publisher for c in cards)
545
+ if publisher is None:
546
+ message = f"{query.strip()!r} is not a web address citewatch can look up."
547
+ else:
548
+ message = (
549
+ f"{normalise(publisher)} was not cited in any of the {answers} answers in this study."
550
+ )
551
+ return NotCited(
552
+ query=query,
553
+ publisher=publisher,
554
+ answers=answers,
555
+ message=message,
556
+ top_for_intent=top[:TOP_FOR_INTENT],
557
+ )