citewatch 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- citewatch/__init__.py +57 -0
- citewatch/cards.py +557 -0
- citewatch/cli.py +863 -0
- citewatch/cost.py +262 -0
- citewatch/dates/__init__.py +34 -0
- citewatch/dates/candidates.py +408 -0
- citewatch/dates/pipeline.py +119 -0
- citewatch/dates/resolve.py +101 -0
- citewatch/dates/rules.py +80 -0
- citewatch/dates/sitemaps.py +135 -0
- citewatch/demo/__init__.py +7 -0
- citewatch/demo/component.py +138 -0
- citewatch/demo/guard.py +153 -0
- citewatch/demo/kv.py +212 -0
- citewatch/demo/live.py +273 -0
- citewatch/demo/logs.py +42 -0
- citewatch/demo/lookup.py +157 -0
- citewatch/demo/make_code_hash.py +49 -0
- citewatch/demo/markup.py +38 -0
- citewatch/demo/passkeys.py +386 -0
- citewatch/demo/settings.py +66 -0
- citewatch/env.py +48 -0
- citewatch/examples/uk-health-summary.json.gz +0 -0
- citewatch/extract/__init__.py +18 -0
- citewatch/extract/extractors.py +147 -0
- citewatch/extract/stage.py +189 -0
- citewatch/extract/urls.py +108 -0
- citewatch/fetch/__init__.py +28 -0
- citewatch/fetch/client.py +434 -0
- citewatch/fetch/history.py +132 -0
- citewatch/fetch/keys.py +125 -0
- citewatch/fetch/log.py +56 -0
- citewatch/fetch/pages.py +266 -0
- citewatch/fetch/rate.py +99 -0
- citewatch/fetch/robots.py +140 -0
- citewatch/handcheck.py +221 -0
- citewatch/ledger.py +75 -0
- citewatch/metrics/__init__.py +5 -0
- citewatch/metrics/compute.py +602 -0
- citewatch/metrics/data.py +166 -0
- citewatch/metrics/stats.py +120 -0
- citewatch/metrics/summary.py +231 -0
- citewatch/metrics/views.py +44 -0
- citewatch/prices.yaml +19 -0
- citewatch/py.typed +0 -0
- citewatch/questions.py +84 -0
- citewatch/report/__init__.py +5 -0
- citewatch/report/dashboard.py +762 -0
- citewatch/report/post.py +235 -0
- citewatch/runners/__init__.py +29 -0
- citewatch/runners/anthropic_runner.py +168 -0
- citewatch/runners/base.py +144 -0
- citewatch/runners/openai_runner.py +117 -0
- citewatch/runners/schedule.py +423 -0
- citewatch/runs.py +202 -0
- citewatch/schemas/__init__.py +61 -0
- citewatch/schemas/_base.py +160 -0
- citewatch/schemas/_iso.py +257 -0
- citewatch/schemas/cards.py +120 -0
- citewatch/schemas/config.py +232 -0
- citewatch/schemas/pages.py +92 -0
- citewatch/schemas/study.py +133 -0
- citewatch/schemas/summary.py +187 -0
- citewatch/schemas/thresholds.py +165 -0
- citewatch/store.py +127 -0
- citewatch/support/__init__.py +5 -0
- citewatch/support/calibrate.py +83 -0
- citewatch/support/judge.py +455 -0
- citewatch/support/report.py +191 -0
- citewatch/support/sample.py +84 -0
- citewatch/templates/RUBRIC.md +9 -0
- citewatch/templates/env.example +3 -0
- citewatch/templates/questions.jsonl +2 -0
- citewatch/templates/study.yaml +23 -0
- citewatch/thresholds.py +129 -0
- citewatch-0.1.0.dist-info/METADATA +210 -0
- citewatch-0.1.0.dist-info/RECORD +80 -0
- citewatch-0.1.0.dist-info/WHEEL +4 -0
- citewatch-0.1.0.dist-info/entry_points.txt +2 -0
- citewatch-0.1.0.dist-info/licenses/LICENSE +21 -0
citewatch/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""citewatch: measure which web pages AI answers with web search cite, and how fresh they are.
|
|
2
|
+
|
|
3
|
+
Public API (semver, see docs/versioning.md): the results format in `citewatch.schemas`, and
|
|
4
|
+
`canonical_url`, `publisher_of`, `UrlError`, `lookup`, `read_cards` and `NotCited`, importable
|
|
5
|
+
from here or from `citewatch.extract.urls` and `citewatch.cards`. They load on first use, so
|
|
6
|
+
`import citewatch` (and the CLI) stays fast.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib import import_module
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
__version__ = version("citewatch")
|
|
17
|
+
except PackageNotFoundError: # running from a source tree that was never installed
|
|
18
|
+
__version__ = "0.0.0"
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from citewatch import schemas
|
|
22
|
+
from citewatch.cards import NotCited, lookup, read_cards
|
|
23
|
+
from citewatch.extract.urls import UrlError, canonical_url, publisher_of
|
|
24
|
+
|
|
25
|
+
_LAZY = {
|
|
26
|
+
"canonical_url": "citewatch.extract.urls",
|
|
27
|
+
"publisher_of": "citewatch.extract.urls",
|
|
28
|
+
"UrlError": "citewatch.extract.urls",
|
|
29
|
+
"lookup": "citewatch.cards",
|
|
30
|
+
"read_cards": "citewatch.cards",
|
|
31
|
+
"NotCited": "citewatch.cards",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"NotCited",
|
|
36
|
+
"UrlError",
|
|
37
|
+
"__version__",
|
|
38
|
+
"canonical_url",
|
|
39
|
+
"lookup",
|
|
40
|
+
"publisher_of",
|
|
41
|
+
"read_cards",
|
|
42
|
+
"schemas",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def __getattr__(name: str) -> Any:
|
|
47
|
+
if name == "schemas":
|
|
48
|
+
return import_module("citewatch.schemas")
|
|
49
|
+
if name in _LAZY:
|
|
50
|
+
value = getattr(import_module(_LAZY[name]), name)
|
|
51
|
+
globals()[name] = value
|
|
52
|
+
return value
|
|
53
|
+
raise AttributeError(f"module 'citewatch' has no attribute {name!r}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def __dir__() -> list[str]:
|
|
57
|
+
return sorted({*globals(), *__all__})
|
citewatch/cards.py
ADDED
|
@@ -0,0 +1,557 @@
|
|
|
1
|
+
"""Domain cards, lookup and the improvement report (node D5.1, Tech Spec §9, OWNER D-65).
|
|
2
|
+
|
|
3
|
+
One `DomainCard` per publisher cited in the scored runs, built from `summary.json` and the
|
|
4
|
+
frozen study data, so the demo answers "is my site cited, how often, how fresh, and what would
|
|
5
|
+
help?" without calling a model. The improvement rows describe sourcing, dates and readability
|
|
6
|
+
only; they never judge medical accuracy (NFR4), and they never state more than the data shows
|
|
7
|
+
about a publisher's site (D-69).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import datetime as dt
|
|
13
|
+
import json
|
|
14
|
+
from collections import Counter, defaultdict
|
|
15
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
from urllib.parse import urlsplit
|
|
20
|
+
|
|
21
|
+
from citewatch.extract.urls import INSTITUTIONAL, UrlError, publisher_of
|
|
22
|
+
from citewatch.metrics.data import StudyData
|
|
23
|
+
from citewatch.schemas import DomainCard, Improvement, StudySummary
|
|
24
|
+
|
|
25
|
+
CARDS_FILE = "domain_cards.json"
|
|
26
|
+
CITED_INSTEAD_TOP = 5
|
|
27
|
+
EXAMPLES = 3
|
|
28
|
+
# date sources a reader never sees on the page: the search tool's page age, sitemaps, headers
|
|
29
|
+
HIDDEN_SOURCES = frozenset({"sitemap", "http_header", "page_age"})
|
|
30
|
+
TOP_FOR_INTENT = 5
|
|
31
|
+
DOCS = ("is a document rather than a web page", "are documents rather than web pages")
|
|
32
|
+
TS_PAGES = "dated pages that loaded and were cited on time-sensitive questions"
|
|
33
|
+
# a missing intent is reported only for publishers cited broadly enough for the gap to mean
|
|
34
|
+
# something: on this many distinct questions, across at least two intents (D-69)
|
|
35
|
+
INTENT_MIN_QUESTIONS = 5
|
|
36
|
+
INTENT_MIN_INTENTS = 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class _Study:
|
|
41
|
+
"""Everything the improvement rows read, loaded once."""
|
|
42
|
+
|
|
43
|
+
meta: Mapping[str, Mapping[str, Any]]
|
|
44
|
+
robots: Mapping[str, int | None] # host -> status of its robots.txt, if never readable
|
|
45
|
+
verdicts: Mapping[str, Mapping[str, str]]
|
|
46
|
+
hand: Mapping[str, str]
|
|
47
|
+
cite_pub: Mapping[str, str]
|
|
48
|
+
newest_ts: Mapping[str, dt.date] # url -> latest time-sensitive answer citing it
|
|
49
|
+
intent_leaders: Mapping[str, tuple[str, ...]]
|
|
50
|
+
stale_days: int
|
|
51
|
+
answers: Sequence[Any] = field(default_factory=tuple)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _key(*parts: object) -> str:
|
|
55
|
+
return "/".join(str(p) for p in parts)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _rows(path: Path) -> Iterable[dict[str, Any]]:
|
|
59
|
+
if path.is_file():
|
|
60
|
+
for x in path.read_text(encoding="utf-8").splitlines():
|
|
61
|
+
if x.strip():
|
|
62
|
+
yield json.loads(x)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _unreadable_robots(base: Path) -> dict[str, int | None]:
|
|
66
|
+
"""Hosts whose robots.txt never returned 200, with the last status seen (H1, D-69): the
|
|
67
|
+
fetcher treats an unreadable robots.txt as disallowing the host, which is not the same as
|
|
68
|
+
the site's robots.txt disallowing citewatch."""
|
|
69
|
+
seen: dict[str, list[int | None]] = defaultdict(list)
|
|
70
|
+
for r in _rows(base / "data" / "fetch_log.jsonl"):
|
|
71
|
+
if urlsplit(r["url"]).path == "/robots.txt":
|
|
72
|
+
seen[r["host"]].append(r.get("status"))
|
|
73
|
+
|
|
74
|
+
def readable(h: str) -> bool:
|
|
75
|
+
if 200 in seen[h]:
|
|
76
|
+
return True
|
|
77
|
+
# robots rules are per origin: x's robots.txt counts as read only when it redirected
|
|
78
|
+
# (3xx) to its www/non-www twin and that twin answered 200 (pass 2, L-d)
|
|
79
|
+
twin = h.removeprefix("www.") if h.startswith("www.") else f"www.{h}"
|
|
80
|
+
redirected = any(s is not None and 300 <= s < 400 for s in seen[h])
|
|
81
|
+
return redirected and 200 in seen.get(twin, [])
|
|
82
|
+
|
|
83
|
+
return {h: s[-1] for h, s in list(seen.items()) if not readable(h)}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _verdicts(base: Path) -> dict[str, dict[str, str]]:
|
|
87
|
+
"""citation_ref -> {judge: verdict} from the support check, if it ran."""
|
|
88
|
+
out: dict[str, dict[str, str]] = defaultdict(dict)
|
|
89
|
+
for v in _rows(base / "data" / "support_verdicts.jsonl"):
|
|
90
|
+
out[v["citation_ref"]][v["judge"]] = v["verdict"]
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _citation_publishers(base: Path, runs: set[int]) -> dict[str, str]:
|
|
95
|
+
out = {}
|
|
96
|
+
for c in _rows(base / "data" / "citations.jsonl"):
|
|
97
|
+
if c["run"] in runs:
|
|
98
|
+
out[f"{c['run']}:{c['question_id']}:{c['model']}:{c['ordinal']}"] = c["publisher"]
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _of(k: int, n: int, noun: str, verbs: tuple[str, str] = ("is", "are")) -> str:
|
|
103
|
+
"""'1 of 3 cited pages is' / '2 of 3 cited pages are'."""
|
|
104
|
+
return f"{k} of {n} {noun} {verbs[0] if k == 1 else verbs[1]}"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _robots_failure(status: int | None) -> str:
|
|
108
|
+
"""What happened to a robots.txt request, in words."""
|
|
109
|
+
if status is None:
|
|
110
|
+
return "did not respond"
|
|
111
|
+
if 300 <= status < 400:
|
|
112
|
+
return f"redirected away from /robots.txt (HTTP {status})"
|
|
113
|
+
if status in (401, 403, 429):
|
|
114
|
+
return f"refused the request (HTTP {status})"
|
|
115
|
+
if status >= 500:
|
|
116
|
+
return f"returned a server error (HTTP {status})"
|
|
117
|
+
return f"could not be read (HTTP {status})"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def cited_instead(data: StudyData, publisher: str) -> tuple[str, ...]:
|
|
121
|
+
"""§9: on questions where `publisher` was cited by one model or run but not another, the
|
|
122
|
+
publishers cited in the answers that left it out, by count then name."""
|
|
123
|
+
by_q: dict[str, list[Any]] = defaultdict(list)
|
|
124
|
+
for a in data.answers:
|
|
125
|
+
if a.valid:
|
|
126
|
+
by_q[a.qid].append(a)
|
|
127
|
+
counts: Counter[str] = Counter()
|
|
128
|
+
for answers in by_q.values():
|
|
129
|
+
has = [any(c.publisher == publisher for c in a.cites) for a in answers]
|
|
130
|
+
if any(has) and not all(has):
|
|
131
|
+
for a, h in zip(answers, has, strict=True):
|
|
132
|
+
if not h:
|
|
133
|
+
counts.update({c.publisher for c in a.cites} - {publisher})
|
|
134
|
+
ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
135
|
+
return tuple(p for p, _ in ranked[:CITED_INSTEAD_TOP])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _intent_leaders(summary: StudySummary) -> dict[str, tuple[str, ...]]:
|
|
139
|
+
"""intent -> publishers cited in most answers to that intent (presence k over model/run)."""
|
|
140
|
+
k: dict[str, Counter[str]] = defaultdict(Counter)
|
|
141
|
+
for r in summary.share:
|
|
142
|
+
if r.kind == "presence" and r.intent:
|
|
143
|
+
k[r.intent][r.publisher] += r.share.k or 0
|
|
144
|
+
return {
|
|
145
|
+
i: tuple(p for p, n in sorted(c.items(), key=lambda kv: (-kv[1], kv[0])) if n)[
|
|
146
|
+
:TOP_FOR_INTENT
|
|
147
|
+
]
|
|
148
|
+
for i, c in k.items()
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _improvements(publisher: str, summary: StudySummary, st: _Study) -> list[Improvement]:
|
|
153
|
+
rows: list[Improvement] = []
|
|
154
|
+
answers = st.answers
|
|
155
|
+
urls = sorted({c.url for a in answers for c in a.cites if c.publisher == publisher})
|
|
156
|
+
meta = st.meta
|
|
157
|
+
outcome = {u: meta.get(u, {}).get("outcome", "missing") for u in urls}
|
|
158
|
+
|
|
159
|
+
# intent_absent: broad presence elsewhere, none on an intent (summary presence rows)
|
|
160
|
+
cited_intents = {
|
|
161
|
+
r.intent
|
|
162
|
+
for r in summary.share
|
|
163
|
+
if r.kind == "presence" and r.publisher == publisher and r.intent and (r.share.k or 0) > 0
|
|
164
|
+
}
|
|
165
|
+
intents = sorted({a.intent for a in answers})
|
|
166
|
+
questions = {a.qid for a in answers if any(c.publisher == publisher for c in a.cites)}
|
|
167
|
+
absent = [i for i in intents if i not in cited_intents]
|
|
168
|
+
if (
|
|
169
|
+
absent
|
|
170
|
+
and len(questions) >= INTENT_MIN_QUESTIONS
|
|
171
|
+
and len(cited_intents) >= INTENT_MIN_INTENTS
|
|
172
|
+
):
|
|
173
|
+
names = ", ".join(i.replace("_", " ") for i in absent)
|
|
174
|
+
leaders = sorted(
|
|
175
|
+
{p for i in absent for p in st.intent_leaders.get(i, ())[:3]} - {publisher}
|
|
176
|
+
)
|
|
177
|
+
lead = f" Cited most on those questions: {', '.join(leaders)}." if leaders else ""
|
|
178
|
+
rows.append(
|
|
179
|
+
Improvement(
|
|
180
|
+
code="intent_absent",
|
|
181
|
+
finding=f"Cited on {len(questions)} questions across {len(cited_intents)} topics, "
|
|
182
|
+
f"but on no {names} question. This may simply be outside the site's remit.",
|
|
183
|
+
fix="If the site covers these topics, check that a page answers such questions "
|
|
184
|
+
f"directly, in plain text near the top, with a visible date.{lead}",
|
|
185
|
+
count=len(absent),
|
|
186
|
+
of=len(intents),
|
|
187
|
+
sources=("summary.json",),
|
|
188
|
+
examples=tuple(absent[:EXAMPLES]),
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
ok = [u for u in urls if outcome[u] == "ok"]
|
|
193
|
+
undated = [u for u in ok if not meta[u].get("effective_date")]
|
|
194
|
+
if undated:
|
|
195
|
+
rows.append(
|
|
196
|
+
Improvement(
|
|
197
|
+
code="undated",
|
|
198
|
+
finding=f"{_of(len(undated), len(ok), 'cited pages that loaded', ('has', 'have'))} "
|
|
199
|
+
"no date citewatch could read on the page.",
|
|
200
|
+
fix='Add a visible "last updated" date and schema.org dateModified to each page.',
|
|
201
|
+
count=len(undated),
|
|
202
|
+
of=len(ok),
|
|
203
|
+
sources=("page_meta.jsonl",),
|
|
204
|
+
examples=tuple(undated[:EXAMPLES]),
|
|
205
|
+
)
|
|
206
|
+
)
|
|
207
|
+
dated_ok = [u for u in ok if meta[u].get("effective_date")]
|
|
208
|
+
hidden = [
|
|
209
|
+
u
|
|
210
|
+
for u in dated_ok
|
|
211
|
+
if meta[u].get("effective_source") in HIDDEN_SOURCES
|
|
212
|
+
and not any( # any date shown on the page itself, even a published one (pass 2, M-a)
|
|
213
|
+
c.get("source") not in HIDDEN_SOURCES and not c.get("rejected")
|
|
214
|
+
for c in meta[u].get("candidates", ())
|
|
215
|
+
)
|
|
216
|
+
]
|
|
217
|
+
if hidden:
|
|
218
|
+
rows.append(
|
|
219
|
+
Improvement(
|
|
220
|
+
code="hidden_date",
|
|
221
|
+
finding=f"{_of(len(hidden), len(dated_ok), 'dated pages that loaded')} dated "
|
|
222
|
+
"only by a sitemap, an HTTP header or the search tool's page age; citewatch found "
|
|
223
|
+
"no date on the page itself.",
|
|
224
|
+
fix="Show the last-updated date on the page and mark it up (schema.org "
|
|
225
|
+
"dateModified or a <time> element), so automated readers find the date a "
|
|
226
|
+
"person sees.",
|
|
227
|
+
count=len(hidden),
|
|
228
|
+
of=len(dated_ok),
|
|
229
|
+
sources=("page_meta.jsonl",),
|
|
230
|
+
examples=tuple(hidden[:EXAMPLES]),
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
# stale (§6.3 reading): pages that loaded, cited on time-sensitive questions, dated more
|
|
234
|
+
# than stale_days before the latest such answer
|
|
235
|
+
ts = [u for u in dated_ok if u in st.newest_ts]
|
|
236
|
+
stale = [
|
|
237
|
+
u
|
|
238
|
+
for u in ts
|
|
239
|
+
if (st.newest_ts[u] - dt.date.fromisoformat(meta[u]["effective_date"])).days > st.stale_days
|
|
240
|
+
]
|
|
241
|
+
if stale:
|
|
242
|
+
rows.append(
|
|
243
|
+
Improvement(
|
|
244
|
+
code="stale",
|
|
245
|
+
finding=f"{_of(len(stale), len(ts), TS_PAGES)} dated more than "
|
|
246
|
+
f"{st.stale_days} days before the answers that cited them.",
|
|
247
|
+
fix="Review these pages: if the guidance has changed, update and re-date them; "
|
|
248
|
+
"if it has not, a visible reviewed date says so.",
|
|
249
|
+
count=len(stale),
|
|
250
|
+
of=len(ts),
|
|
251
|
+
sources=("page_meta.jsonl", "answers.jsonl", "questions.jsonl"),
|
|
252
|
+
examples=tuple(stale[:EXAMPLES]),
|
|
253
|
+
)
|
|
254
|
+
)
|
|
255
|
+
disallowed = [u for u in urls if outcome[u] == "robots_disallowed"]
|
|
256
|
+
|
|
257
|
+
def robots_host(u: str) -> str: # the host that refused: the final hop's, where known
|
|
258
|
+
final = urlsplit(meta.get(u, {}).get("final_url") or u).hostname or ""
|
|
259
|
+
return final if final in st.robots else urlsplit(u).hostname or ""
|
|
260
|
+
|
|
261
|
+
unreadable = [u for u in disallowed if robots_host(u) in st.robots]
|
|
262
|
+
real = [u for u in disallowed if u not in set(unreadable)]
|
|
263
|
+
if real:
|
|
264
|
+
rows.append(
|
|
265
|
+
Improvement(
|
|
266
|
+
code="robots_blocked",
|
|
267
|
+
finding=f"{_of(len(real), len(urls), 'cited pages')} disallowed by the site's "
|
|
268
|
+
"robots.txt for citewatch's reader, so the dates could not be checked.",
|
|
269
|
+
fix="Decide whether independent checkers may read these pages; AI search tools "
|
|
270
|
+
"use their own crawlers, so this does not by itself stop citation.",
|
|
271
|
+
count=len(real),
|
|
272
|
+
of=len(urls),
|
|
273
|
+
sources=("page_meta.jsonl", "fetch_log.jsonl"),
|
|
274
|
+
examples=tuple(real[:EXAMPLES]),
|
|
275
|
+
)
|
|
276
|
+
)
|
|
277
|
+
if unreadable:
|
|
278
|
+
statuses = {u: st.robots[robots_host(u)] for u in unreadable}
|
|
279
|
+
kinds = sorted({_robots_failure(v) for v in statuses.values()})
|
|
280
|
+
hosts = {robots_host(u) for u in unreadable}
|
|
281
|
+
elsewhere = sorted(h for h in hosts if all(h != urlsplit(u).hostname for u in unreadable))
|
|
282
|
+
one = len(unreadable) == 1
|
|
283
|
+
where = (
|
|
284
|
+
f"; the failure was at {', '.join(elsewhere)}, where "
|
|
285
|
+
f"{'the page redirects' if one else 'the pages redirect'}"
|
|
286
|
+
if elsewhere
|
|
287
|
+
else ""
|
|
288
|
+
)
|
|
289
|
+
stricter = any(v is not None and (v < 500) for v in statuses.values())
|
|
290
|
+
fixes = []
|
|
291
|
+
if elsewhere:
|
|
292
|
+
fixes.append("The failure was on the site these links redirect to, not on this one.")
|
|
293
|
+
if any(v in (401, 403, 429) for v in statuses.values()):
|
|
294
|
+
fixes.append(
|
|
295
|
+
"Bot protection that refuses robots.txt keeps the site's rules from careful "
|
|
296
|
+
"readers, which then stay away; serving robots.txt to them makes the rules "
|
|
297
|
+
"visible."
|
|
298
|
+
)
|
|
299
|
+
if any(v is None or v >= 500 for v in statuses.values()):
|
|
300
|
+
fixes.append(
|
|
301
|
+
"An unavailable robots.txt leaves well-behaved readers treating the site as off "
|
|
302
|
+
"limits (RFC 9309); keeping it available keeps the site's rules visible."
|
|
303
|
+
)
|
|
304
|
+
if any(v is not None and 300 <= v < 400 for v in statuses.values()):
|
|
305
|
+
fixes.append(
|
|
306
|
+
"citewatch follows a robots.txt redirect only to another /robots.txt; serving "
|
|
307
|
+
"robots.txt at its own address lets strict readers see the rules."
|
|
308
|
+
)
|
|
309
|
+
if not fixes:
|
|
310
|
+
fixes.append("Serving robots.txt with HTTP 200 keeps the site's rules visible.")
|
|
311
|
+
rule = (
|
|
312
|
+
"citewatch reads a page only after reading its site's robots.txt and treats one it "
|
|
313
|
+
"cannot read as off limits"
|
|
314
|
+
+ (" (stricter than RFC 9309 for refusals and redirects)" if stricter else "")
|
|
315
|
+
)
|
|
316
|
+
rows.append(
|
|
317
|
+
Improvement(
|
|
318
|
+
code="robots_unreadable",
|
|
319
|
+
finding=f"{_of(len(unreadable), len(urls), 'cited pages')} on "
|
|
320
|
+
f"{'a host' if len(hosts) == 1 else 'hosts'} whose robots.txt "
|
|
321
|
+
f"{' or '.join(kinds)}{where}. {rule}, so it did not read "
|
|
322
|
+
f"{'it' if one else 'them'}.",
|
|
323
|
+
fix=" ".join(fixes),
|
|
324
|
+
count=len(unreadable),
|
|
325
|
+
of=len(urls),
|
|
326
|
+
sources=("page_meta.jsonl", "fetch_log.jsonl"),
|
|
327
|
+
examples=tuple(unreadable[:EXAMPLES]),
|
|
328
|
+
)
|
|
329
|
+
)
|
|
330
|
+
refused = [u for u in urls if outcome[u] in ("blocked", "gated")]
|
|
331
|
+
if refused:
|
|
332
|
+
rows.append(
|
|
333
|
+
Improvement(
|
|
334
|
+
code="blocked",
|
|
335
|
+
finding=f"{_of(len(refused), len(urls), 'cited pages')} refused to automated "
|
|
336
|
+
"readers or behind a sign-in when checked.",
|
|
337
|
+
fix="If these pages are public guidance, let well-behaved readers load them so "
|
|
338
|
+
"their dates can be checked.",
|
|
339
|
+
count=len(refused),
|
|
340
|
+
of=len(urls),
|
|
341
|
+
sources=("page_meta.jsonl",),
|
|
342
|
+
examples=tuple(refused[:EXAMPLES]),
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
files = [u for u in urls if outcome[u] in ("not_html", "too_large")]
|
|
346
|
+
if files:
|
|
347
|
+
rows.append(
|
|
348
|
+
Improvement(
|
|
349
|
+
code="not_html",
|
|
350
|
+
finding=f"{_of(len(files), len(urls), 'cited links', DOCS)} "
|
|
351
|
+
f"({'for example a PDF' if len(files) == 1 else 'for example PDFs'}); citewatch "
|
|
352
|
+
"reads dates from web pages only.",
|
|
353
|
+
fix="Publish key guidance as an HTML page with a visible date, and link the "
|
|
354
|
+
"document from it.",
|
|
355
|
+
count=len(files),
|
|
356
|
+
of=len(urls),
|
|
357
|
+
sources=("page_meta.jsonl",),
|
|
358
|
+
examples=tuple(files[:EXAMPLES]),
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
broken = [u for u in urls if outcome[u] == "fetch_error"]
|
|
362
|
+
if broken:
|
|
363
|
+
rows.append(
|
|
364
|
+
Improvement(
|
|
365
|
+
code="fetch_error",
|
|
366
|
+
finding=f"{_of(len(broken), len(urls), 'cited pages')} failing to load when "
|
|
367
|
+
"checked (for example a 404 or a timeout).",
|
|
368
|
+
fix="Fix or redirect these URLs; AI answers still link to them.",
|
|
369
|
+
count=len(broken),
|
|
370
|
+
of=len(urls),
|
|
371
|
+
sources=("page_meta.jsonl",),
|
|
372
|
+
examples=tuple(broken[:EXAMPLES]),
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
# unsupported: both AI judges say the claim is not on the page, and the owner's check did
|
|
376
|
+
# not find it supported either (D-69: never contradict the owner's labels)
|
|
377
|
+
mine = {ref: v for ref, v in st.verdicts.items() if st.cite_pub.get(ref) == publisher}
|
|
378
|
+
both_no = sorted(
|
|
379
|
+
r
|
|
380
|
+
for r, v in mine.items()
|
|
381
|
+
if len(v) >= 2 and "supported" not in v.values() and st.hand.get(r) != "supported"
|
|
382
|
+
)
|
|
383
|
+
if both_no:
|
|
384
|
+
rows.append(
|
|
385
|
+
Improvement(
|
|
386
|
+
code="unsupported",
|
|
387
|
+
finding=f"In the support sample, both AI judges found the cited claim not stated "
|
|
388
|
+
f"on the page for {len(both_no)} of {len(mine)} sampled citations, and the "
|
|
389
|
+
"owner's check did not find it supported.",
|
|
390
|
+
fix="State the key facts these answers rely on explicitly in the page text "
|
|
391
|
+
"(numbers, eligibility, steps), not only in images, tables or linked files.",
|
|
392
|
+
count=len(both_no),
|
|
393
|
+
of=len(mine),
|
|
394
|
+
sources=("support_verdicts.jsonl", "support_labels.jsonl"),
|
|
395
|
+
examples=tuple(both_no[:EXAMPLES]),
|
|
396
|
+
)
|
|
397
|
+
)
|
|
398
|
+
return rows
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def build_cards(
|
|
402
|
+
base: Path, data: StudyData, summary: StudySummary, stale_days: int
|
|
403
|
+
) -> list[DomainCard]:
|
|
404
|
+
from citewatch.support.report import load_hand_labels
|
|
405
|
+
|
|
406
|
+
answers = [a for a in data.answers if a.valid]
|
|
407
|
+
newest_ts: dict[str, dt.date] = {}
|
|
408
|
+
for a in answers:
|
|
409
|
+
if a.time_sensitive:
|
|
410
|
+
for c in a.cites:
|
|
411
|
+
d = a.answered_at.date()
|
|
412
|
+
if c.url not in newest_ts or d > newest_ts[c.url]:
|
|
413
|
+
newest_ts[c.url] = d
|
|
414
|
+
st = _Study(
|
|
415
|
+
meta={r["url"]: r for r in _rows(base / "data" / "page_meta.jsonl")},
|
|
416
|
+
robots=_unreadable_robots(base),
|
|
417
|
+
verdicts=_verdicts(base),
|
|
418
|
+
hand=load_hand_labels(base),
|
|
419
|
+
cite_pub=_citation_publishers(base, set(summary.runs)),
|
|
420
|
+
newest_ts=newest_ts,
|
|
421
|
+
intent_leaders=_intent_leaders(summary),
|
|
422
|
+
stale_days=stale_days,
|
|
423
|
+
answers=answers,
|
|
424
|
+
)
|
|
425
|
+
pooled = {r.publisher: r for r in summary.freshness if r.model is None}
|
|
426
|
+
presence = [r for r in summary.share if r.kind == "presence"]
|
|
427
|
+
citations: Counter[str] = Counter()
|
|
428
|
+
citing: dict[str, set[tuple[int, str, str]]] = defaultdict(set)
|
|
429
|
+
questions: dict[str, set[str]] = defaultdict(set)
|
|
430
|
+
for a in answers:
|
|
431
|
+
for c in a.cites:
|
|
432
|
+
citations[c.publisher] += 1
|
|
433
|
+
citing[c.publisher].add((a.run, a.qid, a.model))
|
|
434
|
+
questions[c.publisher].add(a.qid)
|
|
435
|
+
cards = []
|
|
436
|
+
for p in sorted(citations, key=lambda p: (-citations[p], p)):
|
|
437
|
+
fresh = pooled.get(p)
|
|
438
|
+
cards.append(
|
|
439
|
+
DomainCard(
|
|
440
|
+
publisher=p,
|
|
441
|
+
share_by_model={
|
|
442
|
+
_key(r.model, f"run-{r.run}"): r.share
|
|
443
|
+
for r in presence
|
|
444
|
+
if r.publisher == p and r.intent is None
|
|
445
|
+
},
|
|
446
|
+
share_by_intent={
|
|
447
|
+
_key(r.intent, r.model, f"run-{r.run}"): r.share
|
|
448
|
+
for r in presence
|
|
449
|
+
if r.publisher == p and r.intent is not None
|
|
450
|
+
},
|
|
451
|
+
citations=citations[p],
|
|
452
|
+
answers_citing=len(citing[p]),
|
|
453
|
+
median_freshness_lag_days=fresh.median_lag_days if fresh else None,
|
|
454
|
+
dated_pages=fresh.dated_pages if fresh else 0,
|
|
455
|
+
questions_citing=tuple(sorted(questions[p])),
|
|
456
|
+
cited_instead=cited_instead(data, p),
|
|
457
|
+
improvement=tuple(_improvements(p, summary, st)),
|
|
458
|
+
)
|
|
459
|
+
)
|
|
460
|
+
return cards
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def write_cards(base: Path, cards: Iterable[DomainCard]) -> Path:
|
|
464
|
+
path = base / "results" / CARDS_FILE
|
|
465
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
466
|
+
payload = [c.model_dump(mode="json") for c in cards]
|
|
467
|
+
tmp = path.with_suffix(".json.tmp")
|
|
468
|
+
tmp.write_text(json.dumps(payload, indent=1, sort_keys=True) + "\n", encoding="utf-8")
|
|
469
|
+
tmp.replace(path)
|
|
470
|
+
return path
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def read_cards(path: Path) -> list[DomainCard]:
|
|
474
|
+
return [DomainCard.model_validate(c) for c in json.loads(path.read_text(encoding="utf-8"))]
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
@dataclass(frozen=True)
|
|
478
|
+
class NotCited:
|
|
479
|
+
query: str
|
|
480
|
+
publisher: str | None
|
|
481
|
+
answers: int
|
|
482
|
+
message: str
|
|
483
|
+
top_for_intent: tuple[str, ...]
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def normalise(query: str) -> str:
|
|
487
|
+
"""Strip scheme, user info, `www.`, path, query and port; lower-case; IDN shown as Unicode."""
|
|
488
|
+
q = query.strip()
|
|
489
|
+
try:
|
|
490
|
+
host = urlsplit(q if "//" in q else f"//{q}").hostname or ""
|
|
491
|
+
except ValueError:
|
|
492
|
+
return ""
|
|
493
|
+
host = host.removeprefix("www.").rstrip(".").lower()
|
|
494
|
+
try:
|
|
495
|
+
return host.encode("ascii").decode("idna") if "xn--" in host else host
|
|
496
|
+
except UnicodeError:
|
|
497
|
+
return host
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _registrable(domain: str) -> bool:
|
|
501
|
+
from citewatch.extract.urls import _extractor
|
|
502
|
+
|
|
503
|
+
return bool(_extractor()(domain).domain)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _intents(cards: Sequence[DomainCard]) -> set[str]:
|
|
507
|
+
return {k.split("/")[0] for c in cards for k in c.share_by_intent}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def lookup(
|
|
511
|
+
query: str,
|
|
512
|
+
cards: Sequence[DomainCard],
|
|
513
|
+
answers: int,
|
|
514
|
+
intent: str | None = None,
|
|
515
|
+
) -> DomainCard | NotCited:
|
|
516
|
+
"""The card for the publisher `query` maps to, through the extractor's own `publisher_of`;
|
|
517
|
+
else a not-cited result naming the study's `answers` and the top publishers for `intent`.
|
|
518
|
+
An intent the study does not have raises ValueError."""
|
|
519
|
+
if intent is not None and intent not in _intents(cards):
|
|
520
|
+
raise ValueError(f"unknown intent {intent!r}; this study has {sorted(_intents(cards))}")
|
|
521
|
+
host = normalise(query)
|
|
522
|
+
publisher = None
|
|
523
|
+
if host and "." in host and " " not in host and "\\" not in query:
|
|
524
|
+
try:
|
|
525
|
+
publisher = publisher_of(host)[0]
|
|
526
|
+
except UrlError:
|
|
527
|
+
publisher = None
|
|
528
|
+
if publisher and not _registrable(publisher) and publisher not in INSTITUTIONAL:
|
|
529
|
+
publisher = None # a public suffix such as co.uk is not a site (D-4 keeps nhs.uk)
|
|
530
|
+
by = {c.publisher: c for c in cards}
|
|
531
|
+
if publisher in by:
|
|
532
|
+
return by[publisher]
|
|
533
|
+
if intent:
|
|
534
|
+
|
|
535
|
+
def k_for(c: DomainCard) -> int:
|
|
536
|
+
return sum(
|
|
537
|
+
(s.k or 0) for key, s in c.share_by_intent.items() if key.startswith(f"{intent}/")
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
top = tuple(
|
|
541
|
+
c.publisher for c in sorted(cards, key=lambda c: (-k_for(c), c.publisher)) if k_for(c)
|
|
542
|
+
)
|
|
543
|
+
else:
|
|
544
|
+
top = tuple(c.publisher for c in cards)
|
|
545
|
+
if publisher is None:
|
|
546
|
+
message = f"{query.strip()!r} is not a web address citewatch can look up."
|
|
547
|
+
else:
|
|
548
|
+
message = (
|
|
549
|
+
f"{normalise(publisher)} was not cited in any of the {answers} answers in this study."
|
|
550
|
+
)
|
|
551
|
+
return NotCited(
|
|
552
|
+
query=query,
|
|
553
|
+
publisher=publisher,
|
|
554
|
+
answers=answers,
|
|
555
|
+
message=message,
|
|
556
|
+
top_for_intent=top[:TOP_FOR_INTENT],
|
|
557
|
+
)
|