citewatch 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. citewatch-0.1.0/.gitignore +28 -0
  2. citewatch-0.1.0/LICENSE +21 -0
  3. citewatch-0.1.0/PKG-INFO +210 -0
  4. citewatch-0.1.0/README.md +169 -0
  5. citewatch-0.1.0/changelog.md +40 -0
  6. citewatch-0.1.0/pyproject.toml +111 -0
  7. citewatch-0.1.0/src/citewatch/__init__.py +57 -0
  8. citewatch-0.1.0/src/citewatch/cards.py +557 -0
  9. citewatch-0.1.0/src/citewatch/cli.py +863 -0
  10. citewatch-0.1.0/src/citewatch/cost.py +262 -0
  11. citewatch-0.1.0/src/citewatch/dates/__init__.py +34 -0
  12. citewatch-0.1.0/src/citewatch/dates/candidates.py +408 -0
  13. citewatch-0.1.0/src/citewatch/dates/pipeline.py +119 -0
  14. citewatch-0.1.0/src/citewatch/dates/resolve.py +101 -0
  15. citewatch-0.1.0/src/citewatch/dates/rules.py +80 -0
  16. citewatch-0.1.0/src/citewatch/dates/sitemaps.py +135 -0
  17. citewatch-0.1.0/src/citewatch/demo/__init__.py +7 -0
  18. citewatch-0.1.0/src/citewatch/demo/component.py +138 -0
  19. citewatch-0.1.0/src/citewatch/demo/guard.py +153 -0
  20. citewatch-0.1.0/src/citewatch/demo/kv.py +212 -0
  21. citewatch-0.1.0/src/citewatch/demo/live.py +273 -0
  22. citewatch-0.1.0/src/citewatch/demo/logs.py +42 -0
  23. citewatch-0.1.0/src/citewatch/demo/lookup.py +157 -0
  24. citewatch-0.1.0/src/citewatch/demo/make_code_hash.py +49 -0
  25. citewatch-0.1.0/src/citewatch/demo/markup.py +38 -0
  26. citewatch-0.1.0/src/citewatch/demo/passkeys.py +386 -0
  27. citewatch-0.1.0/src/citewatch/demo/settings.py +66 -0
  28. citewatch-0.1.0/src/citewatch/env.py +48 -0
  29. citewatch-0.1.0/src/citewatch/examples/uk-health-summary.json.gz +0 -0
  30. citewatch-0.1.0/src/citewatch/extract/__init__.py +18 -0
  31. citewatch-0.1.0/src/citewatch/extract/extractors.py +147 -0
  32. citewatch-0.1.0/src/citewatch/extract/stage.py +189 -0
  33. citewatch-0.1.0/src/citewatch/extract/urls.py +108 -0
  34. citewatch-0.1.0/src/citewatch/fetch/__init__.py +28 -0
  35. citewatch-0.1.0/src/citewatch/fetch/client.py +434 -0
  36. citewatch-0.1.0/src/citewatch/fetch/history.py +132 -0
  37. citewatch-0.1.0/src/citewatch/fetch/keys.py +125 -0
  38. citewatch-0.1.0/src/citewatch/fetch/log.py +56 -0
  39. citewatch-0.1.0/src/citewatch/fetch/pages.py +266 -0
  40. citewatch-0.1.0/src/citewatch/fetch/rate.py +99 -0
  41. citewatch-0.1.0/src/citewatch/fetch/robots.py +140 -0
  42. citewatch-0.1.0/src/citewatch/handcheck.py +221 -0
  43. citewatch-0.1.0/src/citewatch/ledger.py +75 -0
  44. citewatch-0.1.0/src/citewatch/metrics/__init__.py +5 -0
  45. citewatch-0.1.0/src/citewatch/metrics/compute.py +602 -0
  46. citewatch-0.1.0/src/citewatch/metrics/data.py +166 -0
  47. citewatch-0.1.0/src/citewatch/metrics/stats.py +120 -0
  48. citewatch-0.1.0/src/citewatch/metrics/summary.py +231 -0
  49. citewatch-0.1.0/src/citewatch/metrics/views.py +44 -0
  50. citewatch-0.1.0/src/citewatch/prices.yaml +19 -0
  51. citewatch-0.1.0/src/citewatch/py.typed +0 -0
  52. citewatch-0.1.0/src/citewatch/questions.py +84 -0
  53. citewatch-0.1.0/src/citewatch/report/__init__.py +5 -0
  54. citewatch-0.1.0/src/citewatch/report/dashboard.py +762 -0
  55. citewatch-0.1.0/src/citewatch/report/post.py +235 -0
  56. citewatch-0.1.0/src/citewatch/runners/__init__.py +29 -0
  57. citewatch-0.1.0/src/citewatch/runners/anthropic_runner.py +168 -0
  58. citewatch-0.1.0/src/citewatch/runners/base.py +144 -0
  59. citewatch-0.1.0/src/citewatch/runners/openai_runner.py +117 -0
  60. citewatch-0.1.0/src/citewatch/runners/schedule.py +423 -0
  61. citewatch-0.1.0/src/citewatch/runs.py +202 -0
  62. citewatch-0.1.0/src/citewatch/schemas/__init__.py +61 -0
  63. citewatch-0.1.0/src/citewatch/schemas/_base.py +160 -0
  64. citewatch-0.1.0/src/citewatch/schemas/_iso.py +257 -0
  65. citewatch-0.1.0/src/citewatch/schemas/cards.py +120 -0
  66. citewatch-0.1.0/src/citewatch/schemas/config.py +232 -0
  67. citewatch-0.1.0/src/citewatch/schemas/pages.py +92 -0
  68. citewatch-0.1.0/src/citewatch/schemas/study.py +133 -0
  69. citewatch-0.1.0/src/citewatch/schemas/summary.py +187 -0
  70. citewatch-0.1.0/src/citewatch/schemas/thresholds.py +165 -0
  71. citewatch-0.1.0/src/citewatch/store.py +127 -0
  72. citewatch-0.1.0/src/citewatch/support/__init__.py +5 -0
  73. citewatch-0.1.0/src/citewatch/support/calibrate.py +83 -0
  74. citewatch-0.1.0/src/citewatch/support/judge.py +455 -0
  75. citewatch-0.1.0/src/citewatch/support/report.py +191 -0
  76. citewatch-0.1.0/src/citewatch/support/sample.py +84 -0
  77. citewatch-0.1.0/src/citewatch/templates/RUBRIC.md +9 -0
  78. citewatch-0.1.0/src/citewatch/templates/env.example +3 -0
  79. citewatch-0.1.0/src/citewatch/templates/questions.jsonl +2 -0
  80. citewatch-0.1.0/src/citewatch/templates/study.yaml +23 -0
  81. citewatch-0.1.0/src/citewatch/thresholds.py +129 -0
@@ -0,0 +1,28 @@
1
+ .venv/
2
+ .venv-*/
3
+ __pycache__/
4
+ *.pyc
5
+ dist/
6
+ build/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .harness/
11
+ .env
12
+ .DS_Store
13
+
14
+ # Claude Code local settings
15
+ .claude/settings.local.json
16
+ .claude/scheduled_tasks.lock
17
+
18
+ # Streamlit local secrets and the demo's local development store
19
+ .streamlit/secrets.toml
20
+ .demo-state/
21
+
22
+ # personal notes, not for the public repo (OWNER D-22)
23
+ Portfolio notes.md
24
+
25
+ # sitemap fetch cache: whole-host lastmod maps, reproducible, and the chosen sitemap dates are kept in page_meta (D-53)
26
+ studies/*/data/raw/sitemap/
27
+ data/raw/sitemap/
28
+ studies/*/results/site/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Biplob Chakraborty
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,210 @@
1
+ Metadata-Version: 2.5
2
+ Name: citewatch
3
+ Version: 0.1.0
4
+ Summary: Measure which websites AI answers with web search cite, and how fresh those pages are.
5
+ Project-URL: Homepage, https://github.com/biplob819/ai-visibility
6
+ Project-URL: Issues, https://github.com/biplob819/ai-visibility/issues
7
+ Author: Biplob Chakraborty
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: ai-search,citations,freshness,llm,provenance,research,web-search
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
20
+ Requires-Python: >=3.11
21
+ Requires-Dist: defusedxml>=0.7
22
+ Requires-Dist: duckdb>=1.1
23
+ Requires-Dist: extruct>=0.17
24
+ Requires-Dist: httpx>=0.28
25
+ Requires-Dist: idna>=3.7
26
+ Requires-Dist: numpy>=1.26
27
+ Requires-Dist: protego>=0.4
28
+ Requires-Dist: pydantic>=2.7
29
+ Requires-Dist: python-dateutil>=2.9
30
+ Requires-Dist: pyyaml>=6
31
+ Requires-Dist: tldextract>=5.1
32
+ Requires-Dist: typer>=0.28
33
+ Provides-Extra: anthropic
34
+ Requires-Dist: anthropic>=1; extra == 'anthropic'
35
+ Provides-Extra: demo
36
+ Requires-Dist: streamlit>=1.67; extra == 'demo'
37
+ Requires-Dist: webauthn>=3; extra == 'demo'
38
+ Provides-Extra: openai
39
+ Requires-Dist: openai>=2; extra == 'openai'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # citewatch
43
+
44
+ **Which websites do AI search answers cite, and how fresh are those pages?**
45
+
46
+ citewatch asks AI models questions with web search on, records every source they cite, visits
47
+ each cited page politely to find when it was last updated, and reports who gets cited, how
48
+ fresh it is and how far to trust each number, with counts and 95% intervals throughout. This
49
+ repository holds the tool and its first study: 100 pre-registered UK health questions asked of
50
+ Claude Sonnet 5.5 and GPT-6.1 Sol, twice.
51
+
52
+ **[Dashboard](https://biplob819.github.io/ai-visibility/)** ·
53
+ **[Post](https://biplob819.github.io/ai-visibility/post/)** ·
54
+ **[Signal spec (PDF)](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf)** ·
55
+ **[Data card](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md)** ·
56
+ **[Pre-registration](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md)** ·
57
+ **[Deviations](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md)** ·
58
+ **[Decisions](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md)**
59
+
60
+ It measures **sourcing, not medical accuracy**, and nothing here is medical advice.
61
+
62
+ [![A walkthrough of the dashboard, CLI, demo lookup and post](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/walkthrough.gif)](https://biplob819.github.io/ai-visibility/)
63
+
64
+ ## What the first study found
65
+
66
+ | | GPT-6.1 Sol | Claude Sonnet 5.5 |
67
+ |---|---|---|
68
+ | Answers citing nhs.uk (run 1) | 83% (83/100; 95% CI 75%–90%) | 10% (10/100; 95% CI 5.0%–16%) |
69
+ | Citation weight held by its top 10 publishers (run 1) | 78% (n=100; 95% CI 75%–85%) | 24% (n=100; 95% CI 23%–31%) |
70
+ | Time-sensitive answers whose newest dated source is over a year old (both runs) | 43% (49/114; 95% CI 31%–55%) | 12% (14/114; 95% CI 5.8%–20%) |
71
+
72
+ - The two models rarely cite the same publishers for the same question: Jaccard similarity
73
+ 10% (n=100; 95% CI 7.4%–13%) per question, run 1.
74
+ - The nhs.uk gap is separated in both runs after a Holm correction.
75
+ - How far to trust it: citewatch dated (or logged a reason for) 89% (745/841; 95% CI 86%–91%)
76
+ of cited URLs, against a 90% target. On held-out pages it was within 7 days of the checked date on
77
+ 67% (18/27; 95% CI 48%–81%) (agent-checked, owner-approved). In the support check, the two AI
78
+ judges and the owner disagree, so the support rate is reported, not settled.
79
+ - These findings are directional: one country, two models, 100 questions.
80
+
81
+ [![The findings dashboard](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/screenshots/dashboard.png)](https://biplob819.github.io/ai-visibility/)
82
+
83
+ ## Install
84
+
85
+ ```bash
86
+ pip install citewatch # the core: report, score, dates, lookup
87
+ pip install 'citewatch[anthropic,openai]' # plus the provider SDKs for paid runs
88
+ uvx citewatch report --example # or try it without installing
89
+ ```
90
+
91
+ v0.1.0 is not on PyPI yet; until it is, install from the repository with
92
+ `pip install git+https://github.com/biplob819/ai-visibility`. Python 3.11 or newer. See the [quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) (an example report in under
93
+ five minutes, no keys).
94
+
95
+ [![The citewatch CLI: report, init, doctor and a dry run, with no keys and no spend](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/screenshots/cli.png)](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md)
96
+
97
+ ## How it works
98
+
99
+ ![citewatch data flow](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/dataflow.svg)
100
+
101
+ 1. **Ask.** Each frozen question goes to each model with its native web search, from one
102
+ location, under a cost cap; responses are stored verbatim (`citewatch ask`).
103
+ 2. **Extract.** Citations come only from the APIs' citation fields, with canonical URLs and a
104
+ publisher per URL (`citewatch extract`).
105
+ 3. **Fetch and date.** Each cited page is fetched once, obeying robots.txt at one request per
106
+ second per host; page bodies are held in memory and never stored. A date resolver reads
107
+ seven sources (visible text, schema.org, meta tags, microdata and RDFa, the search tool's
108
+ page age, sitemaps, HTTP headers) and never guesses (`citewatch fetch`).
109
+ 4. **Score.** Shares, freshness, staleness, overlap and stability with a question-cluster
110
+ bootstrap, Holm-corrected comparisons and Wilson intervals; domain cards for every cited
111
+ publisher, each with an improvement report (`citewatch score`).
112
+ 5. **Report.** The dashboard and post are built from `summary.json` (`citewatch report`).
113
+
114
+ Two checks sit under the numbers: a **support check** (do cited pages state the claim? two AI
115
+ judges via [EvalKit](https://github.com/biplob819/evalkit) and the owner's labels) and a
116
+ **held-out date check**. Architecture: [`docs/assets/architecture.svg`](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/architecture.svg).
117
+
118
+ ## Rerun it
119
+
120
+ Look at the results with no keys and no spend:
121
+
122
+ ```bash
123
+ git clone https://github.com/biplob819/ai-visibility && cd ai-visibility
124
+ uv sync
125
+ uv run citewatch report --example # the bundled study's dashboard
126
+ ```
127
+
128
+ Run your own study (paid calls need the provider SDKs and your API keys):
129
+
130
+ ```bash
131
+ uv sync --extra anthropic --extra openai
132
+ uv run citewatch init my-study --contact <a page site owners can reach you at>
133
+ # edit my-study/study.yaml and my-study/questions.jsonl, and set the API keys in my-study/.env
134
+ uv run citewatch doctor --study my-study # names any missing key or SDK
135
+ uv run citewatch ask --run 0 --dry-run --study my-study # cost estimate, no calls
136
+
137
+ # 1. A small paid smoke run measures what an answer really costs
138
+ uv run citewatch ask --run 0 --study my-study
139
+ uv run python scripts/measure_cost.py my-study --run 0 # writes my-study/cost_model.yaml
140
+
141
+ # 2. Before the study runs: freeze the questions and the metric definitions
142
+ # write my-study/preregistration.md, record the sha256 of it and of questions.jsonl in a
143
+ # decisions.md, and commit everything (a study run refuses a dirty or non-git folder)
144
+
145
+ # 3. The study runs, then the analysis
146
+ uv run citewatch ask --run 1 --study my-study # run 2 waits at least two hours
147
+ uv run citewatch extract --study my-study
148
+ uv run citewatch fetch --study my-study
149
+ uv run citewatch score --study my-study
150
+ uv run citewatch report --study my-study --title "my topic"
151
+ ```
152
+
153
+ `citewatch ask --run 1 --dry-run` lists anything the run gate still needs before it will spend.
154
+ The gate and why each check exists are in the Tech Spec (§3 and gates G1, G2).
155
+
156
+ **Cost.** The flagship study's smoke run cost $0.79 and its two study runs (400 answers) cost
157
+ $15.03 in API calls; the dry run estimates yours before any call, and `max_cost_usd` in
158
+ `study.yaml` caps it. Fetching costs nothing but time. Thresholds in `study.yaml` can fail a CI
159
+ job when a metric crosses a bound ([`docs/thresholds.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md)).
160
+
161
+ ## Guides
162
+
163
+ | Guide | What it covers |
164
+ |---|---|
165
+ | [Quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) | the bundled study's dashboard in under five minutes, no keys |
166
+ | [Run your own study](https://github.com/biplob819/ai-visibility/blob/main/docs/own-study.md) | settings, the questions format, intents, country, models, the run gate |
167
+ | [How dates are resolved](https://github.com/biplob819/ai-visibility/blob/main/docs/dates.md) | the seven date sources, the rules and their limits |
168
+ | [Controlling cost](https://github.com/biplob819/ai-visibility/blob/main/docs/cost.md) | estimates, the hard cap, resume, what the flagship study cost |
169
+ | [Thresholds in CI](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md) | rules that fail a CI job when a metric crosses a bound |
170
+ | [Responsible use](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md) | politeness, no telemetry, privacy, not for gaming AI answers |
171
+ | [Versioning](https://github.com/biplob819/ai-visibility/blob/main/docs/versioning.md) | semver and the public API |
172
+
173
+ ## Responsible use
174
+
175
+ citewatch obeys robots.txt and a fixed politeness floor that no setting can loosen, sends
176
+ nothing anywhere except to the AI providers you configure and the cited sites, and is not a
177
+ tool for gaming AI answers ([`docs/responsible-use.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md)).
178
+
179
+ ## Repository map
180
+
181
+ | Path | What it holds |
182
+ |---|---|
183
+ | `src/citewatch/` | the package: runners, extract, fetch, dates, metrics, support, cards, report, demo |
184
+ | `studies/uk-health/` | the flagship study: questions, settings, frozen data, results |
185
+ | `docs/` | pre-registration, deviations, decisions, signal spec, data-flow and architecture diagrams |
186
+ | `demo/` | the Streamlit demo (lookup; live mode behind passkeys) |
187
+ | `specs/`, `docs/graph/` | the build plan: one spec per node and the capability catalog |
188
+
189
+ ## Links
190
+
191
+ | | |
192
+ |---|---|
193
+ | Repository | https://github.com/biplob819/ai-visibility |
194
+ | Package | https://pypi.org/project/citewatch/ |
195
+ | Dashboard | https://biplob819.github.io/ai-visibility/ |
196
+ | Post | https://biplob819.github.io/ai-visibility/post/ |
197
+ | Signal spec | [docs/signal-spec.pdf](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf) |
198
+ | Data card | [DATA_CARD.md](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md) |
199
+ | Pre-registration | [docs/preregistration.md](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md) |
200
+ | Deviations | [docs/deviations.md](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md) |
201
+ | Decisions | [docs/decisions.md](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md) |
202
+ | Changelog | [changelog.md](https://github.com/biplob819/ai-visibility/blob/main/changelog.md) |
203
+ | Support judging | [EvalKit](https://github.com/biplob819/evalkit) |
204
+
205
+ `scripts/check_links.py --online` checks every link in the README and docs.
206
+
207
+ ## Licence
208
+
209
+ MIT ([`LICENSE`](https://github.com/biplob819/ai-visibility/blob/main/LICENSE)). Contributions: [`CONTRIBUTING.md`](https://github.com/biplob819/ai-visibility/blob/main/CONTRIBUTING.md); security
210
+ reports: [`SECURITY.md`](https://github.com/biplob819/ai-visibility/blob/main/SECURITY.md).
@@ -0,0 +1,169 @@
1
+ # citewatch
2
+
3
+ **Which websites do AI search answers cite, and how fresh are those pages?**
4
+
5
+ citewatch asks AI models questions with web search on, records every source they cite, visits
6
+ each cited page politely to find when it was last updated, and reports who gets cited, how
7
+ fresh it is and how far to trust each number, with counts and 95% intervals throughout. This
8
+ repository holds the tool and its first study: 100 pre-registered UK health questions asked of
9
+ Claude Sonnet 5.5 and GPT-6.1 Sol, twice.
10
+
11
+ **[Dashboard](https://biplob819.github.io/ai-visibility/)** ·
12
+ **[Post](https://biplob819.github.io/ai-visibility/post/)** ·
13
+ **[Signal spec (PDF)](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf)** ·
14
+ **[Data card](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md)** ·
15
+ **[Pre-registration](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md)** ·
16
+ **[Deviations](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md)** ·
17
+ **[Decisions](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md)**
18
+
19
+ It measures **sourcing, not medical accuracy**, and nothing here is medical advice.
20
+
21
+ [![A walkthrough of the dashboard, CLI, demo lookup and post](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/walkthrough.gif)](https://biplob819.github.io/ai-visibility/)
22
+
23
+ ## What the first study found
24
+
25
+ | | GPT-6.1 Sol | Claude Sonnet 5.5 |
26
+ |---|---|---|
27
+ | Answers citing nhs.uk (run 1) | 83% (83/100; 95% CI 75%–90%) | 10% (10/100; 95% CI 5.0%–16%) |
28
+ | Citation weight held by its top 10 publishers (run 1) | 78% (n=100; 95% CI 75%–85%) | 24% (n=100; 95% CI 23%–31%) |
29
+ | Time-sensitive answers whose newest dated source is over a year old (both runs) | 43% (49/114; 95% CI 31%–55%) | 12% (14/114; 95% CI 5.8%–20%) |
30
+
31
+ - The two models rarely cite the same publishers for the same question: Jaccard similarity
32
+ 10% (n=100; 95% CI 7.4%–13%) per question, run 1.
33
+ - The nhs.uk gap is separated in both runs after a Holm correction.
34
+ - How far to trust it: citewatch dated (or logged a reason for) 89% (745/841; 95% CI 86%–91%)
35
+ of cited URLs, against a 90% target. On held-out pages it was within 7 days of the checked date on
36
+ 67% (18/27; 95% CI 48%–81%) (agent-checked, owner-approved). In the support check, the two AI
37
+ judges and the owner disagree, so the support rate is reported, not settled.
38
+ - These findings are directional: one country, two models, 100 questions.
39
+
40
+ [![The findings dashboard](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/screenshots/dashboard.png)](https://biplob819.github.io/ai-visibility/)
41
+
42
+ ## Install
43
+
44
+ ```bash
45
+ pip install citewatch # the core: report, score, dates, lookup
46
+ pip install 'citewatch[anthropic,openai]' # plus the provider SDKs for paid runs
47
+ uvx citewatch report --example # or try it without installing
48
+ ```
49
+
50
+ v0.1.0 is not on PyPI yet; until it is, install from the repository with
51
+ `pip install git+https://github.com/biplob819/ai-visibility`. Python 3.11 or newer. See the [quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) (an example report in under
52
+ five minutes, no keys).
53
+
54
+ [![The citewatch CLI: report, init, doctor and a dry run, with no keys and no spend](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/screenshots/cli.png)](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md)
55
+
56
+ ## How it works
57
+
58
+ ![citewatch data flow](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/dataflow.svg)
59
+
60
+ 1. **Ask.** Each frozen question goes to each model with its native web search, from one
61
+ location, under a cost cap; responses are stored verbatim (`citewatch ask`).
62
+ 2. **Extract.** Citations come only from the APIs' citation fields, with canonical URLs and a
63
+ publisher per URL (`citewatch extract`).
64
+ 3. **Fetch and date.** Each cited page is fetched once, obeying robots.txt at one request per
65
+ second per host; page bodies are held in memory and never stored. A date resolver reads
66
+ seven sources (visible text, schema.org, meta tags, microdata and RDFa, the search tool's
67
+ page age, sitemaps, HTTP headers) and never guesses (`citewatch fetch`).
68
+ 4. **Score.** Shares, freshness, staleness, overlap and stability with a question-cluster
69
+ bootstrap, Holm-corrected comparisons and Wilson intervals; domain cards for every cited
70
+ publisher, each with an improvement report (`citewatch score`).
71
+ 5. **Report.** The dashboard and post are built from `summary.json` (`citewatch report`).
72
+
73
+ Two checks sit under the numbers: a **support check** (do cited pages state the claim? two AI
74
+ judges via [EvalKit](https://github.com/biplob819/evalkit) and the owner's labels) and a
75
+ **held-out date check**. Architecture: [`docs/assets/architecture.svg`](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/architecture.svg).
76
+
77
+ ## Rerun it
78
+
79
+ Look at the results with no keys and no spend:
80
+
81
+ ```bash
82
+ git clone https://github.com/biplob819/ai-visibility && cd ai-visibility
83
+ uv sync
84
+ uv run citewatch report --example # the bundled study's dashboard
85
+ ```
86
+
87
+ Run your own study (paid calls need the provider SDKs and your API keys):
88
+
89
+ ```bash
90
+ uv sync --extra anthropic --extra openai
91
+ uv run citewatch init my-study --contact <a page site owners can reach you at>
92
+ # edit my-study/study.yaml and my-study/questions.jsonl, and set the API keys in my-study/.env
93
+ uv run citewatch doctor --study my-study # names any missing key or SDK
94
+ uv run citewatch ask --run 0 --dry-run --study my-study # cost estimate, no calls
95
+
96
+ # 1. A small paid smoke run measures what an answer really costs
97
+ uv run citewatch ask --run 0 --study my-study
98
+ uv run python scripts/measure_cost.py my-study --run 0 # writes my-study/cost_model.yaml
99
+
100
+ # 2. Before the study runs: freeze the questions and the metric definitions
101
+ # write my-study/preregistration.md, record the sha256 of it and of questions.jsonl in a
102
+ # decisions.md, and commit everything (a study run refuses a dirty or non-git folder)
103
+
104
+ # 3. The study runs, then the analysis
105
+ uv run citewatch ask --run 1 --study my-study # run 2 waits at least two hours
106
+ uv run citewatch extract --study my-study
107
+ uv run citewatch fetch --study my-study
108
+ uv run citewatch score --study my-study
109
+ uv run citewatch report --study my-study --title "my topic"
110
+ ```
111
+
112
+ `citewatch ask --run 1 --dry-run` lists anything the run gate still needs before it will spend.
113
+ The gate and why each check exists are in the Tech Spec (§3 and gates G1, G2).
114
+
115
+ **Cost.** The flagship study's smoke run cost $0.79 and its two study runs (400 answers) cost
116
+ $15.03 in API calls; the dry run estimates yours before any call, and `max_cost_usd` in
117
+ `study.yaml` caps it. Fetching costs nothing but time. Thresholds in `study.yaml` can fail a CI
118
+ job when a metric crosses a bound ([`docs/thresholds.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md)).
119
+
120
+ ## Guides
121
+
122
+ | Guide | What it covers |
123
+ |---|---|
124
+ | [Quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) | the bundled study's dashboard in under five minutes, no keys |
125
+ | [Run your own study](https://github.com/biplob819/ai-visibility/blob/main/docs/own-study.md) | settings, the questions format, intents, country, models, the run gate |
126
+ | [How dates are resolved](https://github.com/biplob819/ai-visibility/blob/main/docs/dates.md) | the seven date sources, the rules and their limits |
127
+ | [Controlling cost](https://github.com/biplob819/ai-visibility/blob/main/docs/cost.md) | estimates, the hard cap, resume, what the flagship study cost |
128
+ | [Thresholds in CI](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md) | rules that fail a CI job when a metric crosses a bound |
129
+ | [Responsible use](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md) | politeness, no telemetry, privacy, not for gaming AI answers |
130
+ | [Versioning](https://github.com/biplob819/ai-visibility/blob/main/docs/versioning.md) | semver and the public API |
131
+
132
+ ## Responsible use
133
+
134
+ citewatch obeys robots.txt and a fixed politeness floor that no setting can loosen, sends
135
+ nothing anywhere except to the AI providers you configure and the cited sites, and is not a
136
+ tool for gaming AI answers ([`docs/responsible-use.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md)).
137
+
138
+ ## Repository map
139
+
140
+ | Path | What it holds |
141
+ |---|---|
142
+ | `src/citewatch/` | the package: runners, extract, fetch, dates, metrics, support, cards, report, demo |
143
+ | `studies/uk-health/` | the flagship study: questions, settings, frozen data, results |
144
+ | `docs/` | pre-registration, deviations, decisions, signal spec, data-flow and architecture diagrams |
145
+ | `demo/` | the Streamlit demo (lookup; live mode behind passkeys) |
146
+ | `specs/`, `docs/graph/` | the build plan: one spec per node and the capability catalog |
147
+
148
+ ## Links
149
+
150
+ | | |
151
+ |---|---|
152
+ | Repository | https://github.com/biplob819/ai-visibility |
153
+ | Package | https://pypi.org/project/citewatch/ |
154
+ | Dashboard | https://biplob819.github.io/ai-visibility/ |
155
+ | Post | https://biplob819.github.io/ai-visibility/post/ |
156
+ | Signal spec | [docs/signal-spec.pdf](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf) |
157
+ | Data card | [DATA_CARD.md](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md) |
158
+ | Pre-registration | [docs/preregistration.md](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md) |
159
+ | Deviations | [docs/deviations.md](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md) |
160
+ | Decisions | [docs/decisions.md](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md) |
161
+ | Changelog | [changelog.md](https://github.com/biplob819/ai-visibility/blob/main/changelog.md) |
162
+ | Support judging | [EvalKit](https://github.com/biplob819/evalkit) |
163
+
164
+ `scripts/check_links.py --online` checks every link in the README and docs.
165
+
166
+ ## Licence
167
+
168
+ MIT ([`LICENSE`](https://github.com/biplob819/ai-visibility/blob/main/LICENSE)). Contributions: [`CONTRIBUTING.md`](https://github.com/biplob819/ai-visibility/blob/main/CONTRIBUTING.md); security
169
+ reports: [`SECURITY.md`](https://github.com/biplob819/ai-visibility/blob/main/SECURITY.md).
@@ -0,0 +1,40 @@
1
+ # Changelog
2
+
3
+ Format: `[TYPE] [NODE] — summary`. Newest first.
4
+
5
+ - [PREPARED] [B4.5] — v0.1.0, not yet on PyPI (publishes when the owner tags it): the citewatch tool and the flagship UK health study (100 pre-registered questions, Claude Sonnet 5.5 and GPT-6.1 Sol, two runs, $15.03). nhs.uk cited in 83/100 GPT answers vs 10/100 Claude (run 1); stale time-sensitive answers 49/114 vs 14/114; cross-model Jaccard 10%; date coverage 745/841; held-out date check 18/27 (agent-checked, owner-approved). Guides, thresholds for CI, public API under semver
6
+ - [GREEN] [B4.4] — verified over five passes (D-80–D-83); README, deviations record, hardened full-history audit (clean)
7
+ - [FEAT] [B4.4] — public repo: README with the finding, method, rerun steps and cost; deviations record; full-history audit (clean); screenshots; diagram and PRD updated for D-70, D-73
8
+ - [GREEN] [B4.2] — verified over two passes (D-76); post live at /post/, B9 approved (OWNER D-74)
9
+ - [FEAT] [D5.2] — demo: lookup, top ten by topic, passkey-gated live mode (owner's registration code), bring your own keys, spend guard, four-field logs; deploy prep (D-78)
10
+ - [FEAT] [B4.5] — groundwork from lane 2: thresholds and CI gate example (D-77), responsible use, community files, versioning and public API contract
11
+ - [GREEN] [D5.1] — verified over five passes (D-69, D-71, D-72, D-75); 361 domain cards with an improvement report that states only what citewatch observed
12
+ - [GREEN] [B4.3] — verified over two passes; one-page signal spec with a build stamp tying the PDF to its HTML
13
+ - [FEAT] [B4.2] — the write-up at /post/: opens with the finding and scope, three charts from summary.json, links the dashboard and signal spec; B9 review approved (OWNER D-74)
14
+ - [GREEN] [B4.1] — verified over two passes; live on GitHub Pages (OWNER D-70), served bytes equal the local build
15
+ - [FEAT] [B4.3] — one-page signal spec (docs/signal-spec.pdf): fields, buyers, delivery, pricing logic as a unit of value and tiers; every quoted finding from summary.json
16
+ - [FEAT] [B4.1] — findings dashboard: one self-contained HTML page from summary.json (inline SVG, table views, light and dark), `citewatch report` and `report --example`
17
+ - [FEAT] [D5.1] — domain cards for 361 cited publishers with `cited_instead` and an improvement report (OWNER D-65); `lookup()` through the extractor's `publisher_of`; written by `citewatch score`
18
+ - [GREEN] [S3.1] — verified over two passes; $15.03 recomputed from provider usage; run data, raw responses and support records frozen by hash
19
+ - [FEAT] [S3.1] — study runs 1 and 2 ($15.03, 400 answers, 0 errors), spend reconciled exactly, run data frozen by hash (RUN-04)
20
+ - [GREEN] [M3.3] — verified over three passes (D-67, D-68); held-out rule over distinct sampled pages; checker load failures counted, never scored; 67% within 7 days (18/27)
21
+ - [GREEN] [M3.2] — verified over three passes (D-67, D-68); support rates per model and judge, owner labels, EvalKit calibration, gaps; body cache finalised
22
+ - [FEAT] [M3.3] — held-out date check on 55 pages, agent-checked and owner-approved (OWNER D-63, D-64): 67% within 7 days (18/27 judgeable, Wilson 48–81%); label freeze tests
23
+ - [FEAT] [M3.2] — support judging with EvalKit (two judges, OWNER D-60, D-61), owner labels (OWNER D-62), judge calibration with EvalKit's cluster kappa (OWNER D-65), body cache finalised
24
+ - [GREEN] [M3.1] — verified over three passes (D-55, D-56); §6 metrics with the question-cluster bootstrap, Holm and Wilson; `citewatch score` writes summary.json; DuckDB views
25
+ - [GREEN] [A2.4] — paid smoke run 0 ($0.79), measured cost model, parity report, golden replay
26
+ - [GREEN] [N1.4] — 100 owner-approved questions frozen with the pre-registration and data card (OWNER D-47, D-51)
27
+ - [GREEN] [A2.3] — verified over five skeptical passes (D-44, D-45); 21-page real test set confirmed by the owner (OWNER D-46)
28
+ - [FEAT] [A2.3] — date resolver: candidates from the seven §7.3 sources, plausibility rules, precedence, sitemaps from robots.txt, `page_meta.jsonl` from `citewatch fetch`; 21-page real test set with drafted labels awaiting the owner (DAT-04)
29
+ - [GREEN] [A2.2] — verified over nine skeptical passes (L-32, L-33; D-42); robots verdict equals protego's on the path actually sent, wire spacing ≥1 s, no markup stored, audits clean on fuzzed multi-run logs
30
+ - [GREEN] [A2.1] — verified over three skeptical passes (D-43); only current answers extracted, every row field pinned with literal values
31
+ - [FEAT] [A2.1] — citation extraction for both providers from API citation fields only, consulted sources with page_age, `canonical_url` (idempotent, utm and click IDs stripped), `publisher_of` per D-4 offline, `citewatch extract`
32
+ - [GREEN] [N1.2] — verified over six skeptical passes (L-27, L-29 to L-31; D-37 to D-41); money recorded for every paid call under errors, interrupts and signals
33
+ - [FEAT] [A2.2] — polite fetcher: protego robots per origin kept for the study, 1 req/s per host and 8 in flight, hand-followed redirects re-checking robots, retries and page history that keep gate 8's rules across runs, HTML-only capped bodies held in memory, gating, fetch log, `citewatch fetch` with `--dry-run`
34
+ - [GREEN] [N1.3] — verified over four skeptical passes (L-28); CI green on all three systems
35
+ - [FEAT] [N1.2] — Claude and OpenAI runners with native web search, cost model, scheduler, `citewatch ask` (dry run, cap, resume), real fixtures
36
+ - [FEAT] [N1.3] — raw store, run ledger, .env loading and the citewatch CLI (init, doctor; other commands exit 2 naming their node)
37
+ - [GREEN] [N1.1] — verified over two skeptical passes (L-23, L-24); 299 tests
38
+ - [FEAT] [N1.1] — results format v1 schemas, study config and question loader, flagship study.yaml
39
+ - [GREEN] [W0.1] — verified over six skeptical passes (L-14 to L-22, D-26); 153 tests; CI green on Python 3.11–3.13 × Linux, macOS, Windows
40
+ - [FEAT] [W0.1] — repo, uv project and the lean Kova build harness ported from EvalKit (12 gates, politeness and storage gates new)
@@ -0,0 +1,111 @@
1
+ [project]
2
+ name = "citewatch"
3
+ version = "0.1.0"
4
+ description = "Measure which websites AI answers with web search cite, and how fresh those pages are."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [{ name = "Biplob Chakraborty" }]
9
+ requires-python = ">=3.11"
10
+ dependencies = [
11
+ "defusedxml>=0.7",
12
+ "duckdb>=1.1",
13
+ "extruct>=0.17",
14
+ "httpx>=0.28",
15
+ "idna>=3.7",
16
+ "numpy>=1.26",
17
+ "protego>=0.4",
18
+ "pydantic>=2.7",
19
+ "python-dateutil>=2.9",
20
+ "pyyaml>=6",
21
+ "tldextract>=5.1",
22
+ "typer>=0.28", # TyperException and the exit-code behaviour cli.main relies on
23
+ ]
24
+ keywords = ["llm", "web-search", "citations", "freshness", "provenance", "ai-search", "research"]
25
+ classifiers = [
26
+ "Development Status :: 3 - Alpha",
27
+ "Intended Audience :: Science/Research",
28
+ "Intended Audience :: Developers",
29
+ "Operating System :: OS Independent",
30
+ "Programming Language :: Python :: 3",
31
+ "Programming Language :: Python :: 3.11",
32
+ "Programming Language :: Python :: 3.12",
33
+ "Programming Language :: Python :: 3.13",
34
+ "Topic :: Scientific/Engineering :: Information Analysis",
35
+ ]
36
+
37
+ [project.urls]
38
+ Homepage = "https://github.com/biplob819/ai-visibility"
39
+ Issues = "https://github.com/biplob819/ai-visibility/issues"
40
+
41
+ [project.scripts]
42
+ citewatch = "citewatch.cli:entry"
43
+
44
+ [project.optional-dependencies]
45
+ anthropic = [
46
+ "anthropic>=1",
47
+ ]
48
+ openai = [
49
+ "openai>=2",
50
+ ]
51
+ # The demo page (D5.2, OWNER D-73): components v2 for the passkey element, py_webauthn to verify.
52
+ demo = [
53
+ "streamlit>=1.67",
54
+ "webauthn>=3",
55
+ ]
56
+
57
+ [dependency-groups]
58
+ dev = [
59
+ "pytest>=8",
60
+ "ruff>=0.6",
61
+ "pyright>=1.1.380",
62
+ "types-pyyaml>=6.0.12",
63
+ "hypothesis>=6.100",
64
+ ]
65
+ # Support check (SUP-03, D-11, D-84): EvalKit's faithfulness scorer at the pinned commit the
66
+ # flagship study used. A dependency group, not an extra: PyPI refuses a published dependency on
67
+ # a git URL, so the pin stays in the repository; `uv sync` installs it by default.
68
+ support = [
69
+ "evalkit-harness @ git+https://github.com/biplob819/evalkit@218192132ad601f02976d486d839e712e39bc4e2",
70
+ ]
71
+
72
+ [tool.uv]
73
+ default-groups = ["dev", "support"]
74
+
75
+ [build-system]
76
+ requires = ["hatchling>=1.26"]
77
+ build-backend = "hatchling.build"
78
+
79
+ [tool.hatch.build.targets.wheel]
80
+ packages = ["src/citewatch"]
81
+
82
+ [tool.hatch.build.targets.sdist]
83
+ include = ["/src/citewatch", "/README.md", "/LICENSE", "/changelog.md"]
84
+
85
+ [tool.ruff]
86
+ line-length = 100
87
+ target-version = "py311"
88
+ src = ["src", "scripts", "tests"]
89
+ extend-exclude = ["*.md"]
90
+
91
+ [tool.ruff.lint]
92
+ select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
93
+ ignore = ["RUF001", "RUF002", "RUF003"] # allow the › separator used in proof cells
94
+
95
+ [tool.pyright]
96
+ include = ["src", "scripts", "tests", "demo"]
97
+ pythonVersion = "3.11"
98
+ typeCheckingMode = "standard"
99
+ extraPaths = ["scripts"]
100
+
101
+ [tool.pytest.ini_options]
102
+ testpaths = ["tests"]
103
+ pythonpath = ["scripts"]
104
+ markers = [
105
+ "contract: schema, provider-format, CLI and imported-API contract tests",
106
+ "data: frozen-data and JSONL validation tests",
107
+ "golden: end-to-end replay of recorded responses",
108
+ "polite: robots, rate-limit, user-agent and fetch-log rules",
109
+ "safety: secrets, storage and demo-content tests",
110
+ "onboarding: no-key CLI flow",
111
+ ]
@@ -0,0 +1,57 @@
1
+ """citewatch: measure which web pages AI answers with web search cite, and how fresh they are.
2
+
3
+ Public API (semver, see docs/versioning.md): the results format in `citewatch.schemas`, and
4
+ `canonical_url`, `publisher_of`, `UrlError`, `lookup`, `read_cards` and `NotCited`, importable
5
+ from here or from `citewatch.extract.urls` and `citewatch.cards`. They load on first use, so
6
+ `import citewatch` (and the CLI) stays fast.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from importlib import import_module
12
+ from importlib.metadata import PackageNotFoundError, version
13
+ from typing import TYPE_CHECKING, Any
14
+
15
+ try:
16
+ __version__ = version("citewatch")
17
+ except PackageNotFoundError: # running from a source tree that was never installed
18
+ __version__ = "0.0.0"
19
+
20
+ if TYPE_CHECKING:
21
+ from citewatch import schemas
22
+ from citewatch.cards import NotCited, lookup, read_cards
23
+ from citewatch.extract.urls import UrlError, canonical_url, publisher_of
24
+
25
+ _LAZY = {
26
+ "canonical_url": "citewatch.extract.urls",
27
+ "publisher_of": "citewatch.extract.urls",
28
+ "UrlError": "citewatch.extract.urls",
29
+ "lookup": "citewatch.cards",
30
+ "read_cards": "citewatch.cards",
31
+ "NotCited": "citewatch.cards",
32
+ }
33
+
34
+ __all__ = [
35
+ "NotCited",
36
+ "UrlError",
37
+ "__version__",
38
+ "canonical_url",
39
+ "lookup",
40
+ "publisher_of",
41
+ "read_cards",
42
+ "schemas",
43
+ ]
44
+
45
+
46
+ def __getattr__(name: str) -> Any:
47
+ if name == "schemas":
48
+ return import_module("citewatch.schemas")
49
+ if name in _LAZY:
50
+ value = getattr(import_module(_LAZY[name]), name)
51
+ globals()[name] = value
52
+ return value
53
+ raise AttributeError(f"module 'citewatch' has no attribute {name!r}")
54
+
55
+
56
+ def __dir__() -> list[str]:
57
+ return sorted({*globals(), *__all__})