citewatch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- citewatch-0.1.0/.gitignore +28 -0
- citewatch-0.1.0/LICENSE +21 -0
- citewatch-0.1.0/PKG-INFO +210 -0
- citewatch-0.1.0/README.md +169 -0
- citewatch-0.1.0/changelog.md +40 -0
- citewatch-0.1.0/pyproject.toml +111 -0
- citewatch-0.1.0/src/citewatch/__init__.py +57 -0
- citewatch-0.1.0/src/citewatch/cards.py +557 -0
- citewatch-0.1.0/src/citewatch/cli.py +863 -0
- citewatch-0.1.0/src/citewatch/cost.py +262 -0
- citewatch-0.1.0/src/citewatch/dates/__init__.py +34 -0
- citewatch-0.1.0/src/citewatch/dates/candidates.py +408 -0
- citewatch-0.1.0/src/citewatch/dates/pipeline.py +119 -0
- citewatch-0.1.0/src/citewatch/dates/resolve.py +101 -0
- citewatch-0.1.0/src/citewatch/dates/rules.py +80 -0
- citewatch-0.1.0/src/citewatch/dates/sitemaps.py +135 -0
- citewatch-0.1.0/src/citewatch/demo/__init__.py +7 -0
- citewatch-0.1.0/src/citewatch/demo/component.py +138 -0
- citewatch-0.1.0/src/citewatch/demo/guard.py +153 -0
- citewatch-0.1.0/src/citewatch/demo/kv.py +212 -0
- citewatch-0.1.0/src/citewatch/demo/live.py +273 -0
- citewatch-0.1.0/src/citewatch/demo/logs.py +42 -0
- citewatch-0.1.0/src/citewatch/demo/lookup.py +157 -0
- citewatch-0.1.0/src/citewatch/demo/make_code_hash.py +49 -0
- citewatch-0.1.0/src/citewatch/demo/markup.py +38 -0
- citewatch-0.1.0/src/citewatch/demo/passkeys.py +386 -0
- citewatch-0.1.0/src/citewatch/demo/settings.py +66 -0
- citewatch-0.1.0/src/citewatch/env.py +48 -0
- citewatch-0.1.0/src/citewatch/examples/uk-health-summary.json.gz +0 -0
- citewatch-0.1.0/src/citewatch/extract/__init__.py +18 -0
- citewatch-0.1.0/src/citewatch/extract/extractors.py +147 -0
- citewatch-0.1.0/src/citewatch/extract/stage.py +189 -0
- citewatch-0.1.0/src/citewatch/extract/urls.py +108 -0
- citewatch-0.1.0/src/citewatch/fetch/__init__.py +28 -0
- citewatch-0.1.0/src/citewatch/fetch/client.py +434 -0
- citewatch-0.1.0/src/citewatch/fetch/history.py +132 -0
- citewatch-0.1.0/src/citewatch/fetch/keys.py +125 -0
- citewatch-0.1.0/src/citewatch/fetch/log.py +56 -0
- citewatch-0.1.0/src/citewatch/fetch/pages.py +266 -0
- citewatch-0.1.0/src/citewatch/fetch/rate.py +99 -0
- citewatch-0.1.0/src/citewatch/fetch/robots.py +140 -0
- citewatch-0.1.0/src/citewatch/handcheck.py +221 -0
- citewatch-0.1.0/src/citewatch/ledger.py +75 -0
- citewatch-0.1.0/src/citewatch/metrics/__init__.py +5 -0
- citewatch-0.1.0/src/citewatch/metrics/compute.py +602 -0
- citewatch-0.1.0/src/citewatch/metrics/data.py +166 -0
- citewatch-0.1.0/src/citewatch/metrics/stats.py +120 -0
- citewatch-0.1.0/src/citewatch/metrics/summary.py +231 -0
- citewatch-0.1.0/src/citewatch/metrics/views.py +44 -0
- citewatch-0.1.0/src/citewatch/prices.yaml +19 -0
- citewatch-0.1.0/src/citewatch/py.typed +0 -0
- citewatch-0.1.0/src/citewatch/questions.py +84 -0
- citewatch-0.1.0/src/citewatch/report/__init__.py +5 -0
- citewatch-0.1.0/src/citewatch/report/dashboard.py +762 -0
- citewatch-0.1.0/src/citewatch/report/post.py +235 -0
- citewatch-0.1.0/src/citewatch/runners/__init__.py +29 -0
- citewatch-0.1.0/src/citewatch/runners/anthropic_runner.py +168 -0
- citewatch-0.1.0/src/citewatch/runners/base.py +144 -0
- citewatch-0.1.0/src/citewatch/runners/openai_runner.py +117 -0
- citewatch-0.1.0/src/citewatch/runners/schedule.py +423 -0
- citewatch-0.1.0/src/citewatch/runs.py +202 -0
- citewatch-0.1.0/src/citewatch/schemas/__init__.py +61 -0
- citewatch-0.1.0/src/citewatch/schemas/_base.py +160 -0
- citewatch-0.1.0/src/citewatch/schemas/_iso.py +257 -0
- citewatch-0.1.0/src/citewatch/schemas/cards.py +120 -0
- citewatch-0.1.0/src/citewatch/schemas/config.py +232 -0
- citewatch-0.1.0/src/citewatch/schemas/pages.py +92 -0
- citewatch-0.1.0/src/citewatch/schemas/study.py +133 -0
- citewatch-0.1.0/src/citewatch/schemas/summary.py +187 -0
- citewatch-0.1.0/src/citewatch/schemas/thresholds.py +165 -0
- citewatch-0.1.0/src/citewatch/store.py +127 -0
- citewatch-0.1.0/src/citewatch/support/__init__.py +5 -0
- citewatch-0.1.0/src/citewatch/support/calibrate.py +83 -0
- citewatch-0.1.0/src/citewatch/support/judge.py +455 -0
- citewatch-0.1.0/src/citewatch/support/report.py +191 -0
- citewatch-0.1.0/src/citewatch/support/sample.py +84 -0
- citewatch-0.1.0/src/citewatch/templates/RUBRIC.md +9 -0
- citewatch-0.1.0/src/citewatch/templates/env.example +3 -0
- citewatch-0.1.0/src/citewatch/templates/questions.jsonl +2 -0
- citewatch-0.1.0/src/citewatch/templates/study.yaml +23 -0
- citewatch-0.1.0/src/citewatch/thresholds.py +129 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
.venv-*/
|
|
3
|
+
__pycache__/
|
|
4
|
+
*.pyc
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.harness/
|
|
11
|
+
.env
|
|
12
|
+
.DS_Store
|
|
13
|
+
|
|
14
|
+
# Claude Code local settings
|
|
15
|
+
.claude/settings.local.json
|
|
16
|
+
.claude/scheduled_tasks.lock
|
|
17
|
+
|
|
18
|
+
# Streamlit local secrets and the demo's local development store
|
|
19
|
+
.streamlit/secrets.toml
|
|
20
|
+
.demo-state/
|
|
21
|
+
|
|
22
|
+
# personal notes, not for the public repo (OWNER D-22)
|
|
23
|
+
Portfolio notes.md
|
|
24
|
+
|
|
25
|
+
# sitemap fetch cache: whole-host lastmod maps, reproducible, and the chosen sitemap dates are kept in page_meta (D-53)
|
|
26
|
+
studies/*/data/raw/sitemap/
|
|
27
|
+
data/raw/sitemap/
|
|
28
|
+
studies/*/results/site/
|
citewatch-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Biplob Chakraborty
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
citewatch-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: citewatch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Measure which websites AI answers with web search cite, and how fresh those pages are.
|
|
5
|
+
Project-URL: Homepage, https://github.com/biplob819/ai-visibility
|
|
6
|
+
Project-URL: Issues, https://github.com/biplob819/ai-visibility/issues
|
|
7
|
+
Author: Biplob Chakraborty
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ai-search,citations,freshness,llm,provenance,research,web-search
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: defusedxml>=0.7
|
|
22
|
+
Requires-Dist: duckdb>=1.1
|
|
23
|
+
Requires-Dist: extruct>=0.17
|
|
24
|
+
Requires-Dist: httpx>=0.28
|
|
25
|
+
Requires-Dist: idna>=3.7
|
|
26
|
+
Requires-Dist: numpy>=1.26
|
|
27
|
+
Requires-Dist: protego>=0.4
|
|
28
|
+
Requires-Dist: pydantic>=2.7
|
|
29
|
+
Requires-Dist: python-dateutil>=2.9
|
|
30
|
+
Requires-Dist: pyyaml>=6
|
|
31
|
+
Requires-Dist: tldextract>=5.1
|
|
32
|
+
Requires-Dist: typer>=0.28
|
|
33
|
+
Provides-Extra: anthropic
|
|
34
|
+
Requires-Dist: anthropic>=1; extra == 'anthropic'
|
|
35
|
+
Provides-Extra: demo
|
|
36
|
+
Requires-Dist: streamlit>=1.67; extra == 'demo'
|
|
37
|
+
Requires-Dist: webauthn>=3; extra == 'demo'
|
|
38
|
+
Provides-Extra: openai
|
|
39
|
+
Requires-Dist: openai>=2; extra == 'openai'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# citewatch
|
|
43
|
+
|
|
44
|
+
**Which websites do AI search answers cite, and how fresh are those pages?**
|
|
45
|
+
|
|
46
|
+
citewatch asks AI models questions with web search on, records every source they cite, visits
|
|
47
|
+
each cited page politely to find when it was last updated, and reports who gets cited, how
|
|
48
|
+
fresh it is and how far to trust each number, with counts and 95% intervals throughout. This
|
|
49
|
+
repository holds the tool and its first study: 100 pre-registered UK health questions asked of
|
|
50
|
+
Claude Sonnet 5.5 and GPT-6.1 Sol, twice.
|
|
51
|
+
|
|
52
|
+
**[Dashboard](https://biplob819.github.io/ai-visibility/)** ·
|
|
53
|
+
**[Post](https://biplob819.github.io/ai-visibility/post/)** ·
|
|
54
|
+
**[Signal spec (PDF)](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf)** ·
|
|
55
|
+
**[Data card](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md)** ·
|
|
56
|
+
**[Pre-registration](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md)** ·
|
|
57
|
+
**[Deviations](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md)** ·
|
|
58
|
+
**[Decisions](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md)**
|
|
59
|
+
|
|
60
|
+
It measures **sourcing, not medical accuracy**, and nothing here is medical advice.
|
|
61
|
+
|
|
62
|
+
[](https://biplob819.github.io/ai-visibility/)
|
|
63
|
+
|
|
64
|
+
## What the first study found
|
|
65
|
+
|
|
66
|
+
| | GPT-6.1 Sol | Claude Sonnet 5.5 |
|
|
67
|
+
|---|---|---|
|
|
68
|
+
| Answers citing nhs.uk (run 1) | 83% (83/100; 95% CI 75%–90%) | 10% (10/100; 95% CI 5.0%–16%) |
|
|
69
|
+
| Citation weight held by its top 10 publishers (run 1) | 78% (n=100; 95% CI 75%–85%) | 24% (n=100; 95% CI 23%–31%) |
|
|
70
|
+
| Time-sensitive answers whose newest dated source is over a year old (both runs) | 43% (49/114; 95% CI 31%–55%) | 12% (14/114; 95% CI 5.8%–20%) |
|
|
71
|
+
|
|
72
|
+
- The two models rarely cite the same publishers for the same question: Jaccard similarity
|
|
73
|
+
10% (n=100; 95% CI 7.4%–13%) per question, run 1.
|
|
74
|
+
- The nhs.uk gap is separated in both runs after a Holm correction.
|
|
75
|
+
- How far to trust it: citewatch dated (or logged a reason for) 89% (745/841; 95% CI 86%–91%)
|
|
76
|
+
of cited URLs, against a 90% target. On held-out pages it was within 7 days of the checked date on
|
|
77
|
+
67% (18/27; 95% CI 48%–81%) (agent-checked, owner-approved). In the support check, the two AI
|
|
78
|
+
judges and the owner disagree, so the support rate is reported, not settled.
|
|
79
|
+
- These findings are directional: one country, two models, 100 questions.
|
|
80
|
+
|
|
81
|
+
[](https://biplob819.github.io/ai-visibility/)
|
|
82
|
+
|
|
83
|
+
## Install
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install citewatch # the core: report, score, dates, lookup
|
|
87
|
+
pip install 'citewatch[anthropic,openai]' # plus the provider SDKs for paid runs
|
|
88
|
+
uvx citewatch report --example # or try it without installing
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
v0.1.0 is not on PyPI yet; until it is, install from the repository with
|
|
92
|
+
`pip install git+https://github.com/biplob819/ai-visibility`. Python 3.11 or newer. See the [quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) (an example report in under
|
|
93
|
+
five minutes, no keys).
|
|
94
|
+
|
|
95
|
+
[](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md)
|
|
96
|
+
|
|
97
|
+
## How it works
|
|
98
|
+
|
|
99
|
+

|
|
100
|
+
|
|
101
|
+
1. **Ask.** Each frozen question goes to each model with its native web search, from one
|
|
102
|
+
location, under a cost cap; responses are stored verbatim (`citewatch ask`).
|
|
103
|
+
2. **Extract.** Citations come only from the APIs' citation fields, with canonical URLs and a
|
|
104
|
+
publisher per URL (`citewatch extract`).
|
|
105
|
+
3. **Fetch and date.** Each cited page is fetched once, obeying robots.txt at one request per
|
|
106
|
+
second per host; page bodies are held in memory and never stored. A date resolver reads
|
|
107
|
+
seven sources (visible text, schema.org, meta tags, microdata and RDFa, the search tool's
|
|
108
|
+
page age, sitemaps, HTTP headers) and never guesses (`citewatch fetch`).
|
|
109
|
+
4. **Score.** Shares, freshness, staleness, overlap and stability with a question-cluster
|
|
110
|
+
bootstrap, Holm-corrected comparisons and Wilson intervals; domain cards for every cited
|
|
111
|
+
publisher, each with an improvement report (`citewatch score`).
|
|
112
|
+
5. **Report.** The dashboard and post are built from `summary.json` (`citewatch report`).
|
|
113
|
+
|
|
114
|
+
Two checks sit under the numbers: a **support check** (do cited pages state the claim? two AI
|
|
115
|
+
judges via [EvalKit](https://github.com/biplob819/evalkit) and the owner's labels) and a
|
|
116
|
+
**held-out date check**. Architecture: [`docs/assets/architecture.svg`](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/architecture.svg).
|
|
117
|
+
|
|
118
|
+
## Rerun it
|
|
119
|
+
|
|
120
|
+
Look at the results with no keys and no spend:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
git clone https://github.com/biplob819/ai-visibility && cd ai-visibility
|
|
124
|
+
uv sync
|
|
125
|
+
uv run citewatch report --example # the bundled study's dashboard
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Run your own study (paid calls need the provider SDKs and your API keys):
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
uv sync --extra anthropic --extra openai
|
|
132
|
+
uv run citewatch init my-study --contact <a page site owners can reach you at>
|
|
133
|
+
# edit my-study/study.yaml and my-study/questions.jsonl, and set the API keys in my-study/.env
|
|
134
|
+
uv run citewatch doctor --study my-study # names any missing key or SDK
|
|
135
|
+
uv run citewatch ask --run 0 --dry-run --study my-study # cost estimate, no calls
|
|
136
|
+
|
|
137
|
+
# 1. A small paid smoke run measures what an answer really costs
|
|
138
|
+
uv run citewatch ask --run 0 --study my-study
|
|
139
|
+
uv run python scripts/measure_cost.py my-study --run 0 # writes my-study/cost_model.yaml
|
|
140
|
+
|
|
141
|
+
# 2. Before the study runs: freeze the questions and the metric definitions
|
|
142
|
+
# write my-study/preregistration.md, record the sha256 of it and of questions.jsonl in a
|
|
143
|
+
# decisions.md, and commit everything (a study run refuses a dirty or non-git folder)
|
|
144
|
+
|
|
145
|
+
# 3. The study runs, then the analysis
|
|
146
|
+
uv run citewatch ask --run 1 --study my-study # run 2 waits at least two hours
|
|
147
|
+
uv run citewatch extract --study my-study
|
|
148
|
+
uv run citewatch fetch --study my-study
|
|
149
|
+
uv run citewatch score --study my-study
|
|
150
|
+
uv run citewatch report --study my-study --title "my topic"
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`citewatch ask --run 1 --dry-run` lists anything the run gate still needs before it will spend.
|
|
154
|
+
The gate and why each check exists are in the Tech Spec (§3 and gates G1, G2).
|
|
155
|
+
|
|
156
|
+
**Cost.** The flagship study's smoke run cost $0.79 and its two study runs (400 answers) cost
|
|
157
|
+
$15.03 in API calls; the dry run estimates yours before any call, and `max_cost_usd` in
|
|
158
|
+
`study.yaml` caps it. Fetching costs nothing but time. Thresholds in `study.yaml` can fail a CI
|
|
159
|
+
job when a metric crosses a bound ([`docs/thresholds.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md)).
|
|
160
|
+
|
|
161
|
+
## Guides
|
|
162
|
+
|
|
163
|
+
| Guide | What it covers |
|
|
164
|
+
|---|---|
|
|
165
|
+
| [Quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) | the bundled study's dashboard in under five minutes, no keys |
|
|
166
|
+
| [Run your own study](https://github.com/biplob819/ai-visibility/blob/main/docs/own-study.md) | settings, the questions format, intents, country, models, the run gate |
|
|
167
|
+
| [How dates are resolved](https://github.com/biplob819/ai-visibility/blob/main/docs/dates.md) | the seven date sources, the rules and their limits |
|
|
168
|
+
| [Controlling cost](https://github.com/biplob819/ai-visibility/blob/main/docs/cost.md) | estimates, the hard cap, resume, what the flagship study cost |
|
|
169
|
+
| [Thresholds in CI](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md) | rules that fail a CI job when a metric crosses a bound |
|
|
170
|
+
| [Responsible use](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md) | politeness, no telemetry, privacy, not for gaming AI answers |
|
|
171
|
+
| [Versioning](https://github.com/biplob819/ai-visibility/blob/main/docs/versioning.md) | semver and the public API |
|
|
172
|
+
|
|
173
|
+
## Responsible use
|
|
174
|
+
|
|
175
|
+
citewatch obeys robots.txt and a fixed politeness floor that no setting can loosen, sends
|
|
176
|
+
nothing anywhere except to the AI providers you configure and the cited sites, and is not a
|
|
177
|
+
tool for gaming AI answers ([`docs/responsible-use.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md)).
|
|
178
|
+
|
|
179
|
+
## Repository map
|
|
180
|
+
|
|
181
|
+
| Path | What it holds |
|
|
182
|
+
|---|---|
|
|
183
|
+
| `src/citewatch/` | the package: runners, extract, fetch, dates, metrics, support, cards, report, demo |
|
|
184
|
+
| `studies/uk-health/` | the flagship study: questions, settings, frozen data, results |
|
|
185
|
+
| `docs/` | pre-registration, deviations, decisions, signal spec, data-flow and architecture diagrams |
|
|
186
|
+
| `demo/` | the Streamlit demo (lookup; live mode behind passkeys) |
|
|
187
|
+
| `specs/`, `docs/graph/` | the build plan: one spec per node and the capability catalog |
|
|
188
|
+
|
|
189
|
+
## Links
|
|
190
|
+
|
|
191
|
+
| | |
|
|
192
|
+
|---|---|
|
|
193
|
+
| Repository | https://github.com/biplob819/ai-visibility |
|
|
194
|
+
| Package | https://pypi.org/project/citewatch/ |
|
|
195
|
+
| Dashboard | https://biplob819.github.io/ai-visibility/ |
|
|
196
|
+
| Post | https://biplob819.github.io/ai-visibility/post/ |
|
|
197
|
+
| Signal spec | [docs/signal-spec.pdf](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf) |
|
|
198
|
+
| Data card | [DATA_CARD.md](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md) |
|
|
199
|
+
| Pre-registration | [docs/preregistration.md](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md) |
|
|
200
|
+
| Deviations | [docs/deviations.md](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md) |
|
|
201
|
+
| Decisions | [docs/decisions.md](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md) |
|
|
202
|
+
| Changelog | [changelog.md](https://github.com/biplob819/ai-visibility/blob/main/changelog.md) |
|
|
203
|
+
| Support judging | [EvalKit](https://github.com/biplob819/evalkit) |
|
|
204
|
+
|
|
205
|
+
`scripts/check_links.py --online` checks every link in the README and docs.
|
|
206
|
+
|
|
207
|
+
## Licence
|
|
208
|
+
|
|
209
|
+
MIT ([`LICENSE`](https://github.com/biplob819/ai-visibility/blob/main/LICENSE)). Contributions: [`CONTRIBUTING.md`](https://github.com/biplob819/ai-visibility/blob/main/CONTRIBUTING.md); security
|
|
210
|
+
reports: [`SECURITY.md`](https://github.com/biplob819/ai-visibility/blob/main/SECURITY.md).
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# citewatch
|
|
2
|
+
|
|
3
|
+
**Which websites do AI search answers cite, and how fresh are those pages?**
|
|
4
|
+
|
|
5
|
+
citewatch asks AI models questions with web search on, records every source they cite, visits
|
|
6
|
+
each cited page politely to find when it was last updated, and reports who gets cited, how
|
|
7
|
+
fresh it is and how far to trust each number, with counts and 95% intervals throughout. This
|
|
8
|
+
repository holds the tool and its first study: 100 pre-registered UK health questions asked of
|
|
9
|
+
Claude Sonnet 5.5 and GPT-6.1 Sol, twice.
|
|
10
|
+
|
|
11
|
+
**[Dashboard](https://biplob819.github.io/ai-visibility/)** ·
|
|
12
|
+
**[Post](https://biplob819.github.io/ai-visibility/post/)** ·
|
|
13
|
+
**[Signal spec (PDF)](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf)** ·
|
|
14
|
+
**[Data card](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md)** ·
|
|
15
|
+
**[Pre-registration](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md)** ·
|
|
16
|
+
**[Deviations](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md)** ·
|
|
17
|
+
**[Decisions](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md)**
|
|
18
|
+
|
|
19
|
+
It measures **sourcing, not medical accuracy**, and nothing here is medical advice.
|
|
20
|
+
|
|
21
|
+
[](https://biplob819.github.io/ai-visibility/)
|
|
22
|
+
|
|
23
|
+
## What the first study found
|
|
24
|
+
|
|
25
|
+
| | GPT-6.1 Sol | Claude Sonnet 5.5 |
|
|
26
|
+
|---|---|---|
|
|
27
|
+
| Answers citing nhs.uk (run 1) | 83% (83/100; 95% CI 75%–90%) | 10% (10/100; 95% CI 5.0%–16%) |
|
|
28
|
+
| Citation weight held by its top 10 publishers (run 1) | 78% (n=100; 95% CI 75%–85%) | 24% (n=100; 95% CI 23%–31%) |
|
|
29
|
+
| Time-sensitive answers whose newest dated source is over a year old (both runs) | 43% (49/114; 95% CI 31%–55%) | 12% (14/114; 95% CI 5.8%–20%) |
|
|
30
|
+
|
|
31
|
+
- The two models rarely cite the same publishers for the same question: Jaccard similarity
|
|
32
|
+
10% (n=100; 95% CI 7.4%–13%) per question, run 1.
|
|
33
|
+
- The nhs.uk gap is separated in both runs after a Holm correction.
|
|
34
|
+
- How far to trust it: citewatch dated (or logged a reason for) 89% (745/841; 95% CI 86%–91%)
|
|
35
|
+
of cited URLs, against a 90% target. On held-out pages it was within 7 days of the checked date on
|
|
36
|
+
67% (18/27; 95% CI 48%–81%) (agent-checked, owner-approved). In the support check, the two AI
|
|
37
|
+
judges and the owner disagree, so the support rate is reported, not settled.
|
|
38
|
+
- These findings are directional: one country, two models, 100 questions.
|
|
39
|
+
|
|
40
|
+
[](https://biplob819.github.io/ai-visibility/)
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install citewatch # the core: report, score, dates, lookup
|
|
46
|
+
pip install 'citewatch[anthropic,openai]' # plus the provider SDKs for paid runs
|
|
47
|
+
uvx citewatch report --example # or try it without installing
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
v0.1.0 is not on PyPI yet; until it is, install from the repository with
|
|
51
|
+
`pip install git+https://github.com/biplob819/ai-visibility`. Python 3.11 or newer. See the [quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) (an example report in under
|
|
52
|
+
five minutes, no keys).
|
|
53
|
+
|
|
54
|
+
[](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md)
|
|
55
|
+
|
|
56
|
+
## How it works
|
|
57
|
+
|
|
58
|
+

|
|
59
|
+
|
|
60
|
+
1. **Ask.** Each frozen question goes to each model with its native web search, from one
|
|
61
|
+
location, under a cost cap; responses are stored verbatim (`citewatch ask`).
|
|
62
|
+
2. **Extract.** Citations come only from the APIs' citation fields, with canonical URLs and a
|
|
63
|
+
publisher per URL (`citewatch extract`).
|
|
64
|
+
3. **Fetch and date.** Each cited page is fetched once, obeying robots.txt at one request per
|
|
65
|
+
second per host; page bodies are held in memory and never stored. A date resolver reads
|
|
66
|
+
seven sources (visible text, schema.org, meta tags, microdata and RDFa, the search tool's
|
|
67
|
+
page age, sitemaps, HTTP headers) and never guesses (`citewatch fetch`).
|
|
68
|
+
4. **Score.** Shares, freshness, staleness, overlap and stability with a question-cluster
|
|
69
|
+
bootstrap, Holm-corrected comparisons and Wilson intervals; domain cards for every cited
|
|
70
|
+
publisher, each with an improvement report (`citewatch score`).
|
|
71
|
+
5. **Report.** The dashboard and post are built from `summary.json` (`citewatch report`).
|
|
72
|
+
|
|
73
|
+
Two checks sit under the numbers: a **support check** (do cited pages state the claim? two AI
|
|
74
|
+
judges via [EvalKit](https://github.com/biplob819/evalkit) and the owner's labels) and a
|
|
75
|
+
**held-out date check**. Architecture: [`docs/assets/architecture.svg`](https://raw.githubusercontent.com/biplob819/ai-visibility/main/docs/assets/architecture.svg).
|
|
76
|
+
|
|
77
|
+
## Rerun it
|
|
78
|
+
|
|
79
|
+
Look at the results with no keys and no spend:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
git clone https://github.com/biplob819/ai-visibility && cd ai-visibility
|
|
83
|
+
uv sync
|
|
84
|
+
uv run citewatch report --example # the bundled study's dashboard
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Run your own study (paid calls need the provider SDKs and your API keys):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
uv sync --extra anthropic --extra openai
|
|
91
|
+
uv run citewatch init my-study --contact <a page site owners can reach you at>
|
|
92
|
+
# edit my-study/study.yaml and my-study/questions.jsonl, and set the API keys in my-study/.env
|
|
93
|
+
uv run citewatch doctor --study my-study # names any missing key or SDK
|
|
94
|
+
uv run citewatch ask --run 0 --dry-run --study my-study # cost estimate, no calls
|
|
95
|
+
|
|
96
|
+
# 1. A small paid smoke run measures what an answer really costs
|
|
97
|
+
uv run citewatch ask --run 0 --study my-study
|
|
98
|
+
uv run python scripts/measure_cost.py my-study --run 0 # writes my-study/cost_model.yaml
|
|
99
|
+
|
|
100
|
+
# 2. Before the study runs: freeze the questions and the metric definitions
|
|
101
|
+
# write my-study/preregistration.md, record the sha256 of it and of questions.jsonl in a
|
|
102
|
+
# decisions.md, and commit everything (a study run refuses a dirty or non-git folder)
|
|
103
|
+
|
|
104
|
+
# 3. The study runs, then the analysis
|
|
105
|
+
uv run citewatch ask --run 1 --study my-study # run 2 waits at least two hours
|
|
106
|
+
uv run citewatch extract --study my-study
|
|
107
|
+
uv run citewatch fetch --study my-study
|
|
108
|
+
uv run citewatch score --study my-study
|
|
109
|
+
uv run citewatch report --study my-study --title "my topic"
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`citewatch ask --run 1 --dry-run` lists anything the run gate still needs before it will spend.
|
|
113
|
+
The gate and why each check exists are in the Tech Spec (§3 and gates G1, G2).
|
|
114
|
+
|
|
115
|
+
**Cost.** The flagship study's smoke run cost $0.79 and its two study runs (400 answers) cost
|
|
116
|
+
$15.03 in API calls; the dry run estimates yours before any call, and `max_cost_usd` in
|
|
117
|
+
`study.yaml` caps it. Fetching costs nothing but time. Thresholds in `study.yaml` can fail a CI
|
|
118
|
+
job when a metric crosses a bound ([`docs/thresholds.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md)).
|
|
119
|
+
|
|
120
|
+
## Guides
|
|
121
|
+
|
|
122
|
+
| Guide | What it covers |
|
|
123
|
+
|---|---|
|
|
124
|
+
| [Quickstart](https://github.com/biplob819/ai-visibility/blob/main/docs/quickstart.md) | the bundled study's dashboard in under five minutes, no keys |
|
|
125
|
+
| [Run your own study](https://github.com/biplob819/ai-visibility/blob/main/docs/own-study.md) | settings, the questions format, intents, country, models, the run gate |
|
|
126
|
+
| [How dates are resolved](https://github.com/biplob819/ai-visibility/blob/main/docs/dates.md) | the seven date sources, the rules and their limits |
|
|
127
|
+
| [Controlling cost](https://github.com/biplob819/ai-visibility/blob/main/docs/cost.md) | estimates, the hard cap, resume, what the flagship study cost |
|
|
128
|
+
| [Thresholds in CI](https://github.com/biplob819/ai-visibility/blob/main/docs/thresholds.md) | rules that fail a CI job when a metric crosses a bound |
|
|
129
|
+
| [Responsible use](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md) | politeness, no telemetry, privacy, not for gaming AI answers |
|
|
130
|
+
| [Versioning](https://github.com/biplob819/ai-visibility/blob/main/docs/versioning.md) | semver and the public API |
|
|
131
|
+
|
|
132
|
+
## Responsible use
|
|
133
|
+
|
|
134
|
+
citewatch obeys robots.txt and a fixed politeness floor that no setting can loosen, sends
|
|
135
|
+
nothing anywhere except to the AI providers you configure and the cited sites, and is not a
|
|
136
|
+
tool for gaming AI answers ([`docs/responsible-use.md`](https://github.com/biplob819/ai-visibility/blob/main/docs/responsible-use.md)).
|
|
137
|
+
|
|
138
|
+
## Repository map
|
|
139
|
+
|
|
140
|
+
| Path | What it holds |
|
|
141
|
+
|---|---|
|
|
142
|
+
| `src/citewatch/` | the package: runners, extract, fetch, dates, metrics, support, cards, report, demo |
|
|
143
|
+
| `studies/uk-health/` | the flagship study: questions, settings, frozen data, results |
|
|
144
|
+
| `docs/` | pre-registration, deviations, decisions, signal spec, data-flow and architecture diagrams |
|
|
145
|
+
| `demo/` | the Streamlit demo (lookup; live mode behind passkeys) |
|
|
146
|
+
| `specs/`, `docs/graph/` | the build plan: one spec per node and the capability catalog |
|
|
147
|
+
|
|
148
|
+
## Links
|
|
149
|
+
|
|
150
|
+
| | |
|
|
151
|
+
|---|---|
|
|
152
|
+
| Repository | https://github.com/biplob819/ai-visibility |
|
|
153
|
+
| Package | https://pypi.org/project/citewatch/ |
|
|
154
|
+
| Dashboard | https://biplob819.github.io/ai-visibility/ |
|
|
155
|
+
| Post | https://biplob819.github.io/ai-visibility/post/ |
|
|
156
|
+
| Signal spec | [docs/signal-spec.pdf](https://github.com/biplob819/ai-visibility/blob/main/docs/signal-spec.pdf) |
|
|
157
|
+
| Data card | [DATA_CARD.md](https://github.com/biplob819/ai-visibility/blob/main/DATA_CARD.md) |
|
|
158
|
+
| Pre-registration | [docs/preregistration.md](https://github.com/biplob819/ai-visibility/blob/main/docs/preregistration.md) |
|
|
159
|
+
| Deviations | [docs/deviations.md](https://github.com/biplob819/ai-visibility/blob/main/docs/deviations.md) |
|
|
160
|
+
| Decisions | [docs/decisions.md](https://github.com/biplob819/ai-visibility/blob/main/docs/decisions.md) |
|
|
161
|
+
| Changelog | [changelog.md](https://github.com/biplob819/ai-visibility/blob/main/changelog.md) |
|
|
162
|
+
| Support judging | [EvalKit](https://github.com/biplob819/evalkit) |
|
|
163
|
+
|
|
164
|
+
`scripts/check_links.py --online` checks every link in the README and docs.
|
|
165
|
+
|
|
166
|
+
## Licence
|
|
167
|
+
|
|
168
|
+
MIT ([`LICENSE`](https://github.com/biplob819/ai-visibility/blob/main/LICENSE)). Contributions: [`CONTRIBUTING.md`](https://github.com/biplob819/ai-visibility/blob/main/CONTRIBUTING.md); security
|
|
169
|
+
reports: [`SECURITY.md`](https://github.com/biplob819/ai-visibility/blob/main/SECURITY.md).
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Format: `[TYPE] [NODE] — summary`. Newest first.
|
|
4
|
+
|
|
5
|
+
- [PREPARED] [B4.5] — v0.1.0, not yet on PyPI (publishes when the owner tags it): the citewatch tool and the flagship UK health study (100 pre-registered questions, Claude Sonnet 5.5 and GPT-6.1 Sol, two runs, $15.03). nhs.uk cited in 83/100 GPT answers vs 10/100 Claude (run 1); stale time-sensitive answers 49/114 vs 14/114; cross-model Jaccard 10%; date coverage 745/841; held-out date check 18/27 (agent-checked, owner-approved). Guides, thresholds for CI, public API under semver
|
|
6
|
+
- [GREEN] [B4.4] — verified over five passes (D-80–D-83); README, deviations record, hardened full-history audit (clean)
|
|
7
|
+
- [FEAT] [B4.4] — public repo: README with the finding, method, rerun steps and cost; deviations record; full-history audit (clean); screenshots; diagram and PRD updated for D-70, D-73
|
|
8
|
+
- [GREEN] [B4.2] — verified over two passes (D-76); post live at /post/, B9 approved (OWNER D-74)
|
|
9
|
+
- [FEAT] [D5.2] — demo: lookup, top ten by topic, passkey-gated live mode (owner's registration code), bring your own keys, spend guard, four-field logs; deploy prep (D-78)
|
|
10
|
+
- [FEAT] [B4.5] — groundwork from lane 2: thresholds and CI gate example (D-77), responsible use, community files, versioning and public API contract
|
|
11
|
+
- [GREEN] [D5.1] — verified over five passes (D-69, D-71, D-72, D-75); 361 domain cards with an improvement report that states only what citewatch observed
|
|
12
|
+
- [GREEN] [B4.3] — verified over two passes; one-page signal spec with a build stamp tying the PDF to its HTML
|
|
13
|
+
- [FEAT] [B4.2] — the write-up at /post/: opens with the finding and scope, three charts from summary.json, links the dashboard and signal spec; B9 review approved (OWNER D-74)
|
|
14
|
+
- [GREEN] [B4.1] — verified over two passes; live on GitHub Pages (OWNER D-70), served bytes equal the local build
|
|
15
|
+
- [FEAT] [B4.3] — one-page signal spec (docs/signal-spec.pdf): fields, buyers, delivery, pricing logic as a unit of value and tiers; every quoted finding from summary.json
|
|
16
|
+
- [FEAT] [B4.1] — findings dashboard: one self-contained HTML page from summary.json (inline SVG, table views, light and dark), `citewatch report` and `report --example`
|
|
17
|
+
- [FEAT] [D5.1] — domain cards for 361 cited publishers with `cited_instead` and an improvement report (OWNER D-65); `lookup()` through the extractor's `publisher_of`; written by `citewatch score`
|
|
18
|
+
- [GREEN] [S3.1] — verified over two passes; $15.03 recomputed from provider usage; run data, raw responses and support records frozen by hash
|
|
19
|
+
- [FEAT] [S3.1] — study runs 1 and 2 ($15.03, 400 answers, 0 errors), spend reconciled exactly, run data frozen by hash (RUN-04)
|
|
20
|
+
- [GREEN] [M3.3] — verified over three passes (D-67, D-68); held-out rule over distinct sampled pages; checker load failures counted, never scored; 67% within 7 days (18/27)
|
|
21
|
+
- [GREEN] [M3.2] — verified over three passes (D-67, D-68); support rates per model and judge, owner labels, EvalKit calibration, gaps; body cache finalised
|
|
22
|
+
- [FEAT] [M3.3] — held-out date check on 55 pages, agent-checked and owner-approved (OWNER D-63, D-64): 67% within 7 days (18/27 judgeable, Wilson 48–81%); label freeze tests
|
|
23
|
+
- [FEAT] [M3.2] — support judging with EvalKit (two judges, OWNER D-60, D-61), owner labels (OWNER D-62), judge calibration with EvalKit's cluster kappa (OWNER D-65), body cache finalised
|
|
24
|
+
- [GREEN] [M3.1] — verified over three passes (D-55, D-56); §6 metrics with the question-cluster bootstrap, Holm and Wilson; `citewatch score` writes summary.json; DuckDB views
|
|
25
|
+
- [GREEN] [A2.4] — paid smoke run 0 ($0.79), measured cost model, parity report, golden replay
|
|
26
|
+
- [GREEN] [N1.4] — 100 owner-approved questions frozen with the pre-registration and data card (OWNER D-47, D-51)
|
|
27
|
+
- [GREEN] [A2.3] — verified over five skeptical passes (D-44, D-45); 21-page real test set confirmed by the owner (OWNER D-46)
|
|
28
|
+
- [FEAT] [A2.3] — date resolver: candidates from the seven §7.3 sources, plausibility rules, precedence, sitemaps from robots.txt, `page_meta.jsonl` from `citewatch fetch`; 21-page real test set with drafted labels awaiting the owner (DAT-04)
|
|
29
|
+
- [GREEN] [A2.2] — verified over nine skeptical passes (L-32, L-33; D-42); robots verdict equals protego's on the path actually sent, wire spacing ≥1 s, no markup stored, audits clean on fuzzed multi-run logs
|
|
30
|
+
- [GREEN] [A2.1] — verified over three skeptical passes (D-43); only current answers extracted, every row field pinned with literal values
|
|
31
|
+
- [FEAT] [A2.1] — citation extraction for both providers from API citation fields only, consulted sources with page_age, `canonical_url` (idempotent, utm and click IDs stripped), `publisher_of` per D-4 offline, `citewatch extract`
|
|
32
|
+
- [GREEN] [N1.2] — verified over six skeptical passes (L-27, L-29 to L-31; D-37 to D-41); money recorded for every paid call under errors, interrupts and signals
|
|
33
|
+
- [FEAT] [A2.2] — polite fetcher: protego robots per origin kept for the study, 1 req/s per host and 8 in flight, hand-followed redirects re-checking robots, retries and page history that keep gate 8's rules across runs, HTML-only capped bodies held in memory, gating, fetch log, `citewatch fetch` with `--dry-run`
|
|
34
|
+
- [GREEN] [N1.3] — verified over four skeptical passes (L-28); CI green on all three systems
|
|
35
|
+
- [FEAT] [N1.2] — Claude and OpenAI runners with native web search, cost model, scheduler, `citewatch ask` (dry run, cap, resume), real fixtures
|
|
36
|
+
- [FEAT] [N1.3] — raw store, run ledger, .env loading and the citewatch CLI (init, doctor; other commands exit 2 naming their node)
|
|
37
|
+
- [GREEN] [N1.1] — verified over two skeptical passes (L-23, L-24); 299 tests
|
|
38
|
+
- [FEAT] [N1.1] — results format v1 schemas, study config and question loader, flagship study.yaml
|
|
39
|
+
- [GREEN] [W0.1] — verified over six skeptical passes (L-14 to L-22, D-26); 153 tests; CI green on Python 3.11–3.13 × Linux, macOS, Windows
|
|
40
|
+
- [FEAT] [W0.1] — repo, uv project and the lean Kova build harness ported from EvalKit (12 gates, politeness and storage gates new)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "citewatch"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Measure which websites AI answers with web search cite, and how fresh those pages are."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Biplob Chakraborty" }]
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"defusedxml>=0.7",
|
|
12
|
+
"duckdb>=1.1",
|
|
13
|
+
"extruct>=0.17",
|
|
14
|
+
"httpx>=0.28",
|
|
15
|
+
"idna>=3.7",
|
|
16
|
+
"numpy>=1.26",
|
|
17
|
+
"protego>=0.4",
|
|
18
|
+
"pydantic>=2.7",
|
|
19
|
+
"python-dateutil>=2.9",
|
|
20
|
+
"pyyaml>=6",
|
|
21
|
+
"tldextract>=5.1",
|
|
22
|
+
"typer>=0.28", # TyperException and the exit-code behaviour cli.main relies on
|
|
23
|
+
]
|
|
24
|
+
keywords = ["llm", "web-search", "citations", "freshness", "provenance", "ai-search", "research"]
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Development Status :: 3 - Alpha",
|
|
27
|
+
"Intended Audience :: Science/Research",
|
|
28
|
+
"Intended Audience :: Developers",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/biplob819/ai-visibility"
|
|
39
|
+
Issues = "https://github.com/biplob819/ai-visibility/issues"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
citewatch = "citewatch.cli:entry"
|
|
43
|
+
|
|
44
|
+
[project.optional-dependencies]
|
|
45
|
+
anthropic = [
|
|
46
|
+
"anthropic>=1",
|
|
47
|
+
]
|
|
48
|
+
openai = [
|
|
49
|
+
"openai>=2",
|
|
50
|
+
]
|
|
51
|
+
# The demo page (D5.2, OWNER D-73): components v2 for the passkey element, py_webauthn to verify.
|
|
52
|
+
demo = [
|
|
53
|
+
"streamlit>=1.67",
|
|
54
|
+
"webauthn>=3",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[dependency-groups]
|
|
58
|
+
dev = [
|
|
59
|
+
"pytest>=8",
|
|
60
|
+
"ruff>=0.6",
|
|
61
|
+
"pyright>=1.1.380",
|
|
62
|
+
"types-pyyaml>=6.0.12",
|
|
63
|
+
"hypothesis>=6.100",
|
|
64
|
+
]
|
|
65
|
+
# Support check (SUP-03, D-11, D-84): EvalKit's faithfulness scorer at the pinned commit the
|
|
66
|
+
# flagship study used. A dependency group, not an extra: PyPI refuses a published dependency on
|
|
67
|
+
# a git URL, so the pin stays in the repository; `uv sync` installs it by default.
|
|
68
|
+
support = [
|
|
69
|
+
"evalkit-harness @ git+https://github.com/biplob819/evalkit@218192132ad601f02976d486d839e712e39bc4e2",
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
[tool.uv]
|
|
73
|
+
default-groups = ["dev", "support"]
|
|
74
|
+
|
|
75
|
+
[build-system]
|
|
76
|
+
requires = ["hatchling>=1.26"]
|
|
77
|
+
build-backend = "hatchling.build"
|
|
78
|
+
|
|
79
|
+
[tool.hatch.build.targets.wheel]
|
|
80
|
+
packages = ["src/citewatch"]
|
|
81
|
+
|
|
82
|
+
[tool.hatch.build.targets.sdist]
|
|
83
|
+
include = ["/src/citewatch", "/README.md", "/LICENSE", "/changelog.md"]
|
|
84
|
+
|
|
85
|
+
[tool.ruff]
|
|
86
|
+
line-length = 100
|
|
87
|
+
target-version = "py311"
|
|
88
|
+
src = ["src", "scripts", "tests"]
|
|
89
|
+
extend-exclude = ["*.md"]
|
|
90
|
+
|
|
91
|
+
[tool.ruff.lint]
|
|
92
|
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
|
|
93
|
+
ignore = ["RUF001", "RUF002", "RUF003"] # allow the › separator used in proof cells
|
|
94
|
+
|
|
95
|
+
[tool.pyright]
|
|
96
|
+
include = ["src", "scripts", "tests", "demo"]
|
|
97
|
+
pythonVersion = "3.11"
|
|
98
|
+
typeCheckingMode = "standard"
|
|
99
|
+
extraPaths = ["scripts"]
|
|
100
|
+
|
|
101
|
+
[tool.pytest.ini_options]
|
|
102
|
+
testpaths = ["tests"]
|
|
103
|
+
pythonpath = ["scripts"]
|
|
104
|
+
markers = [
|
|
105
|
+
"contract: schema, provider-format, CLI and imported-API contract tests",
|
|
106
|
+
"data: frozen-data and JSONL validation tests",
|
|
107
|
+
"golden: end-to-end replay of recorded responses",
|
|
108
|
+
"polite: robots, rate-limit, user-agent and fetch-log rules",
|
|
109
|
+
"safety: secrets, storage and demo-content tests",
|
|
110
|
+
"onboarding: no-key CLI flow",
|
|
111
|
+
]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""citewatch: measure which web pages AI answers with web search cite, and how fresh they are.
|
|
2
|
+
|
|
3
|
+
Public API (semver, see docs/versioning.md): the results format in `citewatch.schemas`, and
|
|
4
|
+
`canonical_url`, `publisher_of`, `UrlError`, `lookup`, `read_cards` and `NotCited`, importable
|
|
5
|
+
from here or from `citewatch.extract.urls` and `citewatch.cards`. They load on first use, so
|
|
6
|
+
`import citewatch` (and the CLI) stays fast.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib import import_module
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
__version__ = version("citewatch")
|
|
17
|
+
except PackageNotFoundError: # running from a source tree that was never installed
|
|
18
|
+
__version__ = "0.0.0"
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from citewatch import schemas
|
|
22
|
+
from citewatch.cards import NotCited, lookup, read_cards
|
|
23
|
+
from citewatch.extract.urls import UrlError, canonical_url, publisher_of
|
|
24
|
+
|
|
25
|
+
_LAZY = {
|
|
26
|
+
"canonical_url": "citewatch.extract.urls",
|
|
27
|
+
"publisher_of": "citewatch.extract.urls",
|
|
28
|
+
"UrlError": "citewatch.extract.urls",
|
|
29
|
+
"lookup": "citewatch.cards",
|
|
30
|
+
"read_cards": "citewatch.cards",
|
|
31
|
+
"NotCited": "citewatch.cards",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"NotCited",
|
|
36
|
+
"UrlError",
|
|
37
|
+
"__version__",
|
|
38
|
+
"canonical_url",
|
|
39
|
+
"lookup",
|
|
40
|
+
"publisher_of",
|
|
41
|
+
"read_cards",
|
|
42
|
+
"schemas",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def __getattr__(name: str) -> Any:
|
|
47
|
+
if name == "schemas":
|
|
48
|
+
return import_module("citewatch.schemas")
|
|
49
|
+
if name in _LAZY:
|
|
50
|
+
value = getattr(import_module(_LAZY[name]), name)
|
|
51
|
+
globals()[name] = value
|
|
52
|
+
return value
|
|
53
|
+
raise AttributeError(f"module 'citewatch' has no attribute {name!r}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def __dir__() -> list[str]:
|
|
57
|
+
return sorted({*globals(), *__all__})
|