starbuck 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- starbuck-0.2.0/.github/workflows/release.yml +35 -0
- starbuck-0.2.0/.github/workflows/tests.yml +16 -0
- starbuck-0.2.0/.gitignore +9 -0
- starbuck-0.2.0/LICENSE +21 -0
- starbuck-0.2.0/PKG-INFO +277 -0
- starbuck-0.2.0/README.md +254 -0
- starbuck-0.2.0/examples/demo.bib +69 -0
- starbuck-0.2.0/examples/demo.qmd +13 -0
- starbuck-0.2.0/pyproject.toml +49 -0
- starbuck-0.2.0/src/starbuck/__init__.py +4 -0
- starbuck-0.2.0/src/starbuck/bibtex.py +222 -0
- starbuck-0.2.0/src/starbuck/check.py +511 -0
- starbuck-0.2.0/src/starbuck/claims.py +308 -0
- starbuck-0.2.0/src/starbuck/cli.py +174 -0
- starbuck-0.2.0/src/starbuck/evidence.py +219 -0
- starbuck-0.2.0/src/starbuck/ids.py +195 -0
- starbuck-0.2.0/src/starbuck/models.py +81 -0
- starbuck-0.2.0/src/starbuck/parse.py +440 -0
- starbuck-0.2.0/src/starbuck/render.py +66 -0
- starbuck-0.2.0/src/starbuck/report.py +384 -0
- starbuck-0.2.0/src/starbuck/server.py +230 -0
- starbuck-0.2.0/src/starbuck/sources.py +464 -0
- starbuck-0.2.0/tests/conftest.py +23 -0
- starbuck-0.2.0/tests/fake_services.py +236 -0
- starbuck-0.2.0/tests/test_check.py +197 -0
- starbuck-0.2.0/tests/test_claims.py +286 -0
- starbuck-0.2.0/tests/test_cli.py +76 -0
- starbuck-0.2.0/tests/test_ids.py +54 -0
- starbuck-0.2.0/tests/test_judge_external.py +85 -0
- starbuck-0.2.0/tests/test_parse.py +233 -0
- starbuck-0.2.0/tests/test_report.py +98 -0
- starbuck-0.2.0/uv.lock +921 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Publishes to PyPI when the version in pyproject.toml is not on PyPI yet, then tags
|
|
2
|
+
# the commit (vX.Y.Z). Runs on pushes to main that change pyproject.toml, on version
|
|
3
|
+
# tags, and by hand. Needs a trusted publisher on PyPI: owner tiagojct, repository
|
|
4
|
+
# starbuck, workflow release.yml, environment pypi.
|
|
5
|
+
name: release
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
paths: [pyproject.toml]
|
|
10
|
+
tags: ["v*"]
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
jobs:
|
|
13
|
+
pypi:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment: pypi
|
|
16
|
+
permissions:
|
|
17
|
+
contents: write
|
|
18
|
+
id-token: write
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: astral-sh/setup-uv@v6
|
|
22
|
+
- id: version
|
|
23
|
+
run: |
|
|
24
|
+
v=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
25
|
+
echo "version=$v" >> "$GITHUB_OUTPUT"
|
|
26
|
+
if curl -sf "https://pypi.org/pypi/starbuck/$v/json" > /dev/null; then
|
|
27
|
+
echo "published=true" >> "$GITHUB_OUTPUT"
|
|
28
|
+
echo "starbuck $v is already on PyPI; nothing to publish."
|
|
29
|
+
fi
|
|
30
|
+
- if: steps.version.outputs.published != 'true'
|
|
31
|
+
run: uv build && uv publish --trusted-publishing always
|
|
32
|
+
- if: steps.version.outputs.published != 'true'
|
|
33
|
+
run: |
|
|
34
|
+
v=${{ steps.version.outputs.version }}
|
|
35
|
+
git ls-remote --exit-code --tags origin "refs/tags/v$v" > /dev/null || (git tag "v$v" && git push origin "v$v")
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
pull_request:
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
strategy:
|
|
8
|
+
fail-fast: false
|
|
9
|
+
matrix:
|
|
10
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
11
|
+
runs-on: ${{ matrix.os }}
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: astral-sh/setup-uv@v6
|
|
15
|
+
- run: uv sync
|
|
16
|
+
- run: uv run pytest -q
|
starbuck-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tiago Jacinto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
starbuck-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: starbuck
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Reference-integrity checks for manuscripts: do the cited works exist, match their citations, and still stand?
|
|
5
|
+
Project-URL: Repository, https://github.com/tiagojct/starbuck
|
|
6
|
+
Project-URL: Issues, https://github.com/tiagojct/starbuck/issues
|
|
7
|
+
Author: Tiago Jacinto
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: citations,crossref,mcp,pubmed,quarto,references,research integrity,retractions
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Education
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: httpx>=0.27
|
|
20
|
+
Requires-Dist: mcp<3,>=2.2
|
|
21
|
+
Requires-Dist: pyyaml>=6
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# Starbuck
|
|
25
|
+
|
|
26
|
+
Reference-integrity checks for manuscripts. Starbuck reads a Markdown or Quarto
|
|
27
|
+
manuscript with its bibliography and checks each cited reference against public
|
|
28
|
+
scholarly records. It writes a Quarto report that renders to HTML or Word.
|
|
29
|
+
|
|
30
|
+
> *"I will have no man in my boat," said Starbuck, "who is not afraid of a whale."*
|
|
31
|
+
> Herman Melville, *Moby-Dick*, chapter 26
|
|
32
|
+
|
|
33
|
+
## What it checks
|
|
34
|
+
|
|
35
|
+
| Level | Question | Sources |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| 1. Exists | Does the DOI, PMID, arXiv ID or ISBN resolve? Without an identifier, can the work be found? | Crossref, DataCite, doi.org, PubMed, arXiv, Open Library, Crossref search |
|
|
38
|
+
| 2. Matches | Do the cited title, first author and year agree with the record? | The record from level 1 |
|
|
39
|
+
| 3. Stands | Is the work retracted, under an expression of concern or corrected? Is it a notice itself? Was a cited preprint published? | Crossref (with Retraction Watch data), PubMed, OpenAlex, arXiv |
|
|
40
|
+
| 4. Supports (optional) | Does the source support the sentence that cites it? | Europe PMC open-access full text or the abstract (PubMed, Europe PMC, Crossref, OpenAlex); a language model judges |
|
|
41
|
+
|
|
42
|
+
Each reference gets one result:
|
|
43
|
+
|
|
44
|
+
| Result | Meaning |
|
|
45
|
+
|---|---|
|
|
46
|
+
| Fail | The identifier does not exist, belongs to a different work, two identifiers disagree, or the work was retracted. |
|
|
47
|
+
| Check | A person should look: a field differs, an expression of concern, a possible match only, an unreachable web page. |
|
|
48
|
+
| Note | Information that needs no change in most cases: a correction, a published version of a preprint, a DOI found by search. |
|
|
49
|
+
| Pass | Nothing found. |
|
|
50
|
+
| Not checked | A service did not answer. Run the check again later. |
|
|
51
|
+
|
|
52
|
+
The report also lists citations without a bibliography entry, entries that
|
|
53
|
+
the text does not cite, and sentences that state a finding without a citation.
|
|
54
|
+
A sentence is listed when it has at least 50 characters, cites nothing, and
|
|
55
|
+
has a number with a unit or a percentage, an effect measure (OR, HR, RR, CI)
|
|
56
|
+
or a claim word such as found, showed, increased, reduced, associated with,
|
|
57
|
+
risk or prevalence. Headings, tables, figures, captions, code, the reference
|
|
58
|
+
list and sections headed Methods or Results are left out: the authors' own
|
|
59
|
+
methods and data need no citation. The rules are simple and English only, so
|
|
60
|
+
the list is for a person to read; it never makes a reference Fail or Check.
|
|
61
|
+
The JSON keeps the first 50 sentences under `uncited_sentences` (the key
|
|
62
|
+
`uncited` lists the bibliography entries that are not cited).
|
|
63
|
+
|
|
64
|
+
Level 4 adds a verdict for each citing sentence: Supported, Partly supported,
|
|
65
|
+
Not supported or Cannot assess. Level 4 never makes a reference Fail: Not
|
|
66
|
+
supported is Check and Partly supported is Note, so a person decides. See
|
|
67
|
+
[Level 4](#level-4-claim-support).
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
Starbuck needs Python 3.11 or later and [uv](https://docs.astral.sh/uv/).
|
|
72
|
+
For HTML and Word reports, install [Quarto](https://quarto.org). If Quarto is
|
|
73
|
+
not installed, Starbuck uses Pandoc when it is available.
|
|
74
|
+
|
|
75
|
+
1. Install Starbuck:
|
|
76
|
+
|
|
77
|
+
```sh
|
|
78
|
+
uv tool install starbuck
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
2. Set a contact address. The services give faster, more reliable access to
|
|
82
|
+
requests that identify their sender:
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
export STARBUCK_EMAIL=you@example.org
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
3. Check the installation:
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
starbuck --version
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Use
|
|
95
|
+
|
|
96
|
+
Check a manuscript. The bibliography comes from the `bibliography:` field in the
|
|
97
|
+
YAML header:
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
starbuck check paper.qmd
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Starbuck writes the report to a folder named `_starbuck` next to the manuscript.
|
|
104
|
+
Quarto ignores folders that start with an underscore, so the report does not
|
|
105
|
+
become part of a Quarto project.
|
|
106
|
+
|
|
107
|
+
More options:
|
|
108
|
+
|
|
109
|
+
```sh
|
|
110
|
+
starbuck check paper.qmd --to html,docx # HTML and Word
|
|
111
|
+
starbuck check paper.md --bib refs.bib # bibliography not in the YAML header
|
|
112
|
+
starbuck check paper.qmd --all-entries # also check entries that are not cited
|
|
113
|
+
starbuck check paper.qmd --out reports/ # another report folder
|
|
114
|
+
starbuck ids 10.1183/09031936.00080312 arXiv:1706.03762
|
|
115
|
+
starbuck check paper.qmd --claims # also level 4 (see below)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
To check a Word manuscript, convert it to Markdown first:
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
quarto pandoc paper.docx -o paper.md
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### Input
|
|
125
|
+
|
|
126
|
+
Starbuck reads two citation styles:
|
|
127
|
+
|
|
128
|
+
- Citation keys: Pandoc citations such as `[@key]`, `[see @key, p. 3; @other]`
|
|
129
|
+
and `@key` in running text. The bibliography can be BibTeX or BibLaTeX
|
|
130
|
+
(`.bib`), CSL JSON (`.json`) or CSL YAML (`.yaml`), or `references:` in the
|
|
131
|
+
YAML header.
|
|
132
|
+
- Numbered references: `[1]`, `[2, 5]`, `[3-4]` in the text, with a numbered list
|
|
133
|
+
under a heading such as References or Sources. AI research agents often write
|
|
134
|
+
this format.
|
|
135
|
+
|
|
136
|
+
### Output
|
|
137
|
+
|
|
138
|
+
| File | Contents |
|
|
139
|
+
|---|---|
|
|
140
|
+
| `<name>-references.qmd` | The report. Edit it or render it again with Quarto. |
|
|
141
|
+
| `<name>-references.html` | The report as one self-contained web page. |
|
|
142
|
+
| `<name>-references.docx` | The report for Word, with `--to docx`. |
|
|
143
|
+
| `<name>-references.json` | Every result, finding and record, for other tools. |
|
|
144
|
+
| `<name>-claims.json` | Level 4 only: passages, source texts and the model's answers. |
|
|
145
|
+
|
|
146
|
+
The report starts with a summary, then the references that need attention, each
|
|
147
|
+
with the reason, the cited and the recorded metadata side by side, the sentence
|
|
148
|
+
that cites it, and a suggested action. A section at the end states the rules and
|
|
149
|
+
the services used.
|
|
150
|
+
|
|
151
|
+
### Exit codes
|
|
152
|
+
|
|
153
|
+
| Code | Meaning |
|
|
154
|
+
|---|---|
|
|
155
|
+
| 0 | No reference failed. |
|
|
156
|
+
| 1 | At least one reference failed, or a cited key has no entry. |
|
|
157
|
+
| 2 | The check could not run (for example, no citations found). |
|
|
158
|
+
| 3 | No reference failed, but some checks did not run. |
|
|
159
|
+
|
|
160
|
+
## Level 4: claim support
|
|
161
|
+
|
|
162
|
+
For each sentence that cites a source, Starbuck takes the best text of that
|
|
163
|
+
source: a local text file you give, open-access full text from Europe PMC, or
|
|
164
|
+
otherwise the abstract. It ranks the passages of that text against the
|
|
165
|
+
sentence (BM25) and gives the best five to a language model. The model answers
|
|
166
|
+
Supported, Partly supported, Not supported or Cannot assess, and quotes the
|
|
167
|
+
passage it relies on. Starbuck then checks that the quotation appears word for
|
|
168
|
+
word in the source text. A quotation that is not there is rejected, and the
|
|
169
|
+
verdict becomes Cannot assess. The report states for each verdict whether it
|
|
170
|
+
rests on the full text or on the abstract only.
|
|
171
|
+
|
|
172
|
+
The report also gives two scores over the judged citations (Cannot assess is
|
|
173
|
+
left out). Citation recall: the share of citing sentences with at least one
|
|
174
|
+
citation judged Supported or Partly supported. Citation precision: the share
|
|
175
|
+
of citations judged Supported or Partly supported. A citation judged Not
|
|
176
|
+
supported in a sentence that another of its sources supports gets a Note
|
|
177
|
+
(overcitation): it adds no support. The idea comes from ScholarQABench
|
|
178
|
+
(OpenScholar).
|
|
179
|
+
|
|
180
|
+
1. Set the model. Any OpenAI-compatible endpoint works (a hosted provider, or a
|
|
181
|
+
local server such as Ollama):
|
|
182
|
+
|
|
183
|
+
```sh
|
|
184
|
+
export STARBUCK_JUDGE_URL=https://api.example.org/v1
|
|
185
|
+
export STARBUCK_JUDGE_MODEL=model-name
|
|
186
|
+
export STARBUCK_JUDGE_KEY=... # if the endpoint needs a key
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
2. Run the check with `--claims`:
|
|
190
|
+
|
|
191
|
+
```sh
|
|
192
|
+
starbuck check paper.qmd --claims
|
|
193
|
+
starbuck check paper.qmd --claims --text smith2020=smith2020.txt # full text you have
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Without `--claims`, or without the model settings, level 4 does not run and the
|
|
197
|
+
report says so. Level 4 sends the citing sentences and the source passages to
|
|
198
|
+
the model's provider. The file `<name>-claims.json` in `_starbuck` keeps the
|
|
199
|
+
passages, the source texts and the model's answers.
|
|
200
|
+
|
|
201
|
+
A language model can be wrong. Read the source before you change the text.
|
|
202
|
+
|
|
203
|
+
## Settings
|
|
204
|
+
|
|
205
|
+
| Variable | Purpose |
|
|
206
|
+
|---|---|
|
|
207
|
+
| `STARBUCK_EMAIL` | Contact address for the services. `ZOTERO_CONTACT_EMAIL` also works. |
|
|
208
|
+
| `NCBI_API_KEY` | Higher request rate for PubMed. |
|
|
209
|
+
| `OPENALEX_API_KEY` | OpenAlex API key. |
|
|
210
|
+
| `QUARTO_PATH` | Path to Quarto when it is not on the PATH. |
|
|
211
|
+
| `STARBUCK_JUDGE_URL`, `STARBUCK_JUDGE_MODEL`, `STARBUCK_JUDGE_KEY` | Level 4 model (OpenAI-compatible endpoint). |
|
|
212
|
+
|
|
213
|
+
## Use from an agent (MCP)
|
|
214
|
+
|
|
215
|
+
`starbuck-mcp` is an MCP server with four tools:
|
|
216
|
+
|
|
217
|
+
- `check_manuscript`: checks a manuscript and writes the report.
|
|
218
|
+
- `check_references`: checks references given directly (an identifier, citation
|
|
219
|
+
details, or both). An agent can use it to check the sources of a text it wrote.
|
|
220
|
+
- `prepare_claims`: level 4, step 1. Checks the references and returns each
|
|
221
|
+
citing sentence with the best passages of its source. A local full text can
|
|
222
|
+
be given per citation key.
|
|
223
|
+
- `record_claims`: level 4, step 2. Takes the agent's verdicts, checks the
|
|
224
|
+
quoted passages and adds level 4 to the report.
|
|
225
|
+
|
|
226
|
+
Inside an agent, the agent's own model judges, so no `STARBUCK_JUDGE_*`
|
|
227
|
+
settings are needed.
|
|
228
|
+
|
|
229
|
+
In [Sub-Sub](https://github.com/tiagojct/subsub), turn on reference checks in
|
|
230
|
+
`subsub init` or with the switch in the web view; Sub-Sub then runs Starbuck as
|
|
231
|
+
its `verify` server.
|
|
232
|
+
|
|
233
|
+
Example configuration for an MCP client:
|
|
234
|
+
|
|
235
|
+
```json
|
|
236
|
+
{
|
|
237
|
+
"mcpServers": {
|
|
238
|
+
"starbuck": {
|
|
239
|
+
"command": "uvx",
|
|
240
|
+
"args": ["--from", "starbuck", "starbuck-mcp"],
|
|
241
|
+
"env": { "STARBUCK_EMAIL": "you@example.org" }
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Privacy
|
|
248
|
+
|
|
249
|
+
Starbuck sends identifiers and reference details (title, authors, year, or the
|
|
250
|
+
reference as written) to the services listed above, and it opens cited web
|
|
251
|
+
pages. Levels 1 to 3 do not send the manuscript text. Level 4 (only with
|
|
252
|
+
`--claims`) sends the citing sentences and source passages to the model you
|
|
253
|
+
set. Starbuck collects no usage data.
|
|
254
|
+
|
|
255
|
+
## Demo
|
|
256
|
+
|
|
257
|
+
`examples/demo.qmd` cites real works, each set up to show one kind of result:
|
|
258
|
+
a correct reference, a wrong year, a DOI that belongs to another paper, a DOI
|
|
259
|
+
that does not exist, a retracted paper, a reference without a DOI, an arXiv
|
|
260
|
+
preprint and a citation without an entry.
|
|
261
|
+
|
|
262
|
+
```sh
|
|
263
|
+
starbuck check examples/demo.qmd --to html,docx
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
## Development
|
|
267
|
+
|
|
268
|
+
```sh
|
|
269
|
+
uv sync
|
|
270
|
+
uv run pytest -q
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
The tests use canned service responses and do not need the network.
|
|
274
|
+
|
|
275
|
+
## License
|
|
276
|
+
|
|
277
|
+
[MIT](LICENSE) © Tiago Jacinto
|
starbuck-0.2.0/README.md
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# Starbuck
|
|
2
|
+
|
|
3
|
+
Reference-integrity checks for manuscripts. Starbuck reads a Markdown or Quarto
|
|
4
|
+
manuscript with its bibliography and checks each cited reference against public
|
|
5
|
+
scholarly records. It writes a Quarto report that renders to HTML or Word.
|
|
6
|
+
|
|
7
|
+
> *"I will have no man in my boat," said Starbuck, "who is not afraid of a whale."*
|
|
8
|
+
> Herman Melville, *Moby-Dick*, chapter 26
|
|
9
|
+
|
|
10
|
+
## What it checks
|
|
11
|
+
|
|
12
|
+
| Level | Question | Sources |
|
|
13
|
+
|---|---|---|
|
|
14
|
+
| 1. Exists | Does the DOI, PMID, arXiv ID or ISBN resolve? Without an identifier, can the work be found? | Crossref, DataCite, doi.org, PubMed, arXiv, Open Library, Crossref search |
|
|
15
|
+
| 2. Matches | Do the cited title, first author and year agree with the record? | The record from level 1 |
|
|
16
|
+
| 3. Stands | Is the work retracted, under an expression of concern or corrected? Is it a notice itself? Was a cited preprint published? | Crossref (with Retraction Watch data), PubMed, OpenAlex, arXiv |
|
|
17
|
+
| 4. Supports (optional) | Does the source support the sentence that cites it? | Europe PMC open-access full text or the abstract (PubMed, Europe PMC, Crossref, OpenAlex); a language model judges |
|
|
18
|
+
|
|
19
|
+
Each reference gets one result:
|
|
20
|
+
|
|
21
|
+
| Result | Meaning |
|
|
22
|
+
|---|---|
|
|
23
|
+
| Fail | The identifier does not exist, belongs to a different work, two identifiers disagree, or the work was retracted. |
|
|
24
|
+
| Check | A person should look: a field differs, an expression of concern, a possible match only, an unreachable web page. |
|
|
25
|
+
| Note | Information that needs no change in most cases: a correction, a published version of a preprint, a DOI found by search. |
|
|
26
|
+
| Pass | Nothing found. |
|
|
27
|
+
| Not checked | A service did not answer. Run the check again later. |
|
|
28
|
+
|
|
29
|
+
The report also lists citations without a bibliography entry, entries that
|
|
30
|
+
the text does not cite, and sentences that state a finding without a citation.
|
|
31
|
+
A sentence is listed when it has at least 50 characters, cites nothing, and
|
|
32
|
+
has a number with a unit or a percentage, an effect measure (OR, HR, RR, CI)
|
|
33
|
+
or a claim word such as found, showed, increased, reduced, associated with,
|
|
34
|
+
risk or prevalence. Headings, tables, figures, captions, code, the reference
|
|
35
|
+
list and sections headed Methods or Results are left out: the authors' own
|
|
36
|
+
methods and data need no citation. The rules are simple and English only, so
|
|
37
|
+
the list is for a person to read; it never makes a reference Fail or Check.
|
|
38
|
+
The JSON keeps the first 50 sentences under `uncited_sentences` (the key
|
|
39
|
+
`uncited` lists the bibliography entries that are not cited).
|
|
40
|
+
|
|
41
|
+
Level 4 adds a verdict for each citing sentence: Supported, Partly supported,
|
|
42
|
+
Not supported or Cannot assess. Level 4 never makes a reference Fail: Not
|
|
43
|
+
supported is Check and Partly supported is Note, so a person decides. See
|
|
44
|
+
[Level 4](#level-4-claim-support).
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
Starbuck needs Python 3.11 or later and [uv](https://docs.astral.sh/uv/).
|
|
49
|
+
For HTML and Word reports, install [Quarto](https://quarto.org). If Quarto is
|
|
50
|
+
not installed, Starbuck uses Pandoc when it is available.
|
|
51
|
+
|
|
52
|
+
1. Install Starbuck:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
uv tool install starbuck
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
2. Set a contact address. The services give faster, more reliable access to
|
|
59
|
+
requests that identify their sender:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
export STARBUCK_EMAIL=you@example.org
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
3. Check the installation:
|
|
66
|
+
|
|
67
|
+
```sh
|
|
68
|
+
starbuck --version
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Use
|
|
72
|
+
|
|
73
|
+
Check a manuscript. The bibliography comes from the `bibliography:` field in the
|
|
74
|
+
YAML header:
|
|
75
|
+
|
|
76
|
+
```sh
|
|
77
|
+
starbuck check paper.qmd
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Starbuck writes the report to a folder named `_starbuck` next to the manuscript.
|
|
81
|
+
Quarto ignores folders that start with an underscore, so the report does not
|
|
82
|
+
become part of a Quarto project.
|
|
83
|
+
|
|
84
|
+
More options:
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
starbuck check paper.qmd --to html,docx # HTML and Word
|
|
88
|
+
starbuck check paper.md --bib refs.bib # bibliography not in the YAML header
|
|
89
|
+
starbuck check paper.qmd --all-entries # also check entries that are not cited
|
|
90
|
+
starbuck check paper.qmd --out reports/ # another report folder
|
|
91
|
+
starbuck ids 10.1183/09031936.00080312 arXiv:1706.03762
|
|
92
|
+
starbuck check paper.qmd --claims # also level 4 (see below)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
To check a Word manuscript, convert it to Markdown first:
|
|
96
|
+
|
|
97
|
+
```sh
|
|
98
|
+
quarto pandoc paper.docx -o paper.md
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Input
|
|
102
|
+
|
|
103
|
+
Starbuck reads two citation styles:
|
|
104
|
+
|
|
105
|
+
- Citation keys: Pandoc citations such as `[@key]`, `[see @key, p. 3; @other]`
|
|
106
|
+
and `@key` in running text. The bibliography can be BibTeX or BibLaTeX
|
|
107
|
+
(`.bib`), CSL JSON (`.json`) or CSL YAML (`.yaml`), or `references:` in the
|
|
108
|
+
YAML header.
|
|
109
|
+
- Numbered references: `[1]`, `[2, 5]`, `[3-4]` in the text, with a numbered list
|
|
110
|
+
under a heading such as References or Sources. AI research agents often write
|
|
111
|
+
this format.
|
|
112
|
+
|
|
113
|
+
### Output
|
|
114
|
+
|
|
115
|
+
| File | Contents |
|
|
116
|
+
|---|---|
|
|
117
|
+
| `<name>-references.qmd` | The report. Edit it or render it again with Quarto. |
|
|
118
|
+
| `<name>-references.html` | The report as one self-contained web page. |
|
|
119
|
+
| `<name>-references.docx` | The report for Word, with `--to docx`. |
|
|
120
|
+
| `<name>-references.json` | Every result, finding and record, for other tools. |
|
|
121
|
+
| `<name>-claims.json` | Level 4 only: passages, source texts and the model's answers. |
|
|
122
|
+
|
|
123
|
+
The report starts with a summary, then the references that need attention, each
|
|
124
|
+
with the reason, the cited and the recorded metadata side by side, the sentence
|
|
125
|
+
that cites it, and a suggested action. A section at the end states the rules and
|
|
126
|
+
the services used.
|
|
127
|
+
|
|
128
|
+
### Exit codes
|
|
129
|
+
|
|
130
|
+
| Code | Meaning |
|
|
131
|
+
|---|---|
|
|
132
|
+
| 0 | No reference failed. |
|
|
133
|
+
| 1 | At least one reference failed, or a cited key has no entry. |
|
|
134
|
+
| 2 | The check could not run (for example, no citations found). |
|
|
135
|
+
| 3 | No reference failed, but some checks did not run. |
|
|
136
|
+
|
|
137
|
+
## Level 4: claim support
|
|
138
|
+
|
|
139
|
+
For each sentence that cites a source, Starbuck takes the best text of that
|
|
140
|
+
source: a local text file you give, open-access full text from Europe PMC, or
|
|
141
|
+
otherwise the abstract. It ranks the passages of that text against the
|
|
142
|
+
sentence (BM25) and gives the best five to a language model. The model answers
|
|
143
|
+
Supported, Partly supported, Not supported or Cannot assess, and quotes the
|
|
144
|
+
passage it relies on. Starbuck then checks that the quotation appears word for
|
|
145
|
+
word in the source text. A quotation that is not there is rejected, and the
|
|
146
|
+
verdict becomes Cannot assess. The report states for each verdict whether it
|
|
147
|
+
rests on the full text or on the abstract only.
|
|
148
|
+
|
|
149
|
+
The report also gives two scores over the judged citations (Cannot assess is
|
|
150
|
+
left out). Citation recall: the share of citing sentences with at least one
|
|
151
|
+
citation judged Supported or Partly supported. Citation precision: the share
|
|
152
|
+
of citations judged Supported or Partly supported. A citation judged Not
|
|
153
|
+
supported in a sentence that another of its sources supports gets a Note
|
|
154
|
+
(overcitation): it adds no support. The idea comes from ScholarQABench
|
|
155
|
+
(OpenScholar).
|
|
156
|
+
|
|
157
|
+
1. Set the model. Any OpenAI-compatible endpoint works (a hosted provider, or a
|
|
158
|
+
local server such as Ollama):
|
|
159
|
+
|
|
160
|
+
```sh
|
|
161
|
+
export STARBUCK_JUDGE_URL=https://api.example.org/v1
|
|
162
|
+
export STARBUCK_JUDGE_MODEL=model-name
|
|
163
|
+
export STARBUCK_JUDGE_KEY=... # if the endpoint needs a key
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
2. Run the check with `--claims`:
|
|
167
|
+
|
|
168
|
+
```sh
|
|
169
|
+
starbuck check paper.qmd --claims
|
|
170
|
+
starbuck check paper.qmd --claims --text smith2020=smith2020.txt # full text you have
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Without `--claims`, or without the model settings, level 4 does not run and the
|
|
174
|
+
report says so. Level 4 sends the citing sentences and the source passages to
|
|
175
|
+
the model's provider. The file `<name>-claims.json` in `_starbuck` keeps the
|
|
176
|
+
passages, the source texts and the model's answers.
|
|
177
|
+
|
|
178
|
+
A language model can be wrong. Read the source before you change the text.
|
|
179
|
+
|
|
180
|
+
## Settings
|
|
181
|
+
|
|
182
|
+
| Variable | Purpose |
|
|
183
|
+
|---|---|
|
|
184
|
+
| `STARBUCK_EMAIL` | Contact address for the services. `ZOTERO_CONTACT_EMAIL` also works. |
|
|
185
|
+
| `NCBI_API_KEY` | Higher request rate for PubMed. |
|
|
186
|
+
| `OPENALEX_API_KEY` | OpenAlex API key. |
|
|
187
|
+
| `QUARTO_PATH` | Path to Quarto when it is not on the PATH. |
|
|
188
|
+
| `STARBUCK_JUDGE_URL`, `STARBUCK_JUDGE_MODEL`, `STARBUCK_JUDGE_KEY` | Level 4 model (OpenAI-compatible endpoint). |
|
|
189
|
+
|
|
190
|
+
## Use from an agent (MCP)
|
|
191
|
+
|
|
192
|
+
`starbuck-mcp` is an MCP server with four tools:
|
|
193
|
+
|
|
194
|
+
- `check_manuscript`: checks a manuscript and writes the report.
|
|
195
|
+
- `check_references`: checks references given directly (an identifier, citation
|
|
196
|
+
details, or both). An agent can use it to check the sources of a text it wrote.
|
|
197
|
+
- `prepare_claims`: level 4, step 1. Checks the references and returns each
|
|
198
|
+
citing sentence with the best passages of its source. A local full text can
|
|
199
|
+
be given per citation key.
|
|
200
|
+
- `record_claims`: level 4, step 2. Takes the agent's verdicts, checks the
|
|
201
|
+
quoted passages and adds level 4 to the report.
|
|
202
|
+
|
|
203
|
+
Inside an agent, the agent's own model judges, so no `STARBUCK_JUDGE_*`
|
|
204
|
+
settings are needed.
|
|
205
|
+
|
|
206
|
+
In [Sub-Sub](https://github.com/tiagojct/subsub), turn on reference checks in
|
|
207
|
+
`subsub init` or with the switch in the web view; Sub-Sub then runs Starbuck as
|
|
208
|
+
its `verify` server.
|
|
209
|
+
|
|
210
|
+
Example configuration for an MCP client:
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
{
|
|
214
|
+
"mcpServers": {
|
|
215
|
+
"starbuck": {
|
|
216
|
+
"command": "uvx",
|
|
217
|
+
"args": ["--from", "starbuck", "starbuck-mcp"],
|
|
218
|
+
"env": { "STARBUCK_EMAIL": "you@example.org" }
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Privacy
|
|
225
|
+
|
|
226
|
+
Starbuck sends identifiers and reference details (title, authors, year, or the
|
|
227
|
+
reference as written) to the services listed above, and it opens cited web
|
|
228
|
+
pages. Levels 1 to 3 do not send the manuscript text. Level 4 (only with
|
|
229
|
+
`--claims`) sends the citing sentences and source passages to the model you
|
|
230
|
+
set. Starbuck collects no usage data.
|
|
231
|
+
|
|
232
|
+
## Demo
|
|
233
|
+
|
|
234
|
+
`examples/demo.qmd` cites real works, each set up to show one kind of result:
|
|
235
|
+
a correct reference, a wrong year, a DOI that belongs to another paper, a DOI
|
|
236
|
+
that does not exist, a retracted paper, a reference without a DOI, an arXiv
|
|
237
|
+
preprint and a citation without an entry.
|
|
238
|
+
|
|
239
|
+
```sh
|
|
240
|
+
starbuck check examples/demo.qmd --to html,docx
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Development
|
|
244
|
+
|
|
245
|
+
```sh
|
|
246
|
+
uv sync
|
|
247
|
+
uv run pytest -q
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
The tests use canned service responses and do not need the network.
|
|
251
|
+
|
|
252
|
+
## License
|
|
253
|
+
|
|
254
|
+
[MIT](LICENSE) © Tiago Jacinto
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
% Demo bibliography. Each entry is set up to show one kind of result.
|
|
2
|
+
|
|
3
|
+
@article{miller2005,
|
|
4
|
+
title = {Standardisation of spirometry},
|
|
5
|
+
author = {Miller, M. R. and Hankinson, J. and Brusasco, V.},
|
|
6
|
+
journal = {European Respiratory Journal},
|
|
7
|
+
year = {2005},
|
|
8
|
+
doi = {10.1183/09031936.05.00034805}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
@article{quanjer2012,
|
|
12
|
+
title = {Multi-ethnic reference values for spirometry for the 3--95-yr age range: the global lung function 2012 equations},
|
|
13
|
+
author = {Quanjer, Philip H. and Stanojevic, Sanja and Cole, Tim J.},
|
|
14
|
+
journal = {European Respiratory Journal},
|
|
15
|
+
year = {2012},
|
|
16
|
+
doi = {10.1183/09031936.00080312}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
% Expected: Check (the year is wrong)
|
|
20
|
+
@article{graham2019,
|
|
21
|
+
title = {Standardization of Spirometry 2019 Update. An Official American Thoracic Society and European Respiratory Society Technical Statement},
|
|
22
|
+
author = {Graham, Brian L. and Steenbruggen, Irene},
|
|
23
|
+
journal = {American Journal of Respiratory and Critical Care Medicine},
|
|
24
|
+
year = {2017},
|
|
25
|
+
doi = {10.1164/rccm.201908-1590ST}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
% Expected: Fail (the DOI belongs to Miller et al. 2005)
|
|
29
|
+
@article{gli2012wrong,
|
|
30
|
+
title = {Multi-ethnic reference values for spirometry for the 3--95-yr age range: the global lung function 2012 equations},
|
|
31
|
+
author = {Quanjer, Philip H.},
|
|
32
|
+
journal = {European Respiratory Journal},
|
|
33
|
+
year = {2012},
|
|
34
|
+
doi = {10.1183/09031936.05.00034805}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
% Expected: Fail (the DOI does not exist)
|
|
38
|
+
@article{invented2024,
|
|
39
|
+
title = {Deep learning interpretation of spirometry in 40,000 astronauts},
|
|
40
|
+
author = {Nobody, A.},
|
|
41
|
+
journal = {European Respiratory Journal},
|
|
42
|
+
year = {2024},
|
|
43
|
+
doi = {10.1183/09031936.99999999}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
% Expected: Fail (retracted in 2010)
|
|
47
|
+
@article{wakefield1998,
|
|
48
|
+
title = {Ileal-lymphoid-nodular hyperplasia, non-specific colitis, and pervasive developmental disorder in children},
|
|
49
|
+
author = {Wakefield, A. J.},
|
|
50
|
+
journal = {The Lancet},
|
|
51
|
+
year = {1998},
|
|
52
|
+
doi = {10.1016/S0140-6736(97)11096-0}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
% Expected: Note (no DOI given; Crossref search should find it)
|
|
56
|
+
@article{millernodoi,
|
|
57
|
+
title = {Standardisation of spirometry},
|
|
58
|
+
author = {Miller, M. R.},
|
|
59
|
+
journal = {European Respiratory Journal},
|
|
60
|
+
year = {2005}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
@online{vaswani2017,
|
|
64
|
+
title = {Attention Is All You Need},
|
|
65
|
+
author = {Vaswani, Ashish},
|
|
66
|
+
year = {2017},
|
|
67
|
+
eprint = {1706.03762},
|
|
68
|
+
eprinttype = {arxiv}
|
|
69
|
+
}
|