scholarmend 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scholarmend-0.1.0/.github/workflows/ci.yml +29 -0
- scholarmend-0.1.0/.github/workflows/release.yml +39 -0
- scholarmend-0.1.0/.gitignore +15 -0
- scholarmend-0.1.0/CHANGELOG.md +25 -0
- scholarmend-0.1.0/LICENSE +21 -0
- scholarmend-0.1.0/PKG-INFO +186 -0
- scholarmend-0.1.0/README.md +157 -0
- scholarmend-0.1.0/docs/openreview-auth.md +82 -0
- scholarmend-0.1.0/pyproject.toml +72 -0
- scholarmend-0.1.0/scripts/repopulate.py +160 -0
- scholarmend-0.1.0/src/scholarmend/__init__.py +3 -0
- scholarmend-0.1.0/src/scholarmend/_version.py +1 -0
- scholarmend-0.1.0/src/scholarmend/cache.py +80 -0
- scholarmend-0.1.0/src/scholarmend/cli.py +211 -0
- scholarmend-0.1.0/src/scholarmend/emit.py +129 -0
- scholarmend-0.1.0/src/scholarmend/http.py +133 -0
- scholarmend-0.1.0/src/scholarmend/ledger.py +88 -0
- scholarmend-0.1.0/src/scholarmend/miners/__init__.py +29 -0
- scholarmend-0.1.0/src/scholarmend/miners/arxiv.py +31 -0
- scholarmend-0.1.0/src/scholarmend/miners/openreview.py +35 -0
- scholarmend-0.1.0/src/scholarmend/miners/pmc.py +32 -0
- scholarmend-0.1.0/src/scholarmend/miners/pmlr.py +39 -0
- scholarmend-0.1.0/src/scholarmend/miners/proceedings.py +90 -0
- scholarmend-0.1.0/src/scholarmend/models.py +71 -0
- scholarmend-0.1.0/src/scholarmend/parse.py +61 -0
- scholarmend-0.1.0/src/scholarmend/pipeline.py +137 -0
- scholarmend-0.1.0/src/scholarmend/py.typed +0 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/__init__.py +0 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/openreview.py +224 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/pmc.py +43 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/pmlr_index.py +79 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/proceedings_page.py +120 -0
- scholarmend-0.1.0/src/scholarmend/resolvers/semanticscholar.py +102 -0
- scholarmend-0.1.0/tests/conftest.py +27 -0
- scholarmend-0.1.0/tests/fixtures/sample.ris +39 -0
- scholarmend-0.1.0/tests/test_abstracts.py +183 -0
- scholarmend-0.1.0/tests/test_acceptance.py +424 -0
- scholarmend-0.1.0/tests/test_cache.py +248 -0
- scholarmend-0.1.0/tests/test_cli.py +205 -0
- scholarmend-0.1.0/tests/test_emit.py +288 -0
- scholarmend-0.1.0/tests/test_ledger.py +108 -0
- scholarmend-0.1.0/tests/test_miner_proceedings.py +87 -0
- scholarmend-0.1.0/tests/test_miners_other.py +68 -0
- scholarmend-0.1.0/tests/test_parse.py +84 -0
- scholarmend-0.1.0/tests/test_pipeline.py +113 -0
- scholarmend-0.1.0/tests/test_repopulate.py +135 -0
- scholarmend-0.1.0/tests/test_resolver_openreview.py +357 -0
- scholarmend-0.1.0/tests/test_resolver_s2.py +66 -0
- scholarmend-0.1.0/tests/test_resolver_volumes.py +160 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
test:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
- uses: actions/setup-python@v7
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
- run: pip install -e ".[dev]"
|
|
25
|
+
# The suite is offline by design: an autouse fixture fails any real urlopen,
|
|
26
|
+
# and the acceptance tests skip without the sibling Trust-Evals-LitReview corpus.
|
|
27
|
+
- run: pytest -q
|
|
28
|
+
- run: ruff check src tests
|
|
29
|
+
- run: mypy src
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI via Trusted Publishing (OIDC). There is deliberately no API
|
|
4
|
+
# token stored in this repository: PyPI verifies the workflow's identity
|
|
5
|
+
# directly, so there is no long-lived credential to leak or rotate.
|
|
6
|
+
on:
|
|
7
|
+
release:
|
|
8
|
+
types: [published]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v7
|
|
18
|
+
- uses: actions/setup-python@v7
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- run: pip install build
|
|
22
|
+
- run: python -m build
|
|
23
|
+
- uses: actions/upload-artifact@v7
|
|
24
|
+
with:
|
|
25
|
+
name: dist
|
|
26
|
+
path: dist/
|
|
27
|
+
|
|
28
|
+
publish:
|
|
29
|
+
needs: build
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
environment: pypi # add required reviewers here to gate releases
|
|
32
|
+
permissions:
|
|
33
|
+
id-token: write # required for Trusted Publishing
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/download-artifact@v8
|
|
36
|
+
with:
|
|
37
|
+
name: dist
|
|
38
|
+
path: dist/
|
|
39
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
First release on PyPI.
|
|
6
|
+
|
|
7
|
+
- Reads Google Scholar RIS exports and recovers venue, year, and track, mostly
|
|
8
|
+
from the proceedings URL Scholar already put in each record. OpenReview,
|
|
9
|
+
PMLR, PMC, and Semantic Scholar fill in what the URL alone can't.
|
|
10
|
+
- OpenReview forums that were never moved to API v2, such as some 2022-2023
|
|
11
|
+
workshops, are read from API v1 instead.
|
|
12
|
+
- A withdrawn or non-public OpenReview forum is cached as hidden, so a rerun
|
|
13
|
+
doesn't ask again. Delete its cache entry to check whether it has since
|
|
14
|
+
been made public.
|
|
15
|
+
- Writes `resolved.json` (every claim behind each field), `mended.ris` (the
|
|
16
|
+
corrected RIS, ready for Covidence), and `report.txt` (what stayed
|
|
17
|
+
unresolved, and why).
|
|
18
|
+
- Never drops a record and never guesses. A venue it can't determine keeps
|
|
19
|
+
Scholar's value and is marked unresolved for a human to check.
|
|
20
|
+
- `--offline` replays a committed response cache and never reaches the
|
|
21
|
+
network. A record whose lookup is missing from the cache falls back to
|
|
22
|
+
URL-only resolution, and the run exits 1 so you know it was partial.
|
|
23
|
+
- `--abstracts` replaces Scholar's snippet with the full abstract, but only
|
|
24
|
+
when the source shows the record's own title.
|
|
25
|
+
- No runtime dependencies. Python 3.10 to 3.13.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SHARE Lab, University of Waterloo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scholarmend
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Recover true venue, year and track for Google Scholar exports, so screening reasons over facts rather than truncations.
|
|
5
|
+
Project-URL: Homepage, https://github.com/uw-share-lab/scholarmend
|
|
6
|
+
Project-URL: Issues, https://github.com/uw-share-lab/scholarmend/issues
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: bibliometrics,google-scholar,openreview,prisma,ris,systematic-review
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: mypy>=1.5; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# scholarmend
|
|
31
|
+
|
|
32
|
+
Recover true bibliographic metadata for Google Scholar exports, so that
|
|
33
|
+
screening tools reason over facts rather than over Scholar's truncations.
|
|
34
|
+
|
|
35
|
+
## Usage
|
|
36
|
+
|
|
37
|
+
pip install scholarmend
|
|
38
|
+
scholarmend --input path/to/scholar-exports --out out
|
|
39
|
+
|
|
40
|
+
`--input` takes a directory of RIS files exported from Google Scholar, or a
|
|
41
|
+
single `.ris`. For development, clone the repo and `pip install -e ".[dev]"`.
|
|
42
|
+
|
|
43
|
+
Tier 1 needs no configuration and resolves venue, year and track for 77% of a
|
|
44
|
+
Scholar corpus. Tier 2 needs an OpenReview account:
|
|
45
|
+
|
|
46
|
+
export SCHOLARMEND_OPENREVIEW_USER='you@example.edu'
|
|
47
|
+
export SCHOLARMEND_OPENREVIEW_PASSWORD='...'
|
|
48
|
+
|
|
49
|
+
Every response is written to `--cache` (default `.scholarmend-cache`). Commit
|
|
50
|
+
it: a rerun then reproduces byte-identically and makes no API calls.
|
|
51
|
+
|
|
52
|
+
scholarmend --input path/to/scholar-exports --out out --offline
|
|
53
|
+
|
|
54
|
+
`--offline` never reaches the network. A record whose lookup is missing from
|
|
55
|
+
the cache falls back to tier 1, and the run exits 1 to say it was partial.
|
|
56
|
+
|
|
57
|
+
### Outputs
|
|
58
|
+
|
|
59
|
+
| File | Contents |
|
|
60
|
+
|------|----------|
|
|
61
|
+
| `resolved.json` | canonical records: winning value per field, plus every claim behind it |
|
|
62
|
+
| `mended.ris` | the RIS projection, for Covidence and venuetriage |
|
|
63
|
+
| `report.txt` | what stayed unresolved, and why |
|
|
64
|
+
|
|
65
|
+
## The problem
|
|
66
|
+
|
|
67
|
+
Google Scholar's `.ris` export is the standard input to a systematic review, and
|
|
68
|
+
it is lossy in ways that stay invisible until they corrupt a result. Measured on
|
|
69
|
+
a 2,413-record corpus of NeurIPS, ICLR and ICML search results:
|
|
70
|
+
|
|
71
|
+
| Defect | Rate |
|
|
72
|
+
|--------|------|
|
|
73
|
+
| Venue string ellipsized (`… Neural Information …`) | 98% |
|
|
74
|
+
| Author list truncated to five plus `...` | 71% |
|
|
75
|
+
| Records carrying a DOI | 2 of 2,413 |
|
|
76
|
+
| Scholar's `PY` disagrees with the year in the record's own URL | 68% |
|
|
77
|
+
|
|
78
|
+
One record shows the whole problem, and its own answer:
|
|
79
|
+
|
|
80
|
+
JF - … Neural Information …
|
|
81
|
+
AU - ...
|
|
82
|
+
PY - 2026///
|
|
83
|
+
UR - https://proceedings.neurips.cc/paper_files/paper/2025/hash/4da4f3c0…-Abstract-Datasets_and_Benchmarks_Track.html
|
|
84
|
+
|
|
85
|
+
Scholar reports an ellipsis for the venue, the wrong year, and no track. The URL
|
|
86
|
+
in the same record states NeurIPS, 2025, Datasets and Benchmarks Track.
|
|
87
|
+
|
|
88
|
+
## The approach
|
|
89
|
+
|
|
90
|
+
The identifier in this corpus is not a DOI — there are two — it is the **URL
|
|
91
|
+
path**. Offline URL mining yields a key for **99%** of records and fully resolves
|
|
92
|
+
venue, year and track for **77%**, with no network access at all. Records are
|
|
93
|
+
then escalated per field through cached, credentialed lookups only where mining
|
|
94
|
+
leaves a gap.
|
|
95
|
+
|
|
96
|
+
Every claim is retained with its source, tier, confidence and evidence, so the
|
|
97
|
+
losing values stay auditable rather than being overwritten silently.
|
|
98
|
+
|
|
99
|
+
**Abstracts, on request.** Scholar's `AB` is a search snippet on every record —
|
|
100
|
+
fragments joined by `…` around the query terms — and that is what a screening
|
|
101
|
+
tool shows reviewers. `--abstracts` replaces it with the paper's abstract, from
|
|
102
|
+
the proceedings page, the OpenReview submission note, or Semantic Scholar, in
|
|
103
|
+
that order, admitting each only when the source names the record's own title.
|
|
104
|
+
Run it on the deduplicated, triaged upload rather than the whole corpus:
|
|
105
|
+
|
|
106
|
+
scholarmend --input ../Trust-Evals-LitReview/out/clean.ris --out out-covidence --abstracts
|
|
107
|
+
|
|
108
|
+
`report.txt` lists every record whose abstract is still Scholar's snippet.
|
|
109
|
+
|
|
110
|
+
**What it does not recover: authors.** Scholar truncates 71% of author lists,
|
|
111
|
+
and scholarmend leaves them as it found them: rewriting `AU` would restructure a
|
|
112
|
+
repeated field, which the projection never does. See `BACKLOG.md` §5.
|
|
113
|
+
|
|
114
|
+
Querying a structured database instead does not work here, and the spec records
|
|
115
|
+
the measurement: on a 30-title sample, OpenAlex matched 63% of titles and
|
|
116
|
+
returned a correct conference venue for **none** of them, describing the arXiv
|
|
117
|
+
preprint instead. The preprint carries a DOI and the proceedings version does
|
|
118
|
+
not, so a DOI-anchored index indexes the preprint.
|
|
119
|
+
|
|
120
|
+
## Validation
|
|
121
|
+
|
|
122
|
+
scholarmend is tested against hand-verified ground truth rather than against
|
|
123
|
+
its own output. The Trust-Evals-LitReview review produced labels for 112
|
|
124
|
+
records that escaped an automated rule table and were resolved individually,
|
|
125
|
+
with the evidence for each recorded.
|
|
126
|
+
|
|
127
|
+
| Check | Bar |
|
|
128
|
+
|-------|-----|
|
|
129
|
+
| Records with an automated resolution path, of those 112 | at least 103 (measured: 104) |
|
|
130
|
+
| …of which, venue determined **and verified against the reviewers' labels** | **103**, zero disagreements |
|
|
131
|
+
| …the remaining 1 | a sibling proceedings URL, reached but not separately asserted |
|
|
132
|
+
| Workshop status against reviewer labels (the 90 OpenReview records) | zero per-record disagreements |
|
|
133
|
+
| PMLR volumes against the reviewers' `venue_true` | 10, all out of scope, all correctly declined |
|
|
134
|
+
| PMC bridge against the reviewers' override decisions | 3, v267 named ICML 2025, v287 and v297 declined |
|
|
135
|
+
| Verdict flips in `decisions.csv` without a recorded reason | 112 flips, **0** unreasoned |
|
|
136
|
+
| Per-record overrides | 5 of 10 reproduced; the other 5 asserted unreachable |
|
|
137
|
+
| Scholar's year losing every disagreement | all 1,264 |
|
|
138
|
+
| Proceedings mining coverage | exactly 1,854 of 2,413 |
|
|
139
|
+
| Records with no miner at all | exactly 17 |
|
|
140
|
+
| Hand-maintained merge list | retired; 4 collapse at tier 1, 6 at tier 2 |
|
|
141
|
+
|
|
142
|
+
### What "settled" does and does not mean
|
|
143
|
+
|
|
144
|
+
The 104 figure counts records for which the miners extract something a live
|
|
145
|
+
resolver consumes — a venue directly, or an OpenReview forum id, a PMLR volume
|
|
146
|
+
or a PMC id. That is a claim about *reach*, not about correctness, and the two
|
|
147
|
+
halves of it are verified to different depths:
|
|
148
|
+
|
|
149
|
+
- **90 records** reached through an OpenReview `venueid`: venue, year and a
|
|
150
|
+
workshop verdict, every one agreeing with the reviewers' hand-verified label.
|
|
151
|
+
Zero disagreements, compared record by record.
|
|
152
|
+
- **10 PMLR volumes**, all out of scope for this review — the Canadian
|
|
153
|
+
Conference on AI, a workshop at ACML, an AAAI bridge programme, a parsimony
|
|
154
|
+
conference. The resolver retrieves each proceedings title and **deliberately
|
|
155
|
+
declines to name a venue**, rather than coerce an unfamiliar conference onto a
|
|
156
|
+
known one. Verified against the volume the reviewers recorded in `venue_true`.
|
|
157
|
+
- **3 records via the PMC bridge**, reproducing the reviewers' override
|
|
158
|
+
decisions exactly: PMC volume 267 → PMLR v267 → ICML 2025, named; v287 (CHIL)
|
|
159
|
+
and v297 (ML4H) retrieved and declined.
|
|
160
|
+
- **1 record** reached by a sibling proceedings URL, not separately asserted.
|
|
161
|
+
- **8 records** have no automated route at all: NSF landing pages, Google Books
|
|
162
|
+
chapters, an SPIE paper, a PDF on a personal page. These needed human
|
|
163
|
+
judgement before and still do — and the suite asserts they resolve to nothing,
|
|
164
|
+
so the day a miner starts covering one, a test says so.
|
|
165
|
+
|
|
166
|
+
So the honest summary is that scholarmend **determines and verifies** the venue
|
|
167
|
+
for **103** of the 112, reaches 1 more without a separate assertion, and leaves
|
|
168
|
+
8 untouched.
|
|
169
|
+
|
|
170
|
+
Run them with the review repository checked out alongside this one:
|
|
171
|
+
|
|
172
|
+
pytest tests/test_acceptance.py -v
|
|
173
|
+
|
|
174
|
+
They skip cleanly when it is not.
|
|
175
|
+
|
|
176
|
+
## Relationship to other tools
|
|
177
|
+
|
|
178
|
+
- [`venuetriage`](../Trust-Evals-LitReview) — consumes scholarmend's output to
|
|
179
|
+
separate workshop from main-track papers before Covidence.
|
|
180
|
+
- [`refaudit`](https://github.com/uw-share-lab/refaudit) — verifies a finished
|
|
181
|
+
bibliography against Crossref, OpenAlex and arXiv. Different job, different
|
|
182
|
+
input; its DOI-centric resolvers reach only ~8% coverage on this corpus.
|
|
183
|
+
|
|
184
|
+
## Licence
|
|
185
|
+
|
|
186
|
+
MIT.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# scholarmend
|
|
2
|
+
|
|
3
|
+
Recover true bibliographic metadata for Google Scholar exports, so that
|
|
4
|
+
screening tools reason over facts rather than over Scholar's truncations.
|
|
5
|
+
|
|
6
|
+
## Usage
|
|
7
|
+
|
|
8
|
+
pip install scholarmend
|
|
9
|
+
scholarmend --input path/to/scholar-exports --out out
|
|
10
|
+
|
|
11
|
+
`--input` takes a directory of RIS files exported from Google Scholar, or a
|
|
12
|
+
single `.ris`. For development, clone the repo and `pip install -e ".[dev]"`.
|
|
13
|
+
|
|
14
|
+
Tier 1 needs no configuration and resolves venue, year and track for 77% of a
|
|
15
|
+
Scholar corpus. Tier 2 needs an OpenReview account:
|
|
16
|
+
|
|
17
|
+
export SCHOLARMEND_OPENREVIEW_USER='you@example.edu'
|
|
18
|
+
export SCHOLARMEND_OPENREVIEW_PASSWORD='...'
|
|
19
|
+
|
|
20
|
+
Every response is written to `--cache` (default `.scholarmend-cache`). Commit
|
|
21
|
+
it: a rerun then reproduces byte-identically and makes no API calls.
|
|
22
|
+
|
|
23
|
+
scholarmend --input path/to/scholar-exports --out out --offline
|
|
24
|
+
|
|
25
|
+
`--offline` never reaches the network. A record whose lookup is missing from
|
|
26
|
+
the cache falls back to tier 1, and the run exits 1 to say it was partial.
|
|
27
|
+
|
|
28
|
+
### Outputs
|
|
29
|
+
|
|
30
|
+
| File | Contents |
|
|
31
|
+
|------|----------|
|
|
32
|
+
| `resolved.json` | canonical records: winning value per field, plus every claim behind it |
|
|
33
|
+
| `mended.ris` | the RIS projection, for Covidence and venuetriage |
|
|
34
|
+
| `report.txt` | what stayed unresolved, and why |
|
|
35
|
+
|
|
36
|
+
## The problem
|
|
37
|
+
|
|
38
|
+
Google Scholar's `.ris` export is the standard input to a systematic review, and
|
|
39
|
+
it is lossy in ways that stay invisible until they corrupt a result. Measured on
|
|
40
|
+
a 2,413-record corpus of NeurIPS, ICLR and ICML search results:
|
|
41
|
+
|
|
42
|
+
| Defect | Rate |
|
|
43
|
+
|--------|------|
|
|
44
|
+
| Venue string ellipsized (`… Neural Information …`) | 98% |
|
|
45
|
+
| Author list truncated to five plus `...` | 71% |
|
|
46
|
+
| Records carrying a DOI | 2 of 2,413 |
|
|
47
|
+
| Scholar's `PY` disagrees with the year in the record's own URL | 68% |
|
|
48
|
+
|
|
49
|
+
One record shows the whole problem, and its own answer:
|
|
50
|
+
|
|
51
|
+
JF - … Neural Information …
|
|
52
|
+
AU - ...
|
|
53
|
+
PY - 2026///
|
|
54
|
+
UR - https://proceedings.neurips.cc/paper_files/paper/2025/hash/4da4f3c0…-Abstract-Datasets_and_Benchmarks_Track.html
|
|
55
|
+
|
|
56
|
+
Scholar reports an ellipsis for the venue, the wrong year, and no track. The URL
|
|
57
|
+
in the same record states NeurIPS, 2025, Datasets and Benchmarks Track.
|
|
58
|
+
|
|
59
|
+
## The approach
|
|
60
|
+
|
|
61
|
+
The identifier in this corpus is not a DOI — there are two — it is the **URL
|
|
62
|
+
path**. Offline URL mining yields a key for **99%** of records and fully resolves
|
|
63
|
+
venue, year and track for **77%**, with no network access at all. Records are
|
|
64
|
+
then escalated per field through cached, credentialed lookups only where mining
|
|
65
|
+
leaves a gap.
|
|
66
|
+
|
|
67
|
+
Every claim is retained with its source, tier, confidence and evidence, so the
|
|
68
|
+
losing values stay auditable rather than being overwritten silently.
|
|
69
|
+
|
|
70
|
+
**Abstracts, on request.** Scholar's `AB` is a search snippet on every record —
|
|
71
|
+
fragments joined by `…` around the query terms — and that is what a screening
|
|
72
|
+
tool shows reviewers. `--abstracts` replaces it with the paper's abstract, from
|
|
73
|
+
the proceedings page, the OpenReview submission note, or Semantic Scholar, in
|
|
74
|
+
that order, admitting each only when the source names the record's own title.
|
|
75
|
+
Run it on the deduplicated, triaged upload rather than the whole corpus:
|
|
76
|
+
|
|
77
|
+
scholarmend --input ../Trust-Evals-LitReview/out/clean.ris --out out-covidence --abstracts
|
|
78
|
+
|
|
79
|
+
`report.txt` lists every record whose abstract is still Scholar's snippet.
|
|
80
|
+
|
|
81
|
+
**What it does not recover: authors.** Scholar truncates 71% of author lists,
|
|
82
|
+
and scholarmend leaves them as it found them: rewriting `AU` would restructure a
|
|
83
|
+
repeated field, which the projection never does. See `BACKLOG.md` §5.
|
|
84
|
+
|
|
85
|
+
Querying a structured database instead does not work here, and the spec records
|
|
86
|
+
the measurement: on a 30-title sample, OpenAlex matched 63% of titles and
|
|
87
|
+
returned a correct conference venue for **none** of them, describing the arXiv
|
|
88
|
+
preprint instead. The preprint carries a DOI and the proceedings version does
|
|
89
|
+
not, so a DOI-anchored index indexes the preprint.
|
|
90
|
+
|
|
91
|
+
## Validation
|
|
92
|
+
|
|
93
|
+
scholarmend is tested against hand-verified ground truth rather than against
|
|
94
|
+
its own output. The Trust-Evals-LitReview review produced labels for 112
|
|
95
|
+
records that escaped an automated rule table and were resolved individually,
|
|
96
|
+
with the evidence for each recorded.
|
|
97
|
+
|
|
98
|
+
| Check | Bar |
|
|
99
|
+
|-------|-----|
|
|
100
|
+
| Records with an automated resolution path, of those 112 | at least 103 (measured: 104) |
|
|
101
|
+
| …of which, venue determined **and verified against the reviewers' labels** | **103**, zero disagreements |
|
|
102
|
+
| …the remaining 1 | a sibling proceedings URL, reached but not separately asserted |
|
|
103
|
+
| Workshop status against reviewer labels (the 90 OpenReview records) | zero per-record disagreements |
|
|
104
|
+
| PMLR volumes against the reviewers' `venue_true` | 10, all out of scope, all correctly declined |
|
|
105
|
+
| PMC bridge against the reviewers' override decisions | 3, v267 named ICML 2025, v287 and v297 declined |
|
|
106
|
+
| Verdict flips in `decisions.csv` without a recorded reason | 112 flips, **0** unreasoned |
|
|
107
|
+
| Per-record overrides | 5 of 10 reproduced; the other 5 asserted unreachable |
|
|
108
|
+
| Scholar's year losing every disagreement | all 1,264 |
|
|
109
|
+
| Proceedings mining coverage | exactly 1,854 of 2,413 |
|
|
110
|
+
| Records with no miner at all | exactly 17 |
|
|
111
|
+
| Hand-maintained merge list | retired; 4 collapse at tier 1, 6 at tier 2 |
|
|
112
|
+
|
|
113
|
+
### What "settled" does and does not mean
|
|
114
|
+
|
|
115
|
+
The 104 figure counts records for which the miners extract something a live
|
|
116
|
+
resolver consumes — a venue directly, or an OpenReview forum id, a PMLR volume
|
|
117
|
+
or a PMC id. That is a claim about *reach*, not about correctness, and the two
|
|
118
|
+
halves of it are verified to different depths:
|
|
119
|
+
|
|
120
|
+
- **90 records** reached through an OpenReview `venueid`: venue, year and a
|
|
121
|
+
workshop verdict, every one agreeing with the reviewers' hand-verified label.
|
|
122
|
+
Zero disagreements, compared record by record.
|
|
123
|
+
- **10 PMLR volumes**, all out of scope for this review — the Canadian
|
|
124
|
+
Conference on AI, a workshop at ACML, an AAAI bridge programme, a parsimony
|
|
125
|
+
conference. The resolver retrieves each proceedings title and **deliberately
|
|
126
|
+
declines to name a venue**, rather than coerce an unfamiliar conference onto a
|
|
127
|
+
known one. Verified against the volume the reviewers recorded in `venue_true`.
|
|
128
|
+
- **3 records via the PMC bridge**, reproducing the reviewers' override
|
|
129
|
+
decisions exactly: PMC volume 267 → PMLR v267 → ICML 2025, named; v287 (CHIL)
|
|
130
|
+
and v297 (ML4H) retrieved and declined.
|
|
131
|
+
- **1 record** reached by a sibling proceedings URL, not separately asserted.
|
|
132
|
+
- **8 records** have no automated route at all: NSF landing pages, Google Books
|
|
133
|
+
chapters, an SPIE paper, a PDF on a personal page. These needed human
|
|
134
|
+
judgement before and still do — and the suite asserts they resolve to nothing,
|
|
135
|
+
so the day a miner starts covering one, a test says so.
|
|
136
|
+
|
|
137
|
+
So the honest summary is that scholarmend **determines and verifies** the venue
|
|
138
|
+
for **103** of the 112, reaches 1 more without a separate assertion, and leaves
|
|
139
|
+
8 untouched.
|
|
140
|
+
|
|
141
|
+
Run them with the review repository checked out alongside this one:
|
|
142
|
+
|
|
143
|
+
pytest tests/test_acceptance.py -v
|
|
144
|
+
|
|
145
|
+
They skip cleanly when it is not.
|
|
146
|
+
|
|
147
|
+
## Relationship to other tools
|
|
148
|
+
|
|
149
|
+
- [`venuetriage`](../Trust-Evals-LitReview) — consumes scholarmend's output to
|
|
150
|
+
separate workshop from main-track papers before Covidence.
|
|
151
|
+
- [`refaudit`](https://github.com/uw-share-lab/refaudit) — verifies a finished
|
|
152
|
+
bibliography against Crossref, OpenAlex and arXiv. Different job, different
|
|
153
|
+
input; its DOI-centric resolvers reach only ~8% coverage on this corpus.
|
|
154
|
+
|
|
155
|
+
## Licence
|
|
156
|
+
|
|
157
|
+
MIT.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# OpenReview authentication spike
|
|
2
|
+
|
|
3
|
+
**Question:** does an OpenReview login token clear the CAPTCHA/challenge that
|
|
4
|
+
blocks anonymous reads of `api2.openreview.net/notes`?
|
|
5
|
+
|
|
6
|
+
**Answer: yes.**
|
|
7
|
+
|
|
8
|
+
## Steps and results
|
|
9
|
+
|
|
10
|
+
1. **Anonymous baseline** — `GET /notes?forum=0GgFeojE4a&limit=1` with no
|
|
11
|
+
auth header still returns `403`. The anonymous challenge is unchanged.
|
|
12
|
+
|
|
13
|
+
2. **Obtaining a token** — `POST /login` with `{"id": <email>, "password":
|
|
14
|
+
<password>}` (`Content-Type: application/json`) returns a JSON body with
|
|
15
|
+
two top-level keys: `token` and `user`. `token` is a JWT.
|
|
16
|
+
|
|
17
|
+
3. **Using the token** — a request to the endpoint that matters, with header
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
Authorization: Bearer <token>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
returns `200` (not `403`) for forum id `0GgFeojE4a`, which is the same
|
|
24
|
+
forum blocked in step 1. This confirms the token clears the challenge.
|
|
25
|
+
|
|
26
|
+
## JSON path to venueid — confirmed, no correction needed
|
|
27
|
+
|
|
28
|
+
The brief's assumed path is correct:
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
notes[0].content.venueid.value
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
For forum `0GgFeojE4a` this returned `ICML.cc/2026/Workshop/AI4GOOD`, which
|
|
35
|
+
matches the gold value in
|
|
36
|
+
`../Trust-Evals-LitReview/verification/openreview-venues.json` for that
|
|
37
|
+
forum id.
|
|
38
|
+
|
|
39
|
+
Other fields present alongside `venueid` in `content` (for reference, not
|
|
40
|
+
needed by Task 7): `title`, `authors`, `authorids`, `keywords`, `TLDR`,
|
|
41
|
+
`abstract`, `pdf`, `email_sharing`, `data_release`, `venue`, `_bibtex`,
|
|
42
|
+
`paperhash`.
|
|
43
|
+
|
|
44
|
+
## Rate-limit behaviour (10 calls, 1s delay between each)
|
|
45
|
+
|
|
46
|
+
All 10 calls against distinct forum ids returned `200`. Response headers
|
|
47
|
+
exposed both a legacy and a standard rate-limit header set (values agreed
|
|
48
|
+
in every response):
|
|
49
|
+
|
|
50
|
+
- `ratelimit-policy: 500;w=3600` — **500 requests per 3600-second (1 hour)
|
|
51
|
+
rolling window**, per authenticated token/user.
|
|
52
|
+
- `ratelimit-remaining` decremented by exactly 1 per call (497 → 488 across
|
|
53
|
+
the 10 calls), consistent with the stated policy.
|
|
54
|
+
- No throttling, no `429`, and no slowdown in response latency (each call
|
|
55
|
+
completed in ~0.09–0.13s) across the 10-call sample.
|
|
56
|
+
|
|
57
|
+
**Implication for tier 2:** the 527 calls tier 2 needs exceed the 500/hour
|
|
58
|
+
budget by 27 calls. A single unbroken run will hit the limit near the end.
|
|
59
|
+
Tier 2 should either (a) pace calls so the run spans slightly over an hour
|
|
60
|
+
(e.g. resume after the window resets, using `ratelimit-reset`, which the
|
|
61
|
+
response gives as seconds until the window rolls over), or (b) split the
|
|
62
|
+
527 calls into two batches separated by a wait for the window to reset.
|
|
63
|
+
Either is a small addition to the existing 1-second-delay loop in the
|
|
64
|
+
brief's Step 4 script.
|
|
65
|
+
|
|
66
|
+
## Token lifetime
|
|
67
|
+
|
|
68
|
+
The JWT's decoded claims (`user`, `iat`, `exp`, `iss` — decoded locally to
|
|
69
|
+
inspect timing only, never printed in full or committed) show:
|
|
70
|
+
|
|
71
|
+
- **`exp - iat` = 86400 seconds = 24 hours.**
|
|
72
|
+
|
|
73
|
+
A token obtained once is valid for the full 527-call run and does not need
|
|
74
|
+
mid-run refresh, as long as the run completes within 24 hours of login.
|
|
75
|
+
|
|
76
|
+
## Conclusion
|
|
77
|
+
|
|
78
|
+
Authentication via `POST /login` + `Authorization: Bearer <token>` clears
|
|
79
|
+
the challenge that blocks anonymous reads. Task 7 does **not** need the
|
|
80
|
+
one-time authenticated-browser-capture fallback: it can call `/login` once
|
|
81
|
+
per run (or once per 24h), reuse the token for all reads, and must budget
|
|
82
|
+
calls to the 500-requests/hour limit (pace or split runs of >500 calls).
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.20"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[tool.hatch.version]
|
|
6
|
+
path = "src/scholarmend/_version.py"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "scholarmend"
|
|
10
|
+
dynamic = ["version"]
|
|
11
|
+
description = "Recover true venue, year and track for Google Scholar exports, so screening reasons over facts rather than truncations."
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
license = { text = "MIT" }
|
|
15
|
+
keywords = ["systematic-review", "google-scholar", "ris", "openreview", "prisma", "bibliometrics"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
28
|
+
"Topic :: Scientific/Engineering",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
# No runtime dependencies, matching refaudit. Tier-1 mining is pure stdlib
|
|
32
|
+
# string work, and the network tiers use urllib rather than pulling in requests.
|
|
33
|
+
dependencies = []
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4"]
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
scholarmend = "scholarmend.cli:main"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/uw-share-lab/scholarmend"
|
|
43
|
+
Issues = "https://github.com/uw-share-lab/scholarmend/issues"
|
|
44
|
+
|
|
45
|
+
# The committed cache backs the repo's reproducibility claim, but it is 8.8 MB
|
|
46
|
+
# of API responses no installed user needs. Keep it, and the agent tooling, in
|
|
47
|
+
# git only.
|
|
48
|
+
[tool.hatch.build.targets.sdist]
|
|
49
|
+
exclude = [".scholarmend-cache", ".claude", "docs/superpowers", "BACKLOG.md", "CLAUDE.md"]
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.wheel]
|
|
52
|
+
packages = ["src/scholarmend"]
|
|
53
|
+
artifacts = ["src/scholarmend/py.typed"]
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
addopts = "-q"
|
|
58
|
+
markers = [
|
|
59
|
+
# The suite is hermetic: tier-1 miners are pure functions and the network
|
|
60
|
+
# tiers are served from the committed cache. The live check is the deliberate
|
|
61
|
+
# exception and needs real OpenReview credentials.
|
|
62
|
+
"live: talks to the real APIs; needs SCHOLARMEND_OPENREVIEW_USER and _PASSWORD",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[tool.ruff]
|
|
66
|
+
line-length = 100
|
|
67
|
+
target-version = "py310"
|
|
68
|
+
|
|
69
|
+
[tool.mypy]
|
|
70
|
+
python_version = "3.10"
|
|
71
|
+
warn_unused_ignores = true
|
|
72
|
+
disallow_untyped_defs = false
|