citations 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- citations/__init__.py +6 -0
- citations/build.py +345 -0
- citations/cli.py +178 -0
- citations/init.py +101 -0
- citations/link_pdfs.py +103 -0
- citations/lint.py +140 -0
- citations/paths.py +91 -0
- citations/resolve.py +306 -0
- citations/verify.py +177 -0
- citations-0.1.0.dist-info/METADATA +127 -0
- citations-0.1.0.dist-info/RECORD +14 -0
- citations-0.1.0.dist-info/WHEEL +4 -0
- citations-0.1.0.dist-info/entry_points.txt +2 -0
- citations-0.1.0.dist-info/licenses/LICENSE +18 -0
citations/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""A library of quotations checked against the sources they came from.
|
|
2
|
+
|
|
3
|
+
The point is not to find text faster. It is to accumulate quotations that have been verified
|
|
4
|
+
against a pinned artifact, so later work quotes from the library instead of from memory.
|
|
5
|
+
"""
|
|
6
|
+
__version__ = "0.1.0"
|
citations/build.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Build the shared citation database from every paper that cites into it.
|
|
2
|
+
|
|
3
|
+
One record per work, organized by the source rather than by the paper. Each record carries the
|
|
4
|
+
bibliographic facts once -- who wrote it, where it appeared, the DOI or arXiv id needed to
|
|
5
|
+
fetch it, the sha256 of the copy that was read -- and then, under `cited_by`, what each of my
|
|
6
|
+
papers does with it: the citation key that paper uses, and any passages it quotes.
|
|
7
|
+
|
|
8
|
+
Reading it source-first is the point. "Craver 2007: mechanistic-validity cites it as
|
|
9
|
+
craver2007explaining and quotes three passages under nomological validity; mechanistic-views
|
|
10
|
+
cites the same book as craver2007 and quotes a different passage." That is invisible when each
|
|
11
|
+
paper keeps its own bibliography, and it is where the duplicated reading, the divergent keys
|
|
12
|
+
and the contradictory year fields all show up.
|
|
13
|
+
|
|
14
|
+
Works are joined on DOI or arXiv id, never on citation key. Ten works are already cited under
|
|
15
|
+
two different keys across two of these repos, so the key cannot identify anything.
|
|
16
|
+
|
|
17
|
+
python build.py --scan # report what each paper contributes, write nothing
|
|
18
|
+
python build.py # write records/
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import hashlib
|
|
24
|
+
import pathlib
|
|
25
|
+
import re
|
|
26
|
+
import sys
|
|
27
|
+
import unicodedata
|
|
28
|
+
|
|
29
|
+
import yaml
|
|
30
|
+
|
|
31
|
+
from citations.paths import home as _home
|
|
32
|
+
|
|
33
|
+
ROOT = _home()
|
|
34
|
+
RECORDS = ROOT / "records"
|
|
35
|
+
ENRICHMENT = ROOT / "enrichment.yaml" # facts resolved after a .bib was written
|
|
36
|
+
GITHUB = pathlib.Path.home() / "Documents" / "GitHub"
|
|
37
|
+
|
|
38
|
+
# Each paper: where its bibliography lives.
|
|
39
|
+
#
|
|
40
|
+
# The three mechanistic-* papers read from their -NEW repositories, which exist because those
|
|
41
|
+
# were rebuilt clean for submission. The others read from their working repositories directly:
|
|
42
|
+
# they are research repos where the experiments, data and pre-registrations are the substance,
|
|
43
|
+
# and a stripped-down copy of one would be a different artifact, not a tidier version of it.
|
|
44
|
+
PAPERS = {
|
|
45
|
+
"mechanistic-validity": {
|
|
46
|
+
"bib": GITHUB / "mechanistic-validity-NEW2" / "paper" / "references.bib",
|
|
47
|
+
"sources": GITHUB / "mechanistic-validity-NEW2" / "sources",
|
|
48
|
+
"claims": GITHUB / "mechanistic-validity-NEW2" / "claims",
|
|
49
|
+
},
|
|
50
|
+
"mechanistic-reference": {
|
|
51
|
+
"bib": GITHUB / "mechanistic-reference-NEW" / "paper" / "references.bib",
|
|
52
|
+
},
|
|
53
|
+
"epistatic-circuits": {
|
|
54
|
+
"bib": GITHUB / "epistatic-circuits" / "paper" / "references.bib",
|
|
55
|
+
},
|
|
56
|
+
"neural-geometry-reliability": {
|
|
57
|
+
"bib": GITHUB / "neural-geometry-reliability" / "paper" / "references.bib",
|
|
58
|
+
},
|
|
59
|
+
"knockout-epistasis-dynamics": {
|
|
60
|
+
"bib": GITHUB / "knockout-epistasis-dynamics" / "paper" / "refs.bib",
|
|
61
|
+
},
|
|
62
|
+
"msms-subspace-collapse": {
|
|
63
|
+
"bib": GITHUB / "msms-subspace-collapse" / "paper" / "references.bib",
|
|
64
|
+
},
|
|
65
|
+
"mechanistic-views": {
|
|
66
|
+
# v30 replaced the hand-written thebibliography block with a real .bib, so this
|
|
67
|
+
# reads structured fields instead of guessing at positional ones
|
|
68
|
+
"bib": GITHUB / "mechanistic-views-NEW" / "paper" / "references.bib",
|
|
69
|
+
},
|
|
70
|
+
"mechanistic-nosology": {
|
|
71
|
+
"bib": GITHUB / "mechanistic-nosology" / "paper" / "references.bib",
|
|
72
|
+
"claims": GITHUB / "mechanistic-nosology" / "claims",
|
|
73
|
+
},
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
FIELD = re.compile(r"(\w+)\s*=\s*[{\"](.*?)[}\"]\s*,?\s*$", re.S)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# LaTeX accents resolved to the character rather than deleted. Stripping backslashes turns
|
|
80
|
+
# Kram\'{a}r into Kram'ar and J\'anos into J'anos, which is how a bibliography ends up
|
|
81
|
+
# misspelling people's names.
|
|
82
|
+
ACCENT = {"'": "\u0301", "`": "\u0300", '"': "\u0308", "^": "\u0302", "~": "\u0303",
|
|
83
|
+
"=": "\u0304", ".": "\u0307", "u": "\u0306", "v": "\u030c", "H": "\u030b",
|
|
84
|
+
"c": "\u0327", "k": "\u0328", "r": "\u030a"}
|
|
85
|
+
LIGATURE = [(r"\\ss\b", "\u00df"), (r"\\o\b", "\u00f8"), (r"\\O\b", "\u00d8"),
|
|
86
|
+
(r"\\ae\b", "\u00e6"), (r"\\AE\b", "\u00c6"), (r"\\aa\b", "\u00e5"),
|
|
87
|
+
(r"\\AA\b", "\u00c5"), (r"\\l\b", "\u0142"), (r"\\L\b", "\u0141"),
|
|
88
|
+
(r"\\i\b", "i"), (r"\\j\b", "j")]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def clean(s: str) -> str:
|
|
92
|
+
s = re.sub(r"\\emph\{([^}]*)\}", r"\1", s or "")
|
|
93
|
+
for pat, ch in LIGATURE:
|
|
94
|
+
s = re.sub(pat, ch, s)
|
|
95
|
+
# \'{a}, \'a and {\'a} all mean the same character
|
|
96
|
+
for mark, comb in ACCENT.items():
|
|
97
|
+
m = re.escape(mark)
|
|
98
|
+
s = re.sub(rf"\\{m}\s*\{{(\w)\}}", lambda g: g.group(1) + comb, s)
|
|
99
|
+
s = re.sub(rf"\\{m}\s*(\w)", lambda g: g.group(1) + comb, s)
|
|
100
|
+
s = unicodedata.normalize("NFC", s)
|
|
101
|
+
s = re.sub(r"[{}]", "", s).replace("\\&", "&").replace("--", "-")
|
|
102
|
+
s = re.sub(r"\\[a-zA-Z]+", "", s).replace("\\", "")
|
|
103
|
+
# No trailing-punctuation strip here: it would take the period off an initial and turn
|
|
104
|
+
# "Fisher, Ronald A." into "Fisher, Ronald A". Trailing junk from a \bibitem author line is
|
|
105
|
+
# that parser's problem, handled where the sentence structure is still visible.
|
|
106
|
+
return " ".join(s.split())
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
CORPORATE = re.compile(r"\b(Administration|Task Force|Committee|Council|Organization|Organisation|Association|Institute|Society|Collaboration|Consortium|Agency|Commission|Academy|Department|Bureau|Office of)\b", re.I)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def split_authors(raw: str) -> tuple[list[str], bool]:
|
|
113
|
+
"""Author list, plus whether BibTeX's `and others` truncated it.
|
|
114
|
+
|
|
115
|
+
"and others" is BibTeX for et al. Read literally it produces a person named "others",
|
|
116
|
+
which 23 records had. It is a property of the list, not a member of it.
|
|
117
|
+
"""
|
|
118
|
+
raw = (raw or "").strip()
|
|
119
|
+
# A corporate author is a single name that may contain "and" -- splitting
|
|
120
|
+
# "U.S. Food and Drug Administration" on it invents two organizations.
|
|
121
|
+
if CORPORATE.search(raw) and "," not in raw.split(" and ")[0]:
|
|
122
|
+
return [raw], False
|
|
123
|
+
parts = [a.strip() for a in re.split(r"\s+and\s+", raw) if a.strip()]
|
|
124
|
+
truncated = any(a.lower().rstrip(".") == "others" for a in parts)
|
|
125
|
+
parts = [a for a in parts if a.lower().rstrip(".") != "others"]
|
|
126
|
+
return [normalize_initials(a) for a in parts], truncated
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def normalize_initials(name: str) -> str:
|
|
130
|
+
"""Give a bare trailing initial its period: "Glennan, Stuart S" -> "Glennan, Stuart S."
|
|
131
|
+
|
|
132
|
+
The missing period is in the source bibliographies, not introduced here. A single capital
|
|
133
|
+
at the end of a name is an initial in every style that matters.
|
|
134
|
+
"""
|
|
135
|
+
# every bare initial, not just the final one: "Ioannidis, John P A" has two
|
|
136
|
+
return re.sub(r"(?<![A-Za-z.])([A-Z])(?=\s|$)", r"\1.", name.strip())
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def slug_for(rec: dict) -> str:
|
|
140
|
+
"""Stable identity: DOI, else arXiv id, else a hash of normalized title+first author."""
|
|
141
|
+
if rec.get("doi"):
|
|
142
|
+
return "doi-" + re.sub(r"[^a-z0-9]+", "-", rec["doi"].lower()).strip("-")
|
|
143
|
+
if rec.get("arxiv"):
|
|
144
|
+
return "arxiv-" + rec["arxiv"].replace(".", "-")
|
|
145
|
+
base = re.sub(r"[^a-z0-9]+", "", (rec.get("title", "") + (rec.get("authors") or [""])[0]).lower())
|
|
146
|
+
return "t-" + hashlib.sha256(base.encode()).hexdigest()[:16]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def arxiv_of(*texts: str) -> str:
|
|
150
|
+
for t in texts:
|
|
151
|
+
m = re.search(r"arxiv[:\s]*(\d{4}\.\d{4,5})", (t or "").lower())
|
|
152
|
+
if m:
|
|
153
|
+
return m.group(1)
|
|
154
|
+
return ""
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def parse_bib(path: pathlib.Path) -> dict[str, dict]:
|
|
158
|
+
out = {}
|
|
159
|
+
for m in re.finditer(r"@(\w+)\s*\{([^,]+),(.*?)\n\}", path.read_text(errors="ignore"), re.S):
|
|
160
|
+
key, body = m.group(2).strip(), m.group(3)
|
|
161
|
+
f = {}
|
|
162
|
+
for line in re.split(r",\s*\n", body):
|
|
163
|
+
fm = FIELD.search(line.strip())
|
|
164
|
+
if fm:
|
|
165
|
+
f[fm.group(1).lower()] = clean(fm.group(2))
|
|
166
|
+
au, truncated = split_authors(f.get("author", ""))
|
|
167
|
+
out[key] = {
|
|
168
|
+
"title": f.get("title", ""), "authors": au, "et_al": truncated,
|
|
169
|
+
"year": f.get("year", ""),
|
|
170
|
+
"venue": f.get("booktitle") or f.get("journal") or f.get("howpublished", ""),
|
|
171
|
+
"doi": f.get("doi", ""), "url": f.get("url", ""),
|
|
172
|
+
"arxiv": arxiv_of(f.get("note", ""), f.get("url", ""), f.get("journal", "")),
|
|
173
|
+
}
|
|
174
|
+
return out
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def parse_bibitem(path: pathlib.Path) -> dict[str, dict]:
|
|
178
|
+
"""A hand-written thebibliography block. Fields are positional, so this is best-effort."""
|
|
179
|
+
txt = path.read_text(errors="ignore")
|
|
180
|
+
out = {}
|
|
181
|
+
chunks = re.split(r"\\bibitem", txt)[1:]
|
|
182
|
+
for ch in chunks:
|
|
183
|
+
km = re.search(r"\{([^}]+)\}", ch)
|
|
184
|
+
if not km:
|
|
185
|
+
continue
|
|
186
|
+
key = km.group(1).strip()
|
|
187
|
+
rest = ch[km.end():]
|
|
188
|
+
blocks = [clean(b) for b in re.split(r"\\newblock", rest)]
|
|
189
|
+
authors_raw = blocks[0] if blocks else ""
|
|
190
|
+
title = blocks[1] if len(blocks) > 1 else ""
|
|
191
|
+
venue = blocks[2] if len(blocks) > 2 else ""
|
|
192
|
+
ym = re.search(r"\b(19|20)\d{2}\b", venue) or re.search(r"\((\d{4})\)", ch)
|
|
193
|
+
# a \bibitem author line ends the sentence, so the last name carries a period
|
|
194
|
+
# that is punctuation rather than an initial
|
|
195
|
+
# a \bibitem author line ends the sentence, so a final period there is
|
|
196
|
+
# punctuation rather than an initial
|
|
197
|
+
raw = [a.strip() for a in re.split(r",| and ", authors_raw) if a.strip()]
|
|
198
|
+
au = []
|
|
199
|
+
for i, a in enumerate(raw):
|
|
200
|
+
if i and a.endswith(".") and not re.search(r"\b[A-Z]\.$", a):
|
|
201
|
+
a = a.rstrip(".")
|
|
202
|
+
au.append(normalize_initials(a))
|
|
203
|
+
truncated = any(x.lower().rstrip(".") == "others" for x in au)
|
|
204
|
+
au = [x for x in au if x.lower().rstrip(".") != "others"]
|
|
205
|
+
url = re.search(r"\\url\{([^}]*)\}", ch)
|
|
206
|
+
out[key] = {
|
|
207
|
+
"title": title.rstrip("."), "authors": au, "et_al": truncated,
|
|
208
|
+
"year": ym.group(0) if ym else "",
|
|
209
|
+
"venue": venue, "doi": "", "url": url.group(1) if url else "",
|
|
210
|
+
"arxiv": arxiv_of(venue, ch),
|
|
211
|
+
}
|
|
212
|
+
return out
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def contributions() -> dict[str, dict[str, dict]]:
|
|
216
|
+
got = {}
|
|
217
|
+
for name, cfg in PAPERS.items():
|
|
218
|
+
if cfg.get("bib") and cfg["bib"].exists():
|
|
219
|
+
got[name] = parse_bib(cfg["bib"])
|
|
220
|
+
elif cfg.get("bibitem") and cfg["bibitem"].exists():
|
|
221
|
+
got[name] = parse_bibitem(cfg["bibitem"])
|
|
222
|
+
else:
|
|
223
|
+
got[name] = {}
|
|
224
|
+
return got
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def enrich_from_claims(entries: dict[str, dict]) -> None:
|
|
228
|
+
"""Pull the pinned artifact out of each audited claim record.
|
|
229
|
+
|
|
230
|
+
The sixteen audited papers record their source PDF and its sha256 in claims/, which is
|
|
231
|
+
where the quote gate reads it from. Those are the artifacts actually read, so they are the
|
|
232
|
+
ones worth linking.
|
|
233
|
+
"""
|
|
234
|
+
d = PAPERS["mechanistic-validity"].get("claims")
|
|
235
|
+
if not d or not d.exists():
|
|
236
|
+
return
|
|
237
|
+
for p in d.glob("*.yaml"):
|
|
238
|
+
r = yaml.safe_load(p.read_text()) or {}
|
|
239
|
+
s = r.get("source") or {}
|
|
240
|
+
e = entries.get(s.get("citation"))
|
|
241
|
+
if not e:
|
|
242
|
+
continue
|
|
243
|
+
for k in ("local", "sha256", "url"):
|
|
244
|
+
if s.get(k) and not e.get(k):
|
|
245
|
+
e[k] = s[k]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def enrich_from_validity(entries: dict[str, dict]) -> None:
|
|
249
|
+
"""Fold in the url/doi/sha256 already resolved in mechanistic-validity's sources/."""
|
|
250
|
+
d = PAPERS["mechanistic-validity"].get("sources")
|
|
251
|
+
if not d or not d.exists():
|
|
252
|
+
return
|
|
253
|
+
for p in d.glob("*.yaml"):
|
|
254
|
+
r = yaml.safe_load(p.read_text()) or {}
|
|
255
|
+
e = entries.get(r.get("citation"))
|
|
256
|
+
if not e:
|
|
257
|
+
continue
|
|
258
|
+
for k in ("url", "doi", "arxiv", "sha256", "local"):
|
|
259
|
+
if r.get(k) and not e.get(k):
|
|
260
|
+
e[k] = r[k]
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def carry_forward(merged: dict) -> int:
|
|
264
|
+
"""Apply facts resolved after the bibliographies were written.
|
|
265
|
+
|
|
266
|
+
Records are generated, so anything learned later -- a year looked up from Crossref, a DOI
|
|
267
|
+
resolved by hand -- has to live somewhere that regenerating does not touch. An earlier
|
|
268
|
+
version read it back out of the records themselves, which works until someone clears the
|
|
269
|
+
directory before rebuilding, at which point sixteen verified years vanish silently. It
|
|
270
|
+
lives in enrichment.yaml instead, keyed by slug.
|
|
271
|
+
|
|
272
|
+
The bibliography still wins where it has a value; this only fills gaps.
|
|
273
|
+
"""
|
|
274
|
+
if not ENRICHMENT.exists():
|
|
275
|
+
return 0
|
|
276
|
+
overlay = yaml.safe_load(ENRICHMENT.read_text()) or {}
|
|
277
|
+
kept = 0
|
|
278
|
+
for slug, extra in overlay.items():
|
|
279
|
+
rec = merged.get(slug)
|
|
280
|
+
if not rec:
|
|
281
|
+
continue
|
|
282
|
+
for k, v in (extra or {}).items():
|
|
283
|
+
if v and not rec.get(k):
|
|
284
|
+
rec[k] = v
|
|
285
|
+
kept += 1
|
|
286
|
+
return kept
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def main() -> int:
|
|
290
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
291
|
+
ap.add_argument("--scan", action="store_true")
|
|
292
|
+
a = ap.parse_args()
|
|
293
|
+
|
|
294
|
+
got = contributions()
|
|
295
|
+
for name, entries in got.items():
|
|
296
|
+
print(f" {name:<24}{len(entries):>4} entries")
|
|
297
|
+
enrich_from_validity(got.get("mechanistic-validity", {}))
|
|
298
|
+
enrich_from_claims(got.get("mechanistic-validity", {}))
|
|
299
|
+
|
|
300
|
+
merged: dict[str, dict] = {}
|
|
301
|
+
for paper, entries in got.items():
|
|
302
|
+
for key, e in entries.items():
|
|
303
|
+
s = slug_for(e)
|
|
304
|
+
rec = merged.setdefault(s, {
|
|
305
|
+
"slug": s, "title": e["title"], "authors": e["authors"], "year": e["year"],
|
|
306
|
+
"venue": e["venue"], "doi": e.get("doi", ""), "arxiv": e.get("arxiv", ""),
|
|
307
|
+
"et_al": e.get("et_al", False),
|
|
308
|
+
"url": e.get("url", ""), "sha256": e.get("sha256", ""),
|
|
309
|
+
"local": e.get("local", ""), "cited_by": {},
|
|
310
|
+
})
|
|
311
|
+
for k in ("doi", "arxiv", "url", "sha256", "venue", "local"):
|
|
312
|
+
if e.get(k) and not rec.get(k):
|
|
313
|
+
rec[k] = e[k]
|
|
314
|
+
if len(e["authors"]) > len(rec["authors"]):
|
|
315
|
+
rec["authors"] = e["authors"]
|
|
316
|
+
rec["cited_by"][paper] = {"key": key}
|
|
317
|
+
|
|
318
|
+
kept = carry_forward(merged)
|
|
319
|
+
shared = {s: r for s, r in merged.items() if len(r["cited_by"]) > 1}
|
|
320
|
+
divergent = {s: r for s, r in shared.items()
|
|
321
|
+
if len({c["key"] for c in r["cited_by"].values()}) > 1}
|
|
322
|
+
print(f"\n distinct works {len(merged):>4}")
|
|
323
|
+
print(f" cited by 2+ papers {len(shared):>4}")
|
|
324
|
+
print(f" ...under divergent keys {len(divergent):>4}")
|
|
325
|
+
unidentified = sum(1 for r in merged.values() if r["slug"].startswith("t-"))
|
|
326
|
+
print(f" with no DOI or arXiv id {unidentified:>4} (joined on title, less reliable)")
|
|
327
|
+
print(f" fields carried forward {kept:>4} (resolved after the .bib was written)")
|
|
328
|
+
|
|
329
|
+
if divergent:
|
|
330
|
+
print("\n same work, different key:")
|
|
331
|
+
for r in list(divergent.values())[:12]:
|
|
332
|
+
keys = ", ".join(f"{p}={c['key']}" for p, c in r["cited_by"].items())
|
|
333
|
+
print(f" {r['title'][:52]:<54}{keys}")
|
|
334
|
+
|
|
335
|
+
if not a.scan:
|
|
336
|
+
RECORDS.mkdir(parents=True, exist_ok=True)
|
|
337
|
+
for s, r in merged.items():
|
|
338
|
+
(RECORDS / f"{s}.yaml").write_text(
|
|
339
|
+
yaml.safe_dump(r, sort_keys=False, allow_unicode=True, width=100))
|
|
340
|
+
print(f"\n wrote {len(merged)} records")
|
|
341
|
+
return 0
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if __name__ == "__main__":
|
|
345
|
+
sys.exit(main())
|
citations/cli.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""The `citations` command.
|
|
2
|
+
|
|
3
|
+
citations init make a library here
|
|
4
|
+
citations verify do my quotations resolve in the sources I pinned?
|
|
5
|
+
citations resolve backfill missing identifiers
|
|
6
|
+
citations build rebuild records from the papers' bibliographies
|
|
7
|
+
citations lint BibTeX correctness, via papis doctor
|
|
8
|
+
citations link point pdfs/ at wherever the papers keep the artifacts
|
|
9
|
+
citations bib emit a .bib for the works a paper cites
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import collections
|
|
15
|
+
import pathlib
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
import yaml
|
|
19
|
+
|
|
20
|
+
from citations import paths, verify as V
|
|
21
|
+
|
|
22
|
+
RESULTS = ["found", "not found", "unchecked"]
|
|
23
|
+
WARNINGS = {"truncated": "stops mid-word or mid-number — the source continues it",
|
|
24
|
+
"short": "the source may qualify this in the next clause",
|
|
25
|
+
"normalized": "matched after ignoring punctuation and spacing",
|
|
26
|
+
"page": "found, but not on the page recorded"}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _records() -> list[dict]:
|
|
30
|
+
d = paths.records()
|
|
31
|
+
return [yaml.safe_load(p.read_text()) or {} for p in sorted(d.glob("*.yaml"))]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _quotes_from_claims(root: pathlib.Path):
|
|
35
|
+
"""A paper's claims/ holds the extraction: source, sha256 and the quotations taken from it."""
|
|
36
|
+
for p in sorted(root.glob("*.yaml")):
|
|
37
|
+
r = yaml.safe_load(p.read_text()) or {}
|
|
38
|
+
src = r.get("source") or {}
|
|
39
|
+
art = src.get("local")
|
|
40
|
+
# Papers name this block either way. Reading only one spelling makes the command find
|
|
41
|
+
# nothing and report it, which is indistinguishable from a paper that has no quotes yet.
|
|
42
|
+
for cid, ev in (r.get("evidence") or r.get("claims") or {}).items():
|
|
43
|
+
for q in (ev.get("quotes") or []):
|
|
44
|
+
yield p.stem, cid, (q.get("exact") or q.get("text") or ""), art, q.get("page")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cmd_verify(a) -> int:
|
|
48
|
+
rep = V.Report()
|
|
49
|
+
counts: collections.Counter = collections.Counter()
|
|
50
|
+
|
|
51
|
+
if a.claims:
|
|
52
|
+
root = pathlib.Path(a.claims).expanduser().resolve()
|
|
53
|
+
base = root.parent
|
|
54
|
+
for claim, cid, text, art, page in _quotes_from_claims(root):
|
|
55
|
+
if not text:
|
|
56
|
+
continue
|
|
57
|
+
rep.checked += 1
|
|
58
|
+
r = V.check_one(text, (base / art) if art else None, page)
|
|
59
|
+
counts[r.state] += 1
|
|
60
|
+
if r.state != "found" or r.warnings:
|
|
61
|
+
rep.problems.append((f"{claim}:{cid}", text[:58], r))
|
|
62
|
+
rep.counts = dict(counts)
|
|
63
|
+
return _report(rep, counts, a, f"claims {root}")
|
|
64
|
+
|
|
65
|
+
lib, origin = paths.find_with_origin()
|
|
66
|
+
source = f"library {lib}" + (
|
|
67
|
+
" (user-level: no .citations/ in this directory or above it)"
|
|
68
|
+
if origin == "user" else "")
|
|
69
|
+
for rec in _records():
|
|
70
|
+
if a.only and a.only not in (rec.get("cited_by") or {}):
|
|
71
|
+
continue
|
|
72
|
+
art = rec.get("local")
|
|
73
|
+
artifact = (paths.home() / art) if art else None
|
|
74
|
+
for q in rec.get("quotes") or []:
|
|
75
|
+
text = q.get("text") or q.get("exact") or ""
|
|
76
|
+
if not text:
|
|
77
|
+
continue
|
|
78
|
+
rep.checked += 1
|
|
79
|
+
r = V.check_one(text, artifact, q.get("page"))
|
|
80
|
+
counts[r.state] += 1
|
|
81
|
+
if r.state != "found" or r.warnings:
|
|
82
|
+
rep.problems.append((rec["slug"], text[:58], r))
|
|
83
|
+
rep.counts = dict(counts)
|
|
84
|
+
return _report(rep, counts, a, source)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _report(rep, counts, a, source: str = "") -> int:
|
|
88
|
+
# What was checked, before how it went. A clean run against the wrong library reads exactly
|
|
89
|
+
# like a clean run against the right one, and the path is the only thing that separates them.
|
|
90
|
+
if source:
|
|
91
|
+
print(f"{source}\n")
|
|
92
|
+
if rep.checked == 0:
|
|
93
|
+
print("nothing to check.\n")
|
|
94
|
+
print("quotes live in a paper's claims/ directory. point at one:")
|
|
95
|
+
print(" citations verify --claims <path>")
|
|
96
|
+
return 2
|
|
97
|
+
|
|
98
|
+
sources = len({s for s, _, _ in rep.problems}) or "?"
|
|
99
|
+
print(f"{rep.checked:,} quotes\n")
|
|
100
|
+
for s in RESULTS:
|
|
101
|
+
n = counts.get(s, 0)
|
|
102
|
+
if not n and s == "not found":
|
|
103
|
+
print(f" {s:<12}{n:>7}")
|
|
104
|
+
continue
|
|
105
|
+
if not n:
|
|
106
|
+
continue
|
|
107
|
+
why = ""
|
|
108
|
+
if s == "unchecked":
|
|
109
|
+
reasons = collections.Counter(r.detail for _, _, r in rep.problems
|
|
110
|
+
if r.state == "unchecked")
|
|
111
|
+
why = (" " + " · ".join(f"{c:,} {d}" for d, c in reasons.most_common())
|
|
112
|
+
if len(reasons) > 1 else f" {reasons.most_common(1)[0][0]}")
|
|
113
|
+
print(f" {s:<12}{n:>7,}{why}")
|
|
114
|
+
|
|
115
|
+
warns = collections.Counter(w for _, _, r in rep.problems for w in r.warnings)
|
|
116
|
+
if warns:
|
|
117
|
+
print("\nwarnings")
|
|
118
|
+
for w, n in warns.most_common():
|
|
119
|
+
print(f" {n:>7,} {w} — {WARNINGS.get(w, '')}")
|
|
120
|
+
|
|
121
|
+
bad = [(s, q, r) for s, q, r in rep.problems if r.state == "not found"]
|
|
122
|
+
if bad and not a.quiet:
|
|
123
|
+
print()
|
|
124
|
+
for slug, text, r in bad[:20]:
|
|
125
|
+
print(f" not found {slug[:30]:<32}{text[:44]}")
|
|
126
|
+
if len(bad) > 20:
|
|
127
|
+
print(f" ... and {len(bad) - 20} more")
|
|
128
|
+
|
|
129
|
+
print()
|
|
130
|
+
if bad:
|
|
131
|
+
print(f"{len(bad)} not found. read the source before concluding anything.")
|
|
132
|
+
elif counts.get("unchecked"):
|
|
133
|
+
print(f"nothing failed. {counts['unchecked']} unchecked — no measurement was made "
|
|
134
|
+
f"for those.")
|
|
135
|
+
else:
|
|
136
|
+
print("all found.")
|
|
137
|
+
return 0 if rep.ok or not a.strict else 1
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _delegate(module: str, name: str, argv: list[str]) -> int:
|
|
141
|
+
import importlib
|
|
142
|
+
m = importlib.import_module(f"citations.{module}")
|
|
143
|
+
sys.argv = [f"citations {name}"] + argv
|
|
144
|
+
return m.main()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def main() -> int:
|
|
148
|
+
ap = argparse.ArgumentParser(prog="citations", description=__doc__.split("\n")[0])
|
|
149
|
+
sub = ap.add_subparsers(dest="cmd")
|
|
150
|
+
|
|
151
|
+
v = sub.add_parser("verify", help="do my quotations resolve in their pinned sources?")
|
|
152
|
+
v.add_argument("--claims", help="a paper's claims/ directory, where quotations live")
|
|
153
|
+
v.add_argument("--only", help="restrict to records cited by this paper")
|
|
154
|
+
v.add_argument("--strict", action="store_true", help="exit 1 on any failure, for CI")
|
|
155
|
+
v.add_argument("--verbose", action="store_true", help="also list loose matches")
|
|
156
|
+
v.add_argument("--quiet", action="store_true")
|
|
157
|
+
v.set_defaults(fn=cmd_verify)
|
|
158
|
+
|
|
159
|
+
for name, helptext in [("init", "make a library here"),
|
|
160
|
+
("resolve", "backfill missing identifiers"),
|
|
161
|
+
("build", "rebuild records from the papers' bibliographies"),
|
|
162
|
+
("lint", "BibTeX correctness, via papis doctor"),
|
|
163
|
+
("link", "point pdfs/ at the papers' artifacts")]:
|
|
164
|
+
p = sub.add_parser(name, help=helptext, add_help=False)
|
|
165
|
+
p.set_defaults(fn=None, delegate={"link": "link_pdfs"}.get(name, name),
|
|
166
|
+
shown=name)
|
|
167
|
+
|
|
168
|
+
args, rest = ap.parse_known_args()
|
|
169
|
+
if not args.cmd:
|
|
170
|
+
ap.print_help()
|
|
171
|
+
return 0
|
|
172
|
+
if getattr(args, "fn", None):
|
|
173
|
+
return args.fn(args)
|
|
174
|
+
return _delegate(args.delegate, args.shown, rest)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__":
|
|
178
|
+
sys.exit(main())
|
citations/init.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Make a library.
|
|
2
|
+
|
|
3
|
+
Git is handled by detection rather than by asking, because each case has one right answer.
|
|
4
|
+
Inside an existing repository the library is just a directory the parent already tracks;
|
|
5
|
+
nesting a repository inside a repository is never what anyone wants. Standing alone it gets its
|
|
6
|
+
own repository, since the records are YAML precisely so that `git diff` shows what changed, and
|
|
7
|
+
an untracked library throws that away.
|
|
8
|
+
|
|
9
|
+
No remote is configured, and this tool never pushes.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import pathlib
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
from citations import paths
|
|
19
|
+
|
|
20
|
+
GITIGNORE = """\
|
|
21
|
+
# artifacts, not records. Copyright usually forbids redistributing them
|
|
22
|
+
pdfs/
|
|
23
|
+
|
|
24
|
+
# secrets
|
|
25
|
+
.env
|
|
26
|
+
|
|
27
|
+
__pycache__/
|
|
28
|
+
*.pyc
|
|
29
|
+
.DS_Store
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
README = """\
|
|
33
|
+
# Citation library
|
|
34
|
+
|
|
35
|
+
Records checked by [`citations`](https://pypi.org/project/citations/).
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
records/ one file per cited work, keyed by DOI or arXiv id
|
|
39
|
+
enrichment.yaml facts resolved after a bibliography was written
|
|
40
|
+
pdfs/ the artifacts. Not committed
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
This holds verbatim passages from the sources you cite. Publishing it republishes that text.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _in_git_repo(d: pathlib.Path) -> bool:
|
|
48
|
+
r = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"],
|
|
49
|
+
cwd=d, capture_output=True, text=True)
|
|
50
|
+
return r.returncode == 0 and r.stdout.strip() == "true"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def make(target: pathlib.Path, git: bool | None = None) -> tuple[pathlib.Path, str]:
|
|
54
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
55
|
+
(target / "records").mkdir(exist_ok=True)
|
|
56
|
+
if not (target / ".gitignore").exists():
|
|
57
|
+
(target / ".gitignore").write_text(GITIGNORE)
|
|
58
|
+
if not (target / "README.md").exists():
|
|
59
|
+
(target / "README.md").write_text(README)
|
|
60
|
+
|
|
61
|
+
tracked = _in_git_repo(target)
|
|
62
|
+
if git is False or tracked:
|
|
63
|
+
note = ("tracked by the repository above" if tracked
|
|
64
|
+
else "not tracked — run `git init` here to keep a history of changes")
|
|
65
|
+
return target, note
|
|
66
|
+
subprocess.run(["git", "init", "--quiet"], cwd=target, capture_output=True)
|
|
67
|
+
return target, "git initialised, no remote"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def main() -> int:
|
|
71
|
+
ap = argparse.ArgumentParser(prog="citations init",
|
|
72
|
+
description=__doc__.split("\n")[0])
|
|
73
|
+
ap.add_argument("--user", action="store_true",
|
|
74
|
+
help="make the shared library instead of one here")
|
|
75
|
+
ap.add_argument("--path", help="make it at this path")
|
|
76
|
+
ap.add_argument("--no-git", action="store_true", help="do not initialise a repository")
|
|
77
|
+
a = ap.parse_args()
|
|
78
|
+
|
|
79
|
+
if a.path:
|
|
80
|
+
target = pathlib.Path(a.path).expanduser()
|
|
81
|
+
elif a.user:
|
|
82
|
+
target = paths.user_library()
|
|
83
|
+
else:
|
|
84
|
+
target = pathlib.Path.cwd() / paths.DIRNAME
|
|
85
|
+
|
|
86
|
+
if (target / "records").is_dir():
|
|
87
|
+
print(f"already a library: {target}")
|
|
88
|
+
return 0
|
|
89
|
+
|
|
90
|
+
target, note = make(target, git=False if a.no_git else None)
|
|
91
|
+
print(f"created {target}")
|
|
92
|
+
print(f"{note}\n")
|
|
93
|
+
print("This library will hold verbatim passages from the sources you cite.")
|
|
94
|
+
print("Publishing it republishes that text. citations commits here but never pushes.")
|
|
95
|
+
if a.user or a.path:
|
|
96
|
+
print(f"\nUse it from anywhere:\n export CITATIONS_HOME={target}")
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
sys.exit(main())
|