paperstack-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- paperstack/__init__.py +1 -0
- paperstack/citations.py +97 -0
- paperstack/cli.py +779 -0
- paperstack/content/__init__.py +1 -0
- paperstack/content/arxiv_pdf.py +103 -0
- paperstack/content/arxiv_source.py +440 -0
- paperstack/content/vendor/latexpand +736 -0
- paperstack/content/vendor/latexpand.LICENSE +31 -0
- paperstack/dblp_index.py +568 -0
- paperstack/entrypoint.py +20 -0
- paperstack/metadata.py +392 -0
- paperstack_cli-0.1.0.dist-info/METADATA +203 -0
- paperstack_cli-0.1.0.dist-info/RECORD +15 -0
- paperstack_cli-0.1.0.dist-info/WHEEL +4 -0
- paperstack_cli-0.1.0.dist-info/entry_points.txt +2 -0
paperstack/metadata.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"""Mechanical paper metadata retrieval with provenance and no source selection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.parse
|
|
11
|
+
import urllib.request
|
|
12
|
+
import xml.etree.ElementTree as ET
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
ARXIV_NS = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
|
|
16
|
+
SOURCES = ("semantic_scholar", "dblp", "crossref", "openreview", "acl_anthology", "arxiv")
|
|
17
|
+
SEARCH_SOURCES = ("s2", "dblp", "crossref", "openreview", "arxiv")
|
|
18
|
+
S2_FIELDS = "paperId,externalIds,venue,title,year,authors,citationCount,influentialCitationCount,referenceCount"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class PaperRef:
|
|
23
|
+
kind: str
|
|
24
|
+
value: str
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def parse(cls, raw: str) -> PaperRef:
|
|
28
|
+
if ":" not in raw:
|
|
29
|
+
raise ValueError("paper reference needs a prefix: arxiv:, doi:, dblp:, or openreview:")
|
|
30
|
+
kind, value = raw.strip().split(":", 1)
|
|
31
|
+
if kind not in ("arxiv", "doi", "dblp", "openreview") or not value:
|
|
32
|
+
raise ValueError("paper reference needs a prefix: arxiv:, doi:, dblp:, or openreview:")
|
|
33
|
+
if kind == "arxiv":
|
|
34
|
+
value = re.sub(r"v\d+$", "", value)
|
|
35
|
+
modern = re.fullmatch(r"\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}", value)
|
|
36
|
+
legacy = re.fullmatch(r"[A-Za-z][A-Za-z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3}", value)
|
|
37
|
+
if not (modern or legacy):
|
|
38
|
+
raise ValueError("invalid arXiv reference")
|
|
39
|
+
return cls(kind, value)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
_last_request: dict[str, float] = {}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def request(
|
|
46
|
+
url: str,
|
|
47
|
+
params: dict | None = None,
|
|
48
|
+
headers: dict | None = None,
|
|
49
|
+
data: bytes | None = None,
|
|
50
|
+
) -> bytes:
|
|
51
|
+
if params:
|
|
52
|
+
url += "?" + urllib.parse.urlencode(params)
|
|
53
|
+
host = urllib.parse.urlparse(url).netloc
|
|
54
|
+
interval = 3.0 if "arxiv.org" in host else 1.1 if "dblp.org" in host else 0.5
|
|
55
|
+
request_headers = {
|
|
56
|
+
"User-Agent": "paperstack (+https://github.com/MilkClouds/my-paperstack)",
|
|
57
|
+
**(headers or {}),
|
|
58
|
+
}
|
|
59
|
+
req = urllib.request.Request(url, headers=request_headers, data=data)
|
|
60
|
+
for attempt in range(3):
|
|
61
|
+
elapsed = time.monotonic() - _last_request.get(host, 0.0)
|
|
62
|
+
if elapsed < interval:
|
|
63
|
+
time.sleep(interval - elapsed)
|
|
64
|
+
try:
|
|
65
|
+
with urllib.request.urlopen(req, timeout=30) as response:
|
|
66
|
+
_last_request[host] = time.monotonic()
|
|
67
|
+
return response.read()
|
|
68
|
+
except urllib.error.HTTPError as exc:
|
|
69
|
+
_last_request[host] = time.monotonic()
|
|
70
|
+
if exc.code != 429 or attempt == 2:
|
|
71
|
+
raise
|
|
72
|
+
time.sleep(5 * (attempt + 1))
|
|
73
|
+
raise RuntimeError("unreachable request retry state")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _get_json(url: str, params: dict | None = None, headers: dict | None = None) -> dict:
|
|
77
|
+
return json.loads(request(url, params, headers))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _get_text(url: str, params: dict | None = None) -> str:
|
|
81
|
+
return request(url, params).decode(errors="replace")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _post_json(url: str, payload: dict, headers: dict | None = None) -> dict:
|
|
85
|
+
request_headers = {"Content-Type": "application/json", **(headers or {})}
|
|
86
|
+
return json.loads(request(url, headers=request_headers, data=json.dumps(payload).encode()))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _result(source: str, url: str, response: dict | None = None, error: str | None = None) -> dict:
|
|
90
|
+
out = {"source": source, "url": url, "status": "ok" if error is None else "error"}
|
|
91
|
+
if response is not None:
|
|
92
|
+
out["response"] = response
|
|
93
|
+
if error is not None:
|
|
94
|
+
out["error"] = error
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _safe(call, source: str, url: str) -> dict:
|
|
99
|
+
try:
|
|
100
|
+
return call()
|
|
101
|
+
except (urllib.error.URLError, TimeoutError, OSError, ValueError, ET.ParseError) as exc:
|
|
102
|
+
return _result(source, url, error=str(exc))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def fetch_arxiv(arxiv_id: str) -> dict:
|
|
106
|
+
url = "https://export.arxiv.org/api/query"
|
|
107
|
+
|
|
108
|
+
def run() -> dict:
|
|
109
|
+
root = ET.fromstring(_get_text(url, {"id_list": arxiv_id, "max_results": "1"}))
|
|
110
|
+
entry = root.find("atom:entry", ARXIV_NS)
|
|
111
|
+
if entry is None:
|
|
112
|
+
return _result("arxiv", url, error="not found")
|
|
113
|
+
authors = [node.findtext("atom:name", "", ARXIV_NS) for node in entry.findall("atom:author", ARXIV_NS)]
|
|
114
|
+
return _result(
|
|
115
|
+
"arxiv",
|
|
116
|
+
f"https://arxiv.org/abs/{arxiv_id}",
|
|
117
|
+
{
|
|
118
|
+
"id": arxiv_id,
|
|
119
|
+
"title": " ".join((entry.findtext("atom:title", "", ARXIV_NS) or "").split()),
|
|
120
|
+
"authors": authors,
|
|
121
|
+
"published": entry.findtext("atom:published", "", ARXIV_NS),
|
|
122
|
+
"updated": entry.findtext("atom:updated", "", ARXIV_NS),
|
|
123
|
+
"comment": entry.findtext("arxiv:comment", None, ARXIV_NS),
|
|
124
|
+
"categories": [node.get("term") for node in entry.findall("atom:category", ARXIV_NS)],
|
|
125
|
+
},
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
return _safe(run, "arxiv", url)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def resolve_s2(ref: PaperRef) -> dict:
|
|
132
|
+
prefix = {"arxiv": "ARXIV", "doi": "DOI", "dblp": "DBLP", "openreview": "URL"}[ref.kind]
|
|
133
|
+
value = f"https://openreview.net/forum?id={ref.value}" if ref.kind == "openreview" else ref.value
|
|
134
|
+
ident = urllib.parse.quote(f"{prefix}:{value}", safe="")
|
|
135
|
+
url = f"https://api.semanticscholar.org/graph/v1/paper/{ident}"
|
|
136
|
+
headers = (
|
|
137
|
+
{"x-api-key": os.environ["SEMANTIC_SCHOLAR_API_KEY"]} if os.environ.get("SEMANTIC_SCHOLAR_API_KEY") else {}
|
|
138
|
+
)
|
|
139
|
+
params = {"fields": S2_FIELDS}
|
|
140
|
+
return _safe(lambda: _result("semantic_scholar", url, _get_json(url, params, headers)), "semantic_scholar", url)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def fetch_crossref(doi: str) -> dict:
|
|
144
|
+
url = f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='/')}"
|
|
145
|
+
|
|
146
|
+
def run() -> dict:
|
|
147
|
+
message = _get_json(url).get("message", {})
|
|
148
|
+
fields = {
|
|
149
|
+
key: message.get(key)
|
|
150
|
+
for key in (
|
|
151
|
+
"title",
|
|
152
|
+
"author",
|
|
153
|
+
"published",
|
|
154
|
+
"container-title",
|
|
155
|
+
"DOI",
|
|
156
|
+
"type",
|
|
157
|
+
"page",
|
|
158
|
+
"volume",
|
|
159
|
+
"issue",
|
|
160
|
+
"publisher",
|
|
161
|
+
"event",
|
|
162
|
+
)
|
|
163
|
+
}
|
|
164
|
+
return _result("crossref", url, fields)
|
|
165
|
+
|
|
166
|
+
return _safe(run, "crossref", url)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def fetch_dblp(*, key: str | None, doi: str | None, title: str | None, local_only: bool = False) -> dict:
|
|
170
|
+
from . import dblp_index
|
|
171
|
+
|
|
172
|
+
if hits := dblp_index.lookup(key=key, doi=doi):
|
|
173
|
+
return _result("dblp", hits[0]["url"], {"matches": hits, "method": "local_index"})
|
|
174
|
+
if title and (hits := dblp_index.search(title)):
|
|
175
|
+
source = hits[0]["url"] if len(hits) == 1 else str(dblp_index.index_path())
|
|
176
|
+
return _result("dblp", source, {"matches": hits, "method": "local_index"})
|
|
177
|
+
if local_only:
|
|
178
|
+
return {
|
|
179
|
+
"source": "dblp",
|
|
180
|
+
"url": str(dblp_index.index_path()),
|
|
181
|
+
"status": "no_match",
|
|
182
|
+
"reason": "not found in local index",
|
|
183
|
+
}
|
|
184
|
+
if key:
|
|
185
|
+
url = f"https://dblp.org/rec/{key}.bib?param=0"
|
|
186
|
+
elif doi:
|
|
187
|
+
url = f"https://dblp.org/doi/{doi}.bib?param=0"
|
|
188
|
+
elif title:
|
|
189
|
+
return search("dblp", title)
|
|
190
|
+
else:
|
|
191
|
+
return _result("dblp", "https://dblp.org", error="no DBLP key, DOI, or title")
|
|
192
|
+
return _safe(lambda: _result("dblp", url, {"bibtex": _get_text(url).strip()}), "dblp", url)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def fetch_openreview(openreview_id: str) -> dict:
|
|
196
|
+
page = f"https://openreview.net/forum?id={openreview_id}"
|
|
197
|
+
token = os.environ.get("OPENREVIEW_ACCESS_TOKEN")
|
|
198
|
+
headers = {"Cookie": f"openreview.accessToken={token}"} if token else {}
|
|
199
|
+
endpoint = "https://api2.openreview.net/notes/search"
|
|
200
|
+
result = _safe(
|
|
201
|
+
lambda: _result(
|
|
202
|
+
"openreview",
|
|
203
|
+
page,
|
|
204
|
+
_post_json(endpoint, {"ids": [openreview_id], "source": "all", "limit": 10}, headers),
|
|
205
|
+
),
|
|
206
|
+
"openreview",
|
|
207
|
+
endpoint,
|
|
208
|
+
)
|
|
209
|
+
if result["status"] == "ok" and result.get("response", {}).get("notes"):
|
|
210
|
+
return result
|
|
211
|
+
errors = [result.get("error", f"no note at {endpoint}")]
|
|
212
|
+
endpoint = "https://api.openreview.net/notes"
|
|
213
|
+
result = _safe(
|
|
214
|
+
lambda: _result("openreview", page, _get_json(endpoint, {"id": openreview_id}, headers)),
|
|
215
|
+
"openreview",
|
|
216
|
+
endpoint,
|
|
217
|
+
)
|
|
218
|
+
if result["status"] == "ok" and result.get("response", {}).get("notes"):
|
|
219
|
+
return result
|
|
220
|
+
errors.append(result.get("error", f"no note at {endpoint}"))
|
|
221
|
+
return _result("openreview", page, error="; ".join(errors))
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def fetch_acl(acl_id: str) -> dict:
|
|
225
|
+
url = f"https://aclanthology.org/{acl_id}.bib"
|
|
226
|
+
return _safe(lambda: _result("acl_anthology", url, {"bibtex": _get_text(url).strip()}), "acl_anthology", url)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def fetch_all(
|
|
230
|
+
raw_ref: str | PaperRef,
|
|
231
|
+
enabled: set[str] | None = None,
|
|
232
|
+
*,
|
|
233
|
+
local_only: bool = False,
|
|
234
|
+
) -> list[dict]:
|
|
235
|
+
ref = raw_ref if isinstance(raw_ref, PaperRef) else PaperRef.parse(raw_ref)
|
|
236
|
+
enabled = enabled or set(SOURCES)
|
|
237
|
+
results: list[dict] = []
|
|
238
|
+
ids = {"doi": None, "arxiv": None, "dblp": None, "openreview": None, "acl": None, "title": None}
|
|
239
|
+
ids[ref.kind] = ref.value
|
|
240
|
+
if ref.kind == "doi" and (match := re.match(r"^10\.18653/v1/(.+)$", ref.value)):
|
|
241
|
+
ids["acl"] = match.group(1)
|
|
242
|
+
available = {
|
|
243
|
+
"arxiv": bool(ids["arxiv"]),
|
|
244
|
+
"crossref": bool(ids["doi"]),
|
|
245
|
+
"dblp": bool(ids["dblp"] or ids["doi"] or ids["title"]),
|
|
246
|
+
"openreview": bool(ids["openreview"]),
|
|
247
|
+
"acl_anthology": bool(ids["acl"]),
|
|
248
|
+
}
|
|
249
|
+
needs_resolution = "semantic_scholar" in enabled or any(
|
|
250
|
+
not available.get(source, False) for source in enabled if source != "semantic_scholar"
|
|
251
|
+
)
|
|
252
|
+
s2 = resolve_s2(ref) if needs_resolution else None
|
|
253
|
+
if "semantic_scholar" in enabled and s2 is not None:
|
|
254
|
+
results.append(s2)
|
|
255
|
+
if s2 is not None and s2["status"] == "ok":
|
|
256
|
+
data = s2.get("response", {})
|
|
257
|
+
external = data.get("externalIds") or {}
|
|
258
|
+
ids.update(
|
|
259
|
+
{
|
|
260
|
+
"doi": ids["doi"] or external.get("DOI"),
|
|
261
|
+
"arxiv": ids["arxiv"] or external.get("ArXiv"),
|
|
262
|
+
"dblp": ids["dblp"] or external.get("DBLP"),
|
|
263
|
+
"acl": ids["acl"] or external.get("ACL"),
|
|
264
|
+
"title": data.get("title"),
|
|
265
|
+
}
|
|
266
|
+
)
|
|
267
|
+
if "dblp" in enabled:
|
|
268
|
+
results.append(
|
|
269
|
+
fetch_dblp(
|
|
270
|
+
key=ids["dblp"],
|
|
271
|
+
doi=ids["doi"],
|
|
272
|
+
title=ids["title"],
|
|
273
|
+
local_only=local_only,
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
if "crossref" in enabled and ids["doi"]:
|
|
277
|
+
results.append(fetch_crossref(ids["doi"]))
|
|
278
|
+
elif "crossref" in enabled:
|
|
279
|
+
results.append({"source": "crossref", "status": "unavailable", "reason": "no DOI after discovery"})
|
|
280
|
+
if "openreview" in enabled and ids["openreview"]:
|
|
281
|
+
results.append(fetch_openreview(ids["openreview"]))
|
|
282
|
+
elif "openreview" in enabled:
|
|
283
|
+
results.append({"source": "openreview", "status": "unavailable", "reason": "no OpenReview ID after discovery"})
|
|
284
|
+
if "acl_anthology" in enabled and ids["acl"]:
|
|
285
|
+
results.append(fetch_acl(ids["acl"]))
|
|
286
|
+
elif "acl_anthology" in enabled:
|
|
287
|
+
results.append({"source": "acl_anthology", "status": "unavailable", "reason": "no ACL ID after discovery"})
|
|
288
|
+
if "arxiv" in enabled and ids["arxiv"]:
|
|
289
|
+
results.append(fetch_arxiv(ids["arxiv"]))
|
|
290
|
+
elif "arxiv" in enabled:
|
|
291
|
+
results.append({"source": "arxiv", "status": "unavailable", "reason": "no arXiv ID after discovery"})
|
|
292
|
+
return results
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def search(source: str, query: str, *, local_only: bool = False) -> dict:
|
|
296
|
+
if source == "dblp":
|
|
297
|
+
from . import dblp_index
|
|
298
|
+
|
|
299
|
+
if hits := dblp_index.search(query):
|
|
300
|
+
return _result("dblp", str(dblp_index.index_path()), {"query": query, "matches": hits})
|
|
301
|
+
if local_only:
|
|
302
|
+
return {
|
|
303
|
+
"source": "dblp",
|
|
304
|
+
"url": str(dblp_index.index_path()),
|
|
305
|
+
"status": "no_match",
|
|
306
|
+
"reason": "not found in local index",
|
|
307
|
+
}
|
|
308
|
+
url = "https://dblp.org/search/publ/api"
|
|
309
|
+
return _safe(
|
|
310
|
+
lambda: _result("dblp", url, _get_json(url, {"q": query, "format": "json", "h": 10})), "dblp", url
|
|
311
|
+
)
|
|
312
|
+
if source == "crossref":
|
|
313
|
+
url = "https://api.crossref.org/works"
|
|
314
|
+
return _safe(
|
|
315
|
+
lambda: _result("crossref", url, _get_json(url, {"query.title": query, "rows": 10})), "crossref", url
|
|
316
|
+
)
|
|
317
|
+
if source == "arxiv":
|
|
318
|
+
url = "https://export.arxiv.org/api/query"
|
|
319
|
+
|
|
320
|
+
def arxiv_search() -> dict:
|
|
321
|
+
root = ET.fromstring(_get_text(url, {"search_query": f'ti:"{query}"', "max_results": 10}))
|
|
322
|
+
matches = [
|
|
323
|
+
{
|
|
324
|
+
"id": entry.findtext("atom:id", "", ARXIV_NS),
|
|
325
|
+
"title": " ".join((entry.findtext("atom:title", "", ARXIV_NS) or "").split()),
|
|
326
|
+
"published": entry.findtext("atom:published", "", ARXIV_NS),
|
|
327
|
+
}
|
|
328
|
+
for entry in root.findall("atom:entry", ARXIV_NS)
|
|
329
|
+
]
|
|
330
|
+
return _result("arxiv", url, {"query": query, "matches": matches})
|
|
331
|
+
|
|
332
|
+
return _safe(arxiv_search, "arxiv", url)
|
|
333
|
+
if source == "openreview":
|
|
334
|
+
token = os.environ.get("OPENREVIEW_ACCESS_TOKEN")
|
|
335
|
+
headers = {"Cookie": f"openreview.accessToken={token}"} if token else {}
|
|
336
|
+
endpoints = (
|
|
337
|
+
"https://api2.openreview.net/notes/search",
|
|
338
|
+
"https://api.openreview.net/notes/search",
|
|
339
|
+
)
|
|
340
|
+
matches = []
|
|
341
|
+
errors = []
|
|
342
|
+
for endpoint in endpoints:
|
|
343
|
+
result = _safe(
|
|
344
|
+
lambda endpoint=endpoint: _result(
|
|
345
|
+
"openreview",
|
|
346
|
+
endpoint,
|
|
347
|
+
_get_json(endpoint, {"query": query, "limit": 10, "source": "forum"}, headers),
|
|
348
|
+
),
|
|
349
|
+
"openreview",
|
|
350
|
+
endpoint,
|
|
351
|
+
)
|
|
352
|
+
if result["status"] == "ok":
|
|
353
|
+
matches.extend(result.get("response", {}).get("notes", []))
|
|
354
|
+
else:
|
|
355
|
+
errors.append(result.get("error", endpoint))
|
|
356
|
+
if not matches and errors:
|
|
357
|
+
return _result("openreview", endpoints[0], error="; ".join(errors))
|
|
358
|
+
unique = {}
|
|
359
|
+
for note in matches:
|
|
360
|
+
unique[note.get("forum") or note.get("id") or json.dumps(note, sort_keys=True)] = note
|
|
361
|
+
return _result(
|
|
362
|
+
"openreview",
|
|
363
|
+
endpoints[0],
|
|
364
|
+
{"query": query, "matches": list(unique.values()), "api_endpoints": list(endpoints)},
|
|
365
|
+
)
|
|
366
|
+
if source == "s2":
|
|
367
|
+
url = "https://api.semanticscholar.org/graph/v1/paper/search"
|
|
368
|
+
headers = (
|
|
369
|
+
{"x-api-key": os.environ["SEMANTIC_SCHOLAR_API_KEY"]} if os.environ.get("SEMANTIC_SCHOLAR_API_KEY") else {}
|
|
370
|
+
)
|
|
371
|
+
params = {"query": query, "limit": 10, "fields": S2_FIELDS}
|
|
372
|
+
return _safe(
|
|
373
|
+
lambda: _result("semantic_scholar", url, _get_json(url, params, headers)), "semantic_scholar", url
|
|
374
|
+
)
|
|
375
|
+
raise ValueError(f"unknown metadata source: {source}")
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def print_results(results: list[dict] | dict, *, json_output: bool) -> None:
|
|
379
|
+
if json_output:
|
|
380
|
+
print(json.dumps(results, indent=2, ensure_ascii=False))
|
|
381
|
+
return
|
|
382
|
+
items = results if isinstance(results, list) else [results]
|
|
383
|
+
for item in items:
|
|
384
|
+
print(f"{item['source']}: {item['status']}")
|
|
385
|
+
if item.get("url"):
|
|
386
|
+
print(f" source: {item['url']}")
|
|
387
|
+
if item.get("reason"):
|
|
388
|
+
print(f" reason: {item['reason']}")
|
|
389
|
+
if item.get("error"):
|
|
390
|
+
print(f" error: {item['error']}")
|
|
391
|
+
elif item.get("response") is not None:
|
|
392
|
+
print(json.dumps(item["response"], indent=2, ensure_ascii=False))
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: paperstack-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Review, inspect, and retrieve research papers from one CLI
|
|
5
|
+
Project-URL: Repository, https://github.com/MilkClouds/my-paperstack
|
|
6
|
+
Requires-Python: >=3.11.4
|
|
7
|
+
Requires-Dist: polars>=1.43
|
|
8
|
+
Requires-Dist: python-dotenv>=1.2.2
|
|
9
|
+
Requires-Dist: pyyaml>=6
|
|
10
|
+
Provides-Extra: pdf
|
|
11
|
+
Requires-Dist: pymupdf4llm>=0.0.17; extra == 'pdf'
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# Critical Reads
|
|
15
|
+
|
|
16
|
+
Some papers mistake their own story for evidence. Some are optimized for acceptance rather than truth. Others are
|
|
17
|
+
careful, competent work on directions that do not matter. Critical Reads exists to tell them apart, preserve the
|
|
18
|
+
insights that survive scrutiny, and choose research directions worth pursuing.
|
|
19
|
+
|
|
20
|
+
Body-read reviews of papers, posts, and talks. Each entry lives in `entries/<citation-key>.md` and carries its identity in frontmatter.
|
|
21
|
+
|
|
22
|
+
- [Collections](#collections)
|
|
23
|
+
- [CLI](#cli)
|
|
24
|
+
- [DBLP index](#dblp-index)
|
|
25
|
+
- [Configuration](#configuration)
|
|
26
|
+
- [Adding a review](#adding-a-review)
|
|
27
|
+
- [Getting the source in front of you](#getting-the-source-in-front-of-you)
|
|
28
|
+
- [Review guide](#review-guide)
|
|
29
|
+
- [For robotics papers](#for-robotics-papers)
|
|
30
|
+
|
|
31
|
+
## Collections
|
|
32
|
+
|
|
33
|
+
Curated, ordered lists live in [`collections.json`](collections.json), the single source used by the viewer and validation.
|
|
34
|
+
Every collection has a `published` or `draft` status. Published collections are stable reference catalogs; draft
|
|
35
|
+
collections are provisional reading paths shown in a separate collapsed section. Select one to filter entries in its
|
|
36
|
+
declared order.
|
|
37
|
+
|
|
38
|
+
CI publishes the generated viewer after GitHub Pages is enabled with GitHub Actions as its source.
|
|
39
|
+
|
|
40
|
+
## CLI
|
|
41
|
+
|
|
42
|
+
Run `make serve` for the local viewer. Entry links open rendered HTML; `.md` URLs expose UTF-8 Markdown source. Install the CLI from PyPI, then authenticate with GitHub for review syncing and DBLP index downloads:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
uv tool install paperstack-cli
|
|
46
|
+
gh auth login
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The command groups have separate data and authority boundaries:
|
|
50
|
+
|
|
51
|
+
| Group | Data | Behavior |
|
|
52
|
+
|---|---|---|
|
|
53
|
+
| `review` | `entries/` review database | Read, initialize, validate, or audit authored judgments |
|
|
54
|
+
| `paper` | External metadata and arXiv content | Return source records or content without choosing a citation |
|
|
55
|
+
| `index` | Optional local indexes | Install and manage lookup data |
|
|
56
|
+
|
|
57
|
+
Review commands:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
paperstack review show black2024pi0 --brief
|
|
61
|
+
paperstack review show arxiv:2410.24164 --json
|
|
62
|
+
paperstack review list --quality poor --tag vla
|
|
63
|
+
paperstack review search "flow matching"
|
|
64
|
+
paperstack review sync --force
|
|
65
|
+
paperstack review init <key> --id arxiv:NNNN.NNNNN --title "Verbatim title" --editor <name>
|
|
66
|
+
paperstack review check --style
|
|
67
|
+
paperstack review audit
|
|
68
|
+
paperstack review citations --fetch
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Paper commands:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
paperstack paper search "Attention Is All You Need" --source dblp
|
|
75
|
+
paperstack paper metadata arxiv:2106.09685
|
|
76
|
+
paperstack paper metadata arxiv:2410.15549 --source semantic_scholar --json
|
|
77
|
+
paperstack paper metadata doi:10.1109/CVPR.2016.90 --source crossref
|
|
78
|
+
paperstack paper read arxiv:2604.23073
|
|
79
|
+
paperstack paper read arxiv:2604.23073 --outline
|
|
80
|
+
paperstack paper read arxiv:2604.23073 --section 6
|
|
81
|
+
paperstack paper pdf arxiv:2602.09017
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`metadata` accepts `arxiv:`, `doi:`, `dblp:`, and `openreview:` references. Semantic Scholar records include total,
|
|
85
|
+
influential, and reference counts. `read` and `pdf` require an `arxiv:` reference. Metadata output keeps
|
|
86
|
+
source records separate and includes provenance; it never selects or generates a citation.
|
|
87
|
+
|
|
88
|
+
Options are scoped to commands that use them. `--json` is available for structured records and `--offline` for
|
|
89
|
+
commands with a local or cached path. PDF conversion requires the optional dependency:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
uv tool install 'paperstack-cli[pdf]'
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
From a clone, replace `paperstack` with `uv run paperstack`.
|
|
96
|
+
For PDF conversion in a clone, use `uv run --extra pdf paperstack paper pdf <arxiv-ref>`.
|
|
97
|
+
`review init` requires a writable review tree; `review check` requires a clone containing `scripts/build/check.sh`.
|
|
98
|
+
`review citations --fetch` updates `citations.json` for every arXiv-backed entry through Semantic Scholar's batch API;
|
|
99
|
+
without `--fetch`, it only removes cached records for entries that no longer exist. The static viewer exposes a minimum-citation
|
|
100
|
+
filter and shows counts alongside entries. `SEMANTIC_SCHOLAR_API_KEY` is optional but raises the API rate limit.
|
|
101
|
+
|
|
102
|
+
### DBLP index
|
|
103
|
+
|
|
104
|
+
The optional index accelerates DBLP search and is required by `review audit`. It covers selected CS venues, not all
|
|
105
|
+
of DBLP. The `2026.08` Parquet snapshot contains 285,521 structured records and is about 25 MiB. Installation is explicit:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
paperstack index dblp status
|
|
109
|
+
paperstack index dblp install
|
|
110
|
+
paperstack index dblp update
|
|
111
|
+
paperstack index dblp remove --yes
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
`install` downloads the pinned Parquet snapshot through authenticated `gh`. `update` discovers the newest
|
|
115
|
+
`dblp-index-YYYY.MM` paperstack Release and replaces an older index only after its SHA-256, schema, and embedded metadata checks pass. `status` reports the version,
|
|
116
|
+
coverage, size, record count, and location. `review audit` reports title and venue matches without editing reviews.
|
|
117
|
+
Snapshots with mismatched metadata or fewer than 250,000 records are rejected. Installed files are immutable and
|
|
118
|
+
content-addressed; an atomic pointer switch preserves the previous index if an update is interrupted.
|
|
119
|
+
See [DBLP snapshot releases](docs/DBLP_RELEASES.md) for the publishing procedure.
|
|
120
|
+
|
|
121
|
+
### Configuration
|
|
122
|
+
|
|
123
|
+
| Variable | Purpose |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `PAPERSTACK_DIR` | Review database to use instead of the surrounding clone or GitHub cache |
|
|
126
|
+
| `PAPERSTACK_REPO` | GitHub review repository; default `MilkClouds/my-paperstack` |
|
|
127
|
+
| `PAPERSTACK_TTL` | Review-cache refresh interval in seconds; default `3600` |
|
|
128
|
+
| `PAPERSTACK_PAPERS_DIR` | arXiv source/PDF cache; default `${XDG_CACHE_HOME:-~/.cache}/paperstack/papers` |
|
|
129
|
+
| `SEMANTIC_SCHOLAR_API_KEY` | Optional key for Semantic Scholar discovery |
|
|
130
|
+
| `OPENREVIEW_ACCESS_TOKEN` | Optional OpenReview `openreview.accessToken` cookie value |
|
|
131
|
+
| `XDG_CACHE_HOME` | Review and paper cache root |
|
|
132
|
+
| `XDG_DATA_HOME` | DBLP index root |
|
|
133
|
+
|
|
134
|
+
Configuration comes from the process environment. The CLI also loads the nearest `.env` without overriding exported
|
|
135
|
+
variables, so a clone can keep local configuration in a gitignored `.env` file.
|
|
136
|
+
|
|
137
|
+
Review lookup reads `$PAPERSTACK_DIR`, a surrounding clone, or a GitHub-backed cache, in that order. Scoped
|
|
138
|
+
`--offline` flags serve cached review or paper data without network access. Exit codes are `0` for
|
|
139
|
+
hits, `1` for no match, `2` for ambiguity, and `3` for unavailable data.
|
|
140
|
+
|
|
141
|
+
## Adding a review
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
paperstack review init <key> --id arxiv:NNNN.NNNNN --title "Verbatim title" --editor <name>
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
This initializes an ungraded scaffold only. The review itself remains a reading and judgment task. Review files are
|
|
148
|
+
stored internally as `entries/<key>.md`. See the [review guide](#review-guide) for editorial guidance.
|
|
149
|
+
|
|
150
|
+
- Name files `<first-author surname><arXiv v1 year><first significant title word>`, lowercase; suffix collisions with `a`, `b`, and so on.
|
|
151
|
+
- Use the established method name or full title as the `#` heading.
|
|
152
|
+
- Use a registered CURIE (`arxiv:`, `doi:`, `hdl:`, `isbn:`) for `id`, or a URL when none exists.
|
|
153
|
+
- `tags` are lowercase and singular. Reuse before inventing.
|
|
154
|
+
- Include only verified affiliations in `lab`. Use a person's name in `editor`, or `model effort (harness)` for an agent.
|
|
155
|
+
- Use CommonMark with GFM, including tables for tabular results.
|
|
156
|
+
|
|
157
|
+
Run `make check`; `make style` adds prose-length warnings. The scripts require `jq` and mikefarah's `yq` (`go-yq` on conda-forge).
|
|
158
|
+
|
|
159
|
+
### Getting the source in front of you
|
|
160
|
+
|
|
161
|
+
The source fetcher uses `latexpand` from PATH, falling back to a vendored copy via Perl.
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
paperstack paper read arxiv:2604.23073 # the complete LaTeX body
|
|
165
|
+
paperstack paper read arxiv:2604.23073 --outline # the section outline
|
|
166
|
+
paperstack paper read arxiv:2604.23073 --section 6
|
|
167
|
+
paperstack paper pdf arxiv:cs/9301101 # when there is no LaTeX source
|
|
168
|
+
yt-dlp --skip-download --write-auto-subs --sub-lang en -o talk <url>
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
- Prefer the LaTeX source; use the PDF fallback only when no source exists.
|
|
172
|
+
- Cross-check malformed tables with `pdftotext -layout <pdf> -`.
|
|
173
|
+
- Treat commented-out results as evidence only when their surviving values match the published version; a mismatched baseline may be an earlier run.
|
|
174
|
+
- Strip timestamps and duplicate cues from video captions.
|
|
175
|
+
|
|
176
|
+
## Review guide
|
|
177
|
+
|
|
178
|
+
Read the body, then write the critical read, paper summary, reason to read, and one-liner, in that order.
|
|
179
|
+
|
|
180
|
+
- Keep the review body within 2,500 visible non-whitespace characters. Use prose for argument, bullets for independent points, and tables for repeated comparisons.
|
|
181
|
+
- Make the summary self-contained: a reader who has not read the paper should understand its core problem, approach, evidence, and findings. Choose the form that reads best; five bullets is one option, not a target.
|
|
182
|
+
- In the critical read, consult prior and subsequent work as needed, then focus on what matters for interpreting, trusting, or using the paper.
|
|
183
|
+
- Keep the one-liner to one sentence and `Why read it` to two. `Why read it` captures significance, originality, or practical value; use `none` when there is no reason.
|
|
184
|
+
- `Quality` measures how much of the title and abstract's main claim survives the evidence:
|
|
185
|
+
- `excellent`: the claim stands and has lasting importance
|
|
186
|
+
- `good`: the claim stands
|
|
187
|
+
- `fair`: only a narrower claim stands
|
|
188
|
+
- `poor`: the claim is not established
|
|
189
|
+
- Grade the advertised claim, not the narrower verdict. Use `fair` only when narrowing scope preserves the core claim; materially replacing it is `poor`.
|
|
190
|
+
- For SOTA or efficiency claims, check the strongest comparable result and name the denominator.
|
|
191
|
+
- If an official protocol fits the claim, an unjustified custom replacement caps `Quality` at `fair` unless matched, interpretable anchors restore comparability.
|
|
192
|
+
- Disclosed weaknesses still count when the paper claims past them. Side contributions do not raise the grade.
|
|
193
|
+
- On `fair` or `poor`, add `Read it anyway.` only with a checked citation count from Hugging Face or Semantic Scholar.
|
|
194
|
+
|
|
195
|
+
### For robotics papers
|
|
196
|
+
|
|
197
|
+
See [What Are We Actually Benchmarking in Robot Manipulation?](entries/jiang2026benchmarking.md).
|
|
198
|
+
|
|
199
|
+
- A benchmark counts only when success requires the claimed capability. LIBERO-only evidence caps Quality at `poor`; so does an uncounted real-world result added to it.
|
|
200
|
+
- Judge the hardest benchmark, note omissions, and account for benchmark age and test-set proximity.
|
|
201
|
+
- Treat margins within evaluation noise as ties; check SOTA claims against the [VLA Evaluation Harness](https://allenai.github.io/vla-evaluation-harness/leaderboard/).
|
|
202
|
+
- Recover exact values and trial counts where possible; otherwise state that they are unavailable.
|
|
203
|
+
- Compare baselines only under the same training and evaluation protocol.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
paperstack/__init__.py,sha256=Cnu12PizQdtp2EOJIPMV8LN-2JXdtSiPodSD3iISedg,39
|
|
2
|
+
paperstack/citations.py,sha256=24WSEDBLzHv1YFKy0Vpoh2xu4GJNaHuugl-5-ef_ctQ,4113
|
|
3
|
+
paperstack/cli.py,sha256=C6Iwt_71p5XvOw1m23sHqQWF68TlL-UKcwTPc1zM_ts,27372
|
|
4
|
+
paperstack/dblp_index.py,sha256=eIADoAuriJesbi_aBSTjORxsWj2NTPuJtZnzJUwrczo,20261
|
|
5
|
+
paperstack/entrypoint.py,sha256=4DOd54fov2qVwlFcbTgh8883zG_k5br4EO00oG7OISY,534
|
|
6
|
+
paperstack/metadata.py,sha256=wgdfpt2kBDQ14ZR_VfaWlULdBoRq27CILcv3f4pE9Tg,15701
|
|
7
|
+
paperstack/content/__init__.py,sha256=rPAfhJSpEpOFyKlPNbbfEZ9-oDTbfJDEqxJHnCgHVp0,54
|
|
8
|
+
paperstack/content/arxiv_pdf.py,sha256=CzC2j7bHS6_0AxlXjNIOVpIbxLHvFTjnoK_DYsc0ACs,3086
|
|
9
|
+
paperstack/content/arxiv_source.py,sha256=zsgDRlIAQUE7IAZbxRMMbf3pfyfw8ImNzS2FhQvM6yI,15125
|
|
10
|
+
paperstack/content/vendor/latexpand,sha256=Z-ylXQmz-5gt7iNE5jF_fhDp8Vg5EC5qHbOOUY-DMDw,23338
|
|
11
|
+
paperstack/content/vendor/latexpand.LICENSE,sha256=hpipH1rQQwKFd1DYN_FtpxSBbYqCBBjsgCKr8YuYJhQ,1529
|
|
12
|
+
paperstack_cli-0.1.0.dist-info/METADATA,sha256=kzGwuhII5as6Q3gImH1MC87VcXTJpNr14lRRaVhCTig,11012
|
|
13
|
+
paperstack_cli-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
14
|
+
paperstack_cli-0.1.0.dist-info/entry_points.txt,sha256=0716S7tM1TuT4_aj291u3335bww-f0K0wQ1u8pS7VxU,58
|
|
15
|
+
paperstack_cli-0.1.0.dist-info/RECORD,,
|