fetchpdf 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. fetchpdf/__init__.py +49 -0
  2. fetchpdf/__main__.py +6 -0
  3. fetchpdf/_env.py +63 -0
  4. fetchpdf/_http.py +40 -0
  5. fetchpdf/fetch_abstract_from_doi.py +549 -0
  6. fetchpdf/fetch_metadata_from_doi.py +225 -0
  7. fetchpdf/fetchpdf.py +7339 -0
  8. fetchpdf/retrieval/__init__.py +36 -0
  9. fetchpdf/retrieval/_util.py +150 -0
  10. fetchpdf/retrieval/artifact.py +87 -0
  11. fetchpdf/retrieval/backends/__init__.py +109 -0
  12. fetchpdf/retrieval/backends/claude_cli.py +111 -0
  13. fetchpdf/retrieval/backends/openrouter.py +244 -0
  14. fetchpdf/retrieval/cache.py +123 -0
  15. fetchpdf/retrieval/chunk.py +195 -0
  16. fetchpdf/retrieval/classify.py +110 -0
  17. fetchpdf/retrieval/context.py +69 -0
  18. fetchpdf/retrieval/corresponding.py +349 -0
  19. fetchpdf/retrieval/engine.py +666 -0
  20. fetchpdf/retrieval/extract_images.py +593 -0
  21. fetchpdf/retrieval/fulltext_scan.py +472 -0
  22. fetchpdf/retrieval/goals.py +145 -0
  23. fetchpdf/retrieval/http.py +669 -0
  24. fetchpdf/retrieval/identifiers.py +149 -0
  25. fetchpdf/retrieval/ladder.json +362 -0
  26. fetchpdf/retrieval/linked_artifacts.py +144 -0
  27. fetchpdf/retrieval/llm_adjudicate.py +278 -0
  28. fetchpdf/retrieval/llm_agent_retrieval.py +495 -0
  29. fetchpdf/retrieval/normalize.py +301 -0
  30. fetchpdf/retrieval/pdf_identity.py +533 -0
  31. fetchpdf/retrieval/pdf_text.py +242 -0
  32. fetchpdf/retrieval/provenance.py +185 -0
  33. fetchpdf/retrieval/ratelimit.py +113 -0
  34. fetchpdf/retrieval/repository_waf.py +152 -0
  35. fetchpdf/retrieval/request_drafts.py +304 -0
  36. fetchpdf/retrieval/resolve.py +481 -0
  37. fetchpdf/retrieval/sources/__init__.py +15 -0
  38. fetchpdf/retrieval/sources/arxiv.py +102 -0
  39. fetchpdf/retrieval/sources/core_text.py +64 -0
  40. fetchpdf/retrieval/sources/crossref_tdm.py +193 -0
  41. fetchpdf/retrieval/sources/epmc.py +133 -0
  42. fetchpdf/retrieval/sources/landing.py +34 -0
  43. fetchpdf/retrieval/sources/legacy_pdf.py +99 -0
  44. fetchpdf/retrieval/sources/pmc.py +59 -0
  45. fetchpdf/retrieval/sources/preprint.py +101 -0
  46. fetchpdf/retrieval/sources/publisher_html.py +154 -0
  47. fetchpdf/retrieval/sources/supplements.py +183 -0
  48. fetchpdf/retrieval/supplement_atypon.py +269 -0
  49. fetchpdf/retrieval/supplement_dspace.py +187 -0
  50. fetchpdf/retrieval/supplement_graph.py +842 -0
  51. fetchpdf/retrieval/supplement_index.py +820 -0
  52. fetchpdf/retrieval/supplement_pmc.py +351 -0
  53. fetchpdf/retrieval/supplement_publishers.py +374 -0
  54. fetchpdf/retrieval/supplementary.py +1770 -0
  55. fetchpdf/retrieval/tiers.py +213 -0
  56. fetchpdf/retrieval/to_markdown.py +661 -0
  57. fetchpdf/retrieval/validate.py +368 -0
  58. fetchpdf/retrieval/verify.py +149 -0
  59. fetchpdf-0.1.0.dist-info/METADATA +1392 -0
  60. fetchpdf-0.1.0.dist-info/RECORD +64 -0
  61. fetchpdf-0.1.0.dist-info/WHEEL +5 -0
  62. fetchpdf-0.1.0.dist-info/entry_points.txt +5 -0
  63. fetchpdf-0.1.0.dist-info/licenses/LICENSE +21 -0
  64. fetchpdf-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,374 @@
1
+ """Publisher supplementary routes that work over plain HTTP.
2
+
3
+ Which publishers those are is not a matter of taste. Verified with a browser
4
+ User-Agent on 2026-07-29:
5
+
6
+ reachable Springer/Nature (media.springernature.com, static-content.springer.com,
7
+ www.nature.com article pages), PLOS (journals.plos.org article/file),
8
+ bioRxiv/medRxiv (.supplementary-material), OUP/Silverchair, supp.apa.org
9
+ 403/blocked PNAS, Science, Taylor & Francis, Sage (all Atypon), Wiley,
10
+ ScienceDirect, MDPI -- Cloudflare interstitials
11
+ worse ACS returns 404 *with* a 57 KB HTML body, so its 404 is not an answer
12
+
13
+ Only the reachable ones live here. The blocked families need a browser and are
14
+ Phase 2, gated on the existing --add-playwright rather than a new switch.
15
+
16
+ The one exception to "Elsevier is blocked" is worth its own note: ScienceDirect
17
+ pages are walled, but the Elsevier object API is not, and it names supplementary
18
+ material outright. Verified on S2589004225028421: 25 choices, 24 of them
19
+ IMAGE-{DOWNSAMPLED,THUMBNAIL,HIGH-RES}, and one @type "APPLICATION" whose $ is
20
+ the mmc1.pdf download URL. @type == "APPLICATION" *is* the marker.
21
+ """
22
+
23
+ import re
24
+ from typing import List, Optional
25
+
26
+ from .._env import ELSEVIER_TDM_API_KEY
27
+ from .supplement_index import ROLE_SUPPLEMENT, SupplementFile, classify_role, jats_sets
28
+
29
+ ELSEVIER_OBJECT = "https://api.elsevier.com/content/object/pii/{pii}"
30
+ ELSEVIER_OBJECT_BY_DOI = "https://api.elsevier.com/content/object/doi/{doi}"
31
+ PLOS_FILE = "https://journals.plos.org/{slug}/article/file"
32
+ BIORXIV_DETAILS = "https://api.biorxiv.org/details/{server}/{doi}"
33
+ BIORXIV_SUPPL = "https://www.{server}.org/content/{doi}v{version}.supplementary-material"
34
+ APA_SUPPL = "https://supp.apa.org/psycarticles/supplemental/{code}/{code}_supp.html"
35
+
36
+ #: Nature/Springer electronic supplementary material, on either CDN host.
37
+ _SPRINGER_ESM_RE = re.compile(
38
+ r"https://(?:media\.springernature\.com/original/springer-static"
39
+ r"|static-content\.springer\.com)"
40
+ r"/esm/[^\"'\s>]+?/MediaObjects/[^\"'\s>]+?_MOESM\d+_ESM\.\w+",
41
+ re.IGNORECASE,
42
+ )
43
+ _SPRINGER_PREFIXES = ("10.1038", "10.1007", "10.1186", "10.1140", "10.1057", "10.1245")
44
+
45
+ #: PLOS journal slugs, keyed on the DOI's journal stem.
46
+ _PLOS_SLUGS = {
47
+ "pone": "plosone", "pbio": "plosbiology", "pmed": "plosmedicine",
48
+ "pcbi": "ploscompbiol", "pgen": "plosgenetics", "ppat": "plospathogens",
49
+ "pntd": "plosntds", "pstr": "plossustain", "pclm": "climate",
50
+ "pwat": "water", "pdig": "digitalhealth", "pgph": "globalpublichealth",
51
+ }
52
+
53
+ _BIORXIV_MEDIA_RE = re.compile(
54
+ r"/content/(?:bio|med)rxiv/early/\d{4}/\d{2}/\d{2}/[^\"'\s>]+?/DC\d+/embed/"
55
+ r"media-\d+\.\w+(?:\?download=true)?",
56
+ re.IGNORECASE,
57
+ )
58
+
59
+ _APA_HREF_RE = re.compile(r"""href=["']([^"'>]+)["']""", re.IGNORECASE)
60
+
61
+ _MAX_PER_PUBLISHER = 40
62
+
63
+
64
+ # -- E10: Elsevier object API -----------------------------------------------
65
+
66
+
67
+ def enumerate_elsevier_objects(ids, ctx) -> List[SupplementFile]:
68
+ """Attachments named by the Elsevier object API.
69
+
70
+ The only route into Elsevier supplementary material that does not need a
71
+ browser. Non-entitled requests come back 200 with an error payload rather
72
+ than a 4xx, so the payload, not the status, decides.
73
+ """
74
+ # The DOI route is not a fallback for missing PIIs, it is the more reliable
75
+ # of the two: a PII resolved from Crossref can be stale or simply wrong
76
+ # (S0749597813001040 for 10.1016/j.obhdp.2013.09.001 is a 404), while the
77
+ # DOI form answers 300 with the full choice list including the mmc1
78
+ # supplement. Try the PII first only because it costs the same and keeps
79
+ # the existing behaviour where it already works.
80
+ for url in _elsevier_object_urls(ids):
81
+ files = _elsevier_objects_at(url, ids, ctx)
82
+ if files:
83
+ return files
84
+ return []
85
+
86
+
87
+ def _elsevier_object_urls(ids) -> List[str]:
88
+ urls = []
89
+ if ids.elsevier_pii:
90
+ urls.append(ELSEVIER_OBJECT.format(pii=ids.elsevier_pii))
91
+ if ids.doi:
92
+ from urllib.parse import quote
93
+ urls.append(ELSEVIER_OBJECT_BY_DOI.format(doi=quote(ids.doi, safe="")))
94
+ return urls
95
+
96
+
97
+ def _elsevier_objects_at(url: str, ids, ctx) -> List[SupplementFile]:
98
+ """One object-API lookup, by whichever identifier the URL carries."""
99
+ response = ctx.http.get(
100
+ url,
101
+ headers={"X-ELS-APIKey": ELSEVIER_TDM_API_KEY, "Accept": "application/json"},
102
+ timeout=30,
103
+ polite=False,
104
+ )
105
+ # HTTP 300 Multiple Choices is this endpoint's SUCCESS response -- it is
106
+ # literally a list of the representations on offer, which is what we came
107
+ # for. Response.ok is 200 <= status < 300, so `if not response.ok` rejected
108
+ # every Elsevier article ever passed to this provider. Six records in one
109
+ # 46-paper corpus had supplements that were silently dropped this way.
110
+ if not response.ok and response.status != 300:
111
+ return []
112
+ try:
113
+ payload = response.json() or {}
114
+ except ValueError:
115
+ return []
116
+ if "service-error" in payload or "error-response" in payload:
117
+ ctx.log(" Elsevier objects: 200 with an error payload (not entitled)")
118
+ return []
119
+
120
+ entries = _elsevier_choices(payload)
121
+ files = []
122
+ for index, choice in enumerate(entries):
123
+ if str(choice.get("@type") or "").upper() != "APPLICATION":
124
+ continue
125
+ url = choice.get("$") or choice.get("@href")
126
+ if not url:
127
+ continue
128
+ ref = str(choice.get("@ref") or f"mmc{index + 1}")
129
+ files.append(SupplementFile(
130
+ name=_elsevier_name(ref, url),
131
+ url=url,
132
+ provider="elsevier_objects",
133
+ listing_index=index,
134
+ mimetype=choice.get("@mimetype"),
135
+ size_bytes=_as_int(choice.get("@filesize")),
136
+ role=ROLE_SUPPLEMENT,
137
+ origin_doi=ids.doi,
138
+ polite=False,
139
+ extra={"pii": ids.elsevier_pii, "ref": ref},
140
+ ))
141
+ if len(files) >= _MAX_PER_PUBLISHER:
142
+ break
143
+ return files
144
+
145
+
146
+ def _elsevier_choices(payload) -> List[dict]:
147
+ choices = ((payload.get("object-response") or payload).get("choices") or {})
148
+ if isinstance(choices, dict):
149
+ choices = choices.get("choice") or []
150
+ if isinstance(choices, dict):
151
+ choices = [choices]
152
+ return [c for c in choices if isinstance(c, dict)]
153
+
154
+
155
+ def _elsevier_name(ref: str, url: str) -> str:
156
+ """mmc1 plus whatever extension the URL admits to."""
157
+ m = re.search(r"-(mmc\d+\.\w+)", url) or re.search(r"/([^/?]+\.\w{1,5})(?:\?|$)", url)
158
+ if m:
159
+ return m.group(1)
160
+ m = re.search(r"httpAccept=\w+/([\w.+-]+)", url)
161
+ return f"{ref}.{m.group(1)}" if m else ref
162
+
163
+
164
+ # -- E11: Springer / Nature ESM ---------------------------------------------
165
+
166
+
167
+ def enumerate_springer_esm(ids, ctx) -> List[SupplementFile]:
168
+ """MOESM objects scraped off the article page.
169
+
170
+ The CDN URL cannot be derived -- the MOESM stem encodes an internal article
171
+ id -- so the page has to be read. Anchors carry data-test="supp-info-link";
172
+ the regex on the CDN hosts is the fallback for pages that do not.
173
+ """
174
+ if not _has_prefix(ids.doi, _SPRINGER_PREFIXES):
175
+ return []
176
+ response = ctx.http.get(f"https://doi.org/{ids.doi}", timeout=45, polite=False)
177
+ if not response.ok or not response.content:
178
+ return []
179
+
180
+ supplements, figures = jats_sets(ctx)
181
+ files = []
182
+ for index, url in enumerate(_unique(_SPRINGER_ESM_RE.findall(response.text))):
183
+ name = url.rsplit("/", 1)[-1]
184
+ files.append(SupplementFile(
185
+ name=name,
186
+ url=url,
187
+ provider="springer_esm",
188
+ listing_index=index,
189
+ role=classify_role(name, jats_supplements=supplements, jats_figures=figures),
190
+ origin_doi=ids.doi,
191
+ polite=False,
192
+ ))
193
+ if len(files) >= _MAX_PER_PUBLISHER:
194
+ break
195
+ return files
196
+
197
+
198
+ # -- E12: PLOS --------------------------------------------------------------
199
+
200
+
201
+ def enumerate_plos(ids, ctx) -> List[SupplementFile]:
202
+ """PLOS supplementary files, enumerated rather than probed.
203
+
204
+ The sNNN ids come from the JATS manifest or from Crossref components. Probing
205
+ s001, s002, ... blindly would work and would also hammer the host for every
206
+ article that has none, so it is not done.
207
+ """
208
+ if not ids.doi or not ids.doi.startswith("10.1371"):
209
+ return []
210
+ slug = _PLOS_SLUGS.get(_plos_stem(ids.doi))
211
+ if not slug:
212
+ return []
213
+
214
+ supplements, _ = jats_sets(ctx)
215
+ ids_seen = sorted({
216
+ m.group(1)
217
+ for name in supplements
218
+ for m in [re.search(r"\.(s\d{3})\b", name)]
219
+ if m
220
+ })
221
+ if not ids_seen:
222
+ return []
223
+
224
+ files = []
225
+ for index, suffix in enumerate(ids_seen):
226
+ files.append(SupplementFile(
227
+ name=f"{ids.doi.rsplit('/', 1)[-1]}.{suffix}",
228
+ url=PLOS_FILE.format(slug=slug) + f"?id={ids.doi}.{suffix}&type=supplementary",
229
+ provider="plos",
230
+ listing_index=index,
231
+ role=ROLE_SUPPLEMENT,
232
+ origin_doi=ids.doi,
233
+ polite=False,
234
+ ))
235
+ return files
236
+
237
+
238
+ def _plos_stem(doi: str) -> str:
239
+ m = re.search(r"journal\.([a-z]+)\.", doi, re.IGNORECASE)
240
+ return m.group(1).lower() if m else ""
241
+
242
+
243
+ # -- E13: bioRxiv / medRxiv -------------------------------------------------
244
+
245
+
246
+ def enumerate_preprint_supplements(ids, ctx) -> List[SupplementFile]:
247
+ """Preprint supplementary media, from the versioned supplement page.
248
+
249
+ A nonexistent DOI or version 302s to /node/ and answers 200 with a generic
250
+ page, so the final URL is checked for the DOI before anything is parsed.
251
+ """
252
+ if not ids.doi or not ids.doi.startswith("10.1101"):
253
+ return []
254
+ server = (ids.preprint_server or "").lower()
255
+ servers = [server] if server in ("biorxiv", "medrxiv") else ["biorxiv", "medrxiv"]
256
+
257
+ for candidate in servers:
258
+ version = _preprint_version(ids, ctx, candidate)
259
+ if version is None:
260
+ continue
261
+ url = BIORXIV_SUPPL.format(server=candidate, doi=ids.doi, version=version)
262
+ response = ctx.http.get(url, timeout=45, polite=False)
263
+ if not response.ok or not response.content:
264
+ continue
265
+ if ids.doi.lower() not in (response.url or "").lower():
266
+ ctx.log(f" {candidate}: redirected away from the DOI, not parsing")
267
+ continue
268
+ files = []
269
+ for index, path in enumerate(_unique(_BIORXIV_MEDIA_RE.findall(response.text))):
270
+ files.append(SupplementFile(
271
+ name=path.split("/")[-1].split("?")[0],
272
+ url=f"https://www.{candidate}.org{path}",
273
+ provider="preprint_supplements",
274
+ listing_index=index,
275
+ role=ROLE_SUPPLEMENT,
276
+ origin_doi=ids.doi,
277
+ polite=False,
278
+ ))
279
+ if len(files) >= _MAX_PER_PUBLISHER:
280
+ break
281
+ if files:
282
+ return files
283
+ return []
284
+
285
+
286
+ def _preprint_version(ids, ctx, server) -> Optional[int]:
287
+ response = ctx.http.get(
288
+ BIORXIV_DETAILS.format(server=server, doi=ids.doi), timeout=30, polite=False
289
+ )
290
+ if not response.ok:
291
+ return None
292
+ try:
293
+ collection = (response.json() or {}).get("collection") or []
294
+ except ValueError:
295
+ return None
296
+ if not collection:
297
+ # A 200 with an empty collection is how the wrong server answers.
298
+ return None
299
+ versions = [_as_int(entry.get("version")) for entry in collection
300
+ if isinstance(entry, dict)]
301
+ versions = [v for v in versions if v]
302
+ return max(versions) if versions else 1
303
+
304
+
305
+ # -- E14: APA supplemental --------------------------------------------------
306
+
307
+
308
+ def enumerate_apa_supplemental(ids, ctx) -> List[SupplementFile]:
309
+ """The APA supplemental index page.
310
+
311
+ try_apa_supplemental_fallback in the legacy chain scrapes the same page, but
312
+ it is built for a different job -- it returns a bool, writes one PDF to
313
+ save_path, stops at the first success, and runs .doc files through
314
+ LibreOffice. Here every file is wanted and wanted in its deposited format, so
315
+ the scraping is lifted rather than the function reused; the legacy one keeps
316
+ working unchanged for its own purpose.
317
+ """
318
+ if not ids.doi or not ids.doi.startswith("10.1037"):
319
+ return []
320
+ code = ids.doi.rsplit("/", 1)[-1].strip().lower()
321
+ if not code:
322
+ return []
323
+
324
+ response = ctx.http.get(APA_SUPPL.format(code=code), timeout=30, polite=False)
325
+ if not response.ok or not response.content:
326
+ return []
327
+
328
+ base = f"https://supp.apa.org/psycarticles/supplemental/{code}/"
329
+ files = []
330
+ for index, href in enumerate(_unique(_APA_HREF_RE.findall(response.text))):
331
+ if href.startswith("#") or href.lower().startswith(("mailto:", "javascript:")):
332
+ continue
333
+ name = href.split("?")[0].rsplit("/", 1)[-1]
334
+ if not name or "." not in name or name.lower().endswith((".html", ".htm", ".css", ".js")):
335
+ continue
336
+ url = href if href.startswith("http") else base + href.lstrip("/")
337
+ files.append(SupplementFile(
338
+ name=name,
339
+ url=url,
340
+ provider="apa_supplemental",
341
+ listing_index=index,
342
+ role=ROLE_SUPPLEMENT,
343
+ origin_doi=ids.doi,
344
+ polite=False,
345
+ ))
346
+ if len(files) >= _MAX_PER_PUBLISHER:
347
+ break
348
+ return files
349
+
350
+
351
+ # -- shared -----------------------------------------------------------------
352
+
353
+
354
+ def _has_prefix(doi: Optional[str], prefixes) -> bool:
355
+ return bool(doi) and any(str(doi).startswith(p) for p in prefixes)
356
+
357
+
358
+ def _unique(values) -> List[str]:
359
+ """Order-preserving dedupe: a page lists the same href more than once."""
360
+ seen = set()
361
+ out = []
362
+ for value in values:
363
+ if value not in seen:
364
+ seen.add(value)
365
+ out.append(value)
366
+ return out
367
+
368
+
369
+ def _as_int(value) -> Optional[int]:
370
+ try:
371
+ parsed = int(value)
372
+ except (TypeError, ValueError):
373
+ return None
374
+ return parsed if parsed > 0 else None