fetchpdf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fetchpdf/__init__.py +49 -0
- fetchpdf/__main__.py +6 -0
- fetchpdf/_env.py +63 -0
- fetchpdf/_http.py +40 -0
- fetchpdf/fetch_abstract_from_doi.py +549 -0
- fetchpdf/fetch_metadata_from_doi.py +225 -0
- fetchpdf/fetchpdf.py +7339 -0
- fetchpdf/retrieval/__init__.py +36 -0
- fetchpdf/retrieval/_util.py +150 -0
- fetchpdf/retrieval/artifact.py +87 -0
- fetchpdf/retrieval/backends/__init__.py +109 -0
- fetchpdf/retrieval/backends/claude_cli.py +111 -0
- fetchpdf/retrieval/backends/openrouter.py +244 -0
- fetchpdf/retrieval/cache.py +123 -0
- fetchpdf/retrieval/chunk.py +195 -0
- fetchpdf/retrieval/classify.py +110 -0
- fetchpdf/retrieval/context.py +69 -0
- fetchpdf/retrieval/corresponding.py +349 -0
- fetchpdf/retrieval/engine.py +666 -0
- fetchpdf/retrieval/extract_images.py +593 -0
- fetchpdf/retrieval/fulltext_scan.py +472 -0
- fetchpdf/retrieval/goals.py +145 -0
- fetchpdf/retrieval/http.py +669 -0
- fetchpdf/retrieval/identifiers.py +149 -0
- fetchpdf/retrieval/ladder.json +362 -0
- fetchpdf/retrieval/linked_artifacts.py +144 -0
- fetchpdf/retrieval/llm_adjudicate.py +278 -0
- fetchpdf/retrieval/llm_agent_retrieval.py +495 -0
- fetchpdf/retrieval/normalize.py +301 -0
- fetchpdf/retrieval/pdf_identity.py +533 -0
- fetchpdf/retrieval/pdf_text.py +242 -0
- fetchpdf/retrieval/provenance.py +185 -0
- fetchpdf/retrieval/ratelimit.py +113 -0
- fetchpdf/retrieval/repository_waf.py +152 -0
- fetchpdf/retrieval/request_drafts.py +304 -0
- fetchpdf/retrieval/resolve.py +481 -0
- fetchpdf/retrieval/sources/__init__.py +15 -0
- fetchpdf/retrieval/sources/arxiv.py +102 -0
- fetchpdf/retrieval/sources/core_text.py +64 -0
- fetchpdf/retrieval/sources/crossref_tdm.py +193 -0
- fetchpdf/retrieval/sources/epmc.py +133 -0
- fetchpdf/retrieval/sources/landing.py +34 -0
- fetchpdf/retrieval/sources/legacy_pdf.py +99 -0
- fetchpdf/retrieval/sources/pmc.py +59 -0
- fetchpdf/retrieval/sources/preprint.py +101 -0
- fetchpdf/retrieval/sources/publisher_html.py +154 -0
- fetchpdf/retrieval/sources/supplements.py +183 -0
- fetchpdf/retrieval/supplement_atypon.py +269 -0
- fetchpdf/retrieval/supplement_dspace.py +187 -0
- fetchpdf/retrieval/supplement_graph.py +842 -0
- fetchpdf/retrieval/supplement_index.py +820 -0
- fetchpdf/retrieval/supplement_pmc.py +351 -0
- fetchpdf/retrieval/supplement_publishers.py +374 -0
- fetchpdf/retrieval/supplementary.py +1770 -0
- fetchpdf/retrieval/tiers.py +213 -0
- fetchpdf/retrieval/to_markdown.py +661 -0
- fetchpdf/retrieval/validate.py +368 -0
- fetchpdf/retrieval/verify.py +149 -0
- fetchpdf-0.1.0.dist-info/METADATA +1392 -0
- fetchpdf-0.1.0.dist-info/RECORD +64 -0
- fetchpdf-0.1.0.dist-info/WHEEL +5 -0
- fetchpdf-0.1.0.dist-info/entry_points.txt +5 -0
- fetchpdf-0.1.0.dist-info/licenses/LICENSE +21 -0
- fetchpdf-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"""Publisher supplementary routes that work over plain HTTP.
|
|
2
|
+
|
|
3
|
+
Which publishers those are is not a matter of taste. Verified with a browser
|
|
4
|
+
User-Agent on 2026-07-29:
|
|
5
|
+
|
|
6
|
+
reachable Springer/Nature (media.springernature.com, static-content.springer.com,
|
|
7
|
+
www.nature.com article pages), PLOS (journals.plos.org article/file),
|
|
8
|
+
bioRxiv/medRxiv (.supplementary-material), OUP/Silverchair, supp.apa.org
|
|
9
|
+
403/blocked PNAS, Science, Taylor & Francis, Sage (all Atypon), Wiley,
|
|
10
|
+
ScienceDirect, MDPI -- Cloudflare interstitials
|
|
11
|
+
worse ACS returns 404 *with* a 57 KB HTML body, so its 404 is not an answer
|
|
12
|
+
|
|
13
|
+
Only the reachable ones live here. The blocked families need a browser and are
|
|
14
|
+
Phase 2, gated on the existing --add-playwright rather than a new switch.
|
|
15
|
+
|
|
16
|
+
The one exception to "Elsevier is blocked" is worth its own note: ScienceDirect
|
|
17
|
+
pages are walled, but the Elsevier object API is not, and it names supplementary
|
|
18
|
+
material outright. Verified on S2589004225028421: 25 choices, 24 of them
|
|
19
|
+
IMAGE-{DOWNSAMPLED,THUMBNAIL,HIGH-RES}, and one @type "APPLICATION" whose $ is
|
|
20
|
+
the mmc1.pdf download URL. @type == "APPLICATION" *is* the marker.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
from typing import List, Optional
|
|
25
|
+
|
|
26
|
+
from .._env import ELSEVIER_TDM_API_KEY
|
|
27
|
+
from .supplement_index import ROLE_SUPPLEMENT, SupplementFile, classify_role, jats_sets
|
|
28
|
+
|
|
29
|
+
ELSEVIER_OBJECT = "https://api.elsevier.com/content/object/pii/{pii}"
|
|
30
|
+
ELSEVIER_OBJECT_BY_DOI = "https://api.elsevier.com/content/object/doi/{doi}"
|
|
31
|
+
PLOS_FILE = "https://journals.plos.org/{slug}/article/file"
|
|
32
|
+
BIORXIV_DETAILS = "https://api.biorxiv.org/details/{server}/{doi}"
|
|
33
|
+
BIORXIV_SUPPL = "https://www.{server}.org/content/{doi}v{version}.supplementary-material"
|
|
34
|
+
APA_SUPPL = "https://supp.apa.org/psycarticles/supplemental/{code}/{code}_supp.html"
|
|
35
|
+
|
|
36
|
+
#: Nature/Springer electronic supplementary material, on either CDN host.
|
|
37
|
+
_SPRINGER_ESM_RE = re.compile(
|
|
38
|
+
r"https://(?:media\.springernature\.com/original/springer-static"
|
|
39
|
+
r"|static-content\.springer\.com)"
|
|
40
|
+
r"/esm/[^\"'\s>]+?/MediaObjects/[^\"'\s>]+?_MOESM\d+_ESM\.\w+",
|
|
41
|
+
re.IGNORECASE,
|
|
42
|
+
)
|
|
43
|
+
_SPRINGER_PREFIXES = ("10.1038", "10.1007", "10.1186", "10.1140", "10.1057", "10.1245")
|
|
44
|
+
|
|
45
|
+
#: PLOS journal slugs, keyed on the DOI's journal stem.
|
|
46
|
+
_PLOS_SLUGS = {
|
|
47
|
+
"pone": "plosone", "pbio": "plosbiology", "pmed": "plosmedicine",
|
|
48
|
+
"pcbi": "ploscompbiol", "pgen": "plosgenetics", "ppat": "plospathogens",
|
|
49
|
+
"pntd": "plosntds", "pstr": "plossustain", "pclm": "climate",
|
|
50
|
+
"pwat": "water", "pdig": "digitalhealth", "pgph": "globalpublichealth",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
_BIORXIV_MEDIA_RE = re.compile(
|
|
54
|
+
r"/content/(?:bio|med)rxiv/early/\d{4}/\d{2}/\d{2}/[^\"'\s>]+?/DC\d+/embed/"
|
|
55
|
+
r"media-\d+\.\w+(?:\?download=true)?",
|
|
56
|
+
re.IGNORECASE,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
_APA_HREF_RE = re.compile(r"""href=["']([^"'>]+)["']""", re.IGNORECASE)
|
|
60
|
+
|
|
61
|
+
_MAX_PER_PUBLISHER = 40
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# -- E10: Elsevier object API -----------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def enumerate_elsevier_objects(ids, ctx) -> List[SupplementFile]:
|
|
68
|
+
"""Attachments named by the Elsevier object API.
|
|
69
|
+
|
|
70
|
+
The only route into Elsevier supplementary material that does not need a
|
|
71
|
+
browser. Non-entitled requests come back 200 with an error payload rather
|
|
72
|
+
than a 4xx, so the payload, not the status, decides.
|
|
73
|
+
"""
|
|
74
|
+
# The DOI route is not a fallback for missing PIIs, it is the more reliable
|
|
75
|
+
# of the two: a PII resolved from Crossref can be stale or simply wrong
|
|
76
|
+
# (S0749597813001040 for 10.1016/j.obhdp.2013.09.001 is a 404), while the
|
|
77
|
+
# DOI form answers 300 with the full choice list including the mmc1
|
|
78
|
+
# supplement. Try the PII first only because it costs the same and keeps
|
|
79
|
+
# the existing behaviour where it already works.
|
|
80
|
+
for url in _elsevier_object_urls(ids):
|
|
81
|
+
files = _elsevier_objects_at(url, ids, ctx)
|
|
82
|
+
if files:
|
|
83
|
+
return files
|
|
84
|
+
return []
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _elsevier_object_urls(ids) -> List[str]:
|
|
88
|
+
urls = []
|
|
89
|
+
if ids.elsevier_pii:
|
|
90
|
+
urls.append(ELSEVIER_OBJECT.format(pii=ids.elsevier_pii))
|
|
91
|
+
if ids.doi:
|
|
92
|
+
from urllib.parse import quote
|
|
93
|
+
urls.append(ELSEVIER_OBJECT_BY_DOI.format(doi=quote(ids.doi, safe="")))
|
|
94
|
+
return urls
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _elsevier_objects_at(url: str, ids, ctx) -> List[SupplementFile]:
|
|
98
|
+
"""One object-API lookup, by whichever identifier the URL carries."""
|
|
99
|
+
response = ctx.http.get(
|
|
100
|
+
url,
|
|
101
|
+
headers={"X-ELS-APIKey": ELSEVIER_TDM_API_KEY, "Accept": "application/json"},
|
|
102
|
+
timeout=30,
|
|
103
|
+
polite=False,
|
|
104
|
+
)
|
|
105
|
+
# HTTP 300 Multiple Choices is this endpoint's SUCCESS response -- it is
|
|
106
|
+
# literally a list of the representations on offer, which is what we came
|
|
107
|
+
# for. Response.ok is 200 <= status < 300, so `if not response.ok` rejected
|
|
108
|
+
# every Elsevier article ever passed to this provider. Six records in one
|
|
109
|
+
# 46-paper corpus had supplements that were silently dropped this way.
|
|
110
|
+
if not response.ok and response.status != 300:
|
|
111
|
+
return []
|
|
112
|
+
try:
|
|
113
|
+
payload = response.json() or {}
|
|
114
|
+
except ValueError:
|
|
115
|
+
return []
|
|
116
|
+
if "service-error" in payload or "error-response" in payload:
|
|
117
|
+
ctx.log(" Elsevier objects: 200 with an error payload (not entitled)")
|
|
118
|
+
return []
|
|
119
|
+
|
|
120
|
+
entries = _elsevier_choices(payload)
|
|
121
|
+
files = []
|
|
122
|
+
for index, choice in enumerate(entries):
|
|
123
|
+
if str(choice.get("@type") or "").upper() != "APPLICATION":
|
|
124
|
+
continue
|
|
125
|
+
url = choice.get("$") or choice.get("@href")
|
|
126
|
+
if not url:
|
|
127
|
+
continue
|
|
128
|
+
ref = str(choice.get("@ref") or f"mmc{index + 1}")
|
|
129
|
+
files.append(SupplementFile(
|
|
130
|
+
name=_elsevier_name(ref, url),
|
|
131
|
+
url=url,
|
|
132
|
+
provider="elsevier_objects",
|
|
133
|
+
listing_index=index,
|
|
134
|
+
mimetype=choice.get("@mimetype"),
|
|
135
|
+
size_bytes=_as_int(choice.get("@filesize")),
|
|
136
|
+
role=ROLE_SUPPLEMENT,
|
|
137
|
+
origin_doi=ids.doi,
|
|
138
|
+
polite=False,
|
|
139
|
+
extra={"pii": ids.elsevier_pii, "ref": ref},
|
|
140
|
+
))
|
|
141
|
+
if len(files) >= _MAX_PER_PUBLISHER:
|
|
142
|
+
break
|
|
143
|
+
return files
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _elsevier_choices(payload) -> List[dict]:
|
|
147
|
+
choices = ((payload.get("object-response") or payload).get("choices") or {})
|
|
148
|
+
if isinstance(choices, dict):
|
|
149
|
+
choices = choices.get("choice") or []
|
|
150
|
+
if isinstance(choices, dict):
|
|
151
|
+
choices = [choices]
|
|
152
|
+
return [c for c in choices if isinstance(c, dict)]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _elsevier_name(ref: str, url: str) -> str:
|
|
156
|
+
"""mmc1 plus whatever extension the URL admits to."""
|
|
157
|
+
m = re.search(r"-(mmc\d+\.\w+)", url) or re.search(r"/([^/?]+\.\w{1,5})(?:\?|$)", url)
|
|
158
|
+
if m:
|
|
159
|
+
return m.group(1)
|
|
160
|
+
m = re.search(r"httpAccept=\w+/([\w.+-]+)", url)
|
|
161
|
+
return f"{ref}.{m.group(1)}" if m else ref
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# -- E11: Springer / Nature ESM ---------------------------------------------
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def enumerate_springer_esm(ids, ctx) -> List[SupplementFile]:
|
|
168
|
+
"""MOESM objects scraped off the article page.
|
|
169
|
+
|
|
170
|
+
The CDN URL cannot be derived -- the MOESM stem encodes an internal article
|
|
171
|
+
id -- so the page has to be read. Anchors carry data-test="supp-info-link";
|
|
172
|
+
the regex on the CDN hosts is the fallback for pages that do not.
|
|
173
|
+
"""
|
|
174
|
+
if not _has_prefix(ids.doi, _SPRINGER_PREFIXES):
|
|
175
|
+
return []
|
|
176
|
+
response = ctx.http.get(f"https://doi.org/{ids.doi}", timeout=45, polite=False)
|
|
177
|
+
if not response.ok or not response.content:
|
|
178
|
+
return []
|
|
179
|
+
|
|
180
|
+
supplements, figures = jats_sets(ctx)
|
|
181
|
+
files = []
|
|
182
|
+
for index, url in enumerate(_unique(_SPRINGER_ESM_RE.findall(response.text))):
|
|
183
|
+
name = url.rsplit("/", 1)[-1]
|
|
184
|
+
files.append(SupplementFile(
|
|
185
|
+
name=name,
|
|
186
|
+
url=url,
|
|
187
|
+
provider="springer_esm",
|
|
188
|
+
listing_index=index,
|
|
189
|
+
role=classify_role(name, jats_supplements=supplements, jats_figures=figures),
|
|
190
|
+
origin_doi=ids.doi,
|
|
191
|
+
polite=False,
|
|
192
|
+
))
|
|
193
|
+
if len(files) >= _MAX_PER_PUBLISHER:
|
|
194
|
+
break
|
|
195
|
+
return files
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# -- E12: PLOS --------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def enumerate_plos(ids, ctx) -> List[SupplementFile]:
|
|
202
|
+
"""PLOS supplementary files, enumerated rather than probed.
|
|
203
|
+
|
|
204
|
+
The sNNN ids come from the JATS manifest or from Crossref components. Probing
|
|
205
|
+
s001, s002, ... blindly would work and would also hammer the host for every
|
|
206
|
+
article that has none, so it is not done.
|
|
207
|
+
"""
|
|
208
|
+
if not ids.doi or not ids.doi.startswith("10.1371"):
|
|
209
|
+
return []
|
|
210
|
+
slug = _PLOS_SLUGS.get(_plos_stem(ids.doi))
|
|
211
|
+
if not slug:
|
|
212
|
+
return []
|
|
213
|
+
|
|
214
|
+
supplements, _ = jats_sets(ctx)
|
|
215
|
+
ids_seen = sorted({
|
|
216
|
+
m.group(1)
|
|
217
|
+
for name in supplements
|
|
218
|
+
for m in [re.search(r"\.(s\d{3})\b", name)]
|
|
219
|
+
if m
|
|
220
|
+
})
|
|
221
|
+
if not ids_seen:
|
|
222
|
+
return []
|
|
223
|
+
|
|
224
|
+
files = []
|
|
225
|
+
for index, suffix in enumerate(ids_seen):
|
|
226
|
+
files.append(SupplementFile(
|
|
227
|
+
name=f"{ids.doi.rsplit('/', 1)[-1]}.{suffix}",
|
|
228
|
+
url=PLOS_FILE.format(slug=slug) + f"?id={ids.doi}.{suffix}&type=supplementary",
|
|
229
|
+
provider="plos",
|
|
230
|
+
listing_index=index,
|
|
231
|
+
role=ROLE_SUPPLEMENT,
|
|
232
|
+
origin_doi=ids.doi,
|
|
233
|
+
polite=False,
|
|
234
|
+
))
|
|
235
|
+
return files
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _plos_stem(doi: str) -> str:
|
|
239
|
+
m = re.search(r"journal\.([a-z]+)\.", doi, re.IGNORECASE)
|
|
240
|
+
return m.group(1).lower() if m else ""
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# -- E13: bioRxiv / medRxiv -------------------------------------------------
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def enumerate_preprint_supplements(ids, ctx) -> List[SupplementFile]:
|
|
247
|
+
"""Preprint supplementary media, from the versioned supplement page.
|
|
248
|
+
|
|
249
|
+
A nonexistent DOI or version 302s to /node/ and answers 200 with a generic
|
|
250
|
+
page, so the final URL is checked for the DOI before anything is parsed.
|
|
251
|
+
"""
|
|
252
|
+
if not ids.doi or not ids.doi.startswith("10.1101"):
|
|
253
|
+
return []
|
|
254
|
+
server = (ids.preprint_server or "").lower()
|
|
255
|
+
servers = [server] if server in ("biorxiv", "medrxiv") else ["biorxiv", "medrxiv"]
|
|
256
|
+
|
|
257
|
+
for candidate in servers:
|
|
258
|
+
version = _preprint_version(ids, ctx, candidate)
|
|
259
|
+
if version is None:
|
|
260
|
+
continue
|
|
261
|
+
url = BIORXIV_SUPPL.format(server=candidate, doi=ids.doi, version=version)
|
|
262
|
+
response = ctx.http.get(url, timeout=45, polite=False)
|
|
263
|
+
if not response.ok or not response.content:
|
|
264
|
+
continue
|
|
265
|
+
if ids.doi.lower() not in (response.url or "").lower():
|
|
266
|
+
ctx.log(f" {candidate}: redirected away from the DOI, not parsing")
|
|
267
|
+
continue
|
|
268
|
+
files = []
|
|
269
|
+
for index, path in enumerate(_unique(_BIORXIV_MEDIA_RE.findall(response.text))):
|
|
270
|
+
files.append(SupplementFile(
|
|
271
|
+
name=path.split("/")[-1].split("?")[0],
|
|
272
|
+
url=f"https://www.{candidate}.org{path}",
|
|
273
|
+
provider="preprint_supplements",
|
|
274
|
+
listing_index=index,
|
|
275
|
+
role=ROLE_SUPPLEMENT,
|
|
276
|
+
origin_doi=ids.doi,
|
|
277
|
+
polite=False,
|
|
278
|
+
))
|
|
279
|
+
if len(files) >= _MAX_PER_PUBLISHER:
|
|
280
|
+
break
|
|
281
|
+
if files:
|
|
282
|
+
return files
|
|
283
|
+
return []
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _preprint_version(ids, ctx, server) -> Optional[int]:
|
|
287
|
+
response = ctx.http.get(
|
|
288
|
+
BIORXIV_DETAILS.format(server=server, doi=ids.doi), timeout=30, polite=False
|
|
289
|
+
)
|
|
290
|
+
if not response.ok:
|
|
291
|
+
return None
|
|
292
|
+
try:
|
|
293
|
+
collection = (response.json() or {}).get("collection") or []
|
|
294
|
+
except ValueError:
|
|
295
|
+
return None
|
|
296
|
+
if not collection:
|
|
297
|
+
# A 200 with an empty collection is how the wrong server answers.
|
|
298
|
+
return None
|
|
299
|
+
versions = [_as_int(entry.get("version")) for entry in collection
|
|
300
|
+
if isinstance(entry, dict)]
|
|
301
|
+
versions = [v for v in versions if v]
|
|
302
|
+
return max(versions) if versions else 1
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# -- E14: APA supplemental --------------------------------------------------
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def enumerate_apa_supplemental(ids, ctx) -> List[SupplementFile]:
|
|
309
|
+
"""The APA supplemental index page.
|
|
310
|
+
|
|
311
|
+
try_apa_supplemental_fallback in the legacy chain scrapes the same page, but
|
|
312
|
+
it is built for a different job -- it returns a bool, writes one PDF to
|
|
313
|
+
save_path, stops at the first success, and runs .doc files through
|
|
314
|
+
LibreOffice. Here every file is wanted and wanted in its deposited format, so
|
|
315
|
+
the scraping is lifted rather than the function reused; the legacy one keeps
|
|
316
|
+
working unchanged for its own purpose.
|
|
317
|
+
"""
|
|
318
|
+
if not ids.doi or not ids.doi.startswith("10.1037"):
|
|
319
|
+
return []
|
|
320
|
+
code = ids.doi.rsplit("/", 1)[-1].strip().lower()
|
|
321
|
+
if not code:
|
|
322
|
+
return []
|
|
323
|
+
|
|
324
|
+
response = ctx.http.get(APA_SUPPL.format(code=code), timeout=30, polite=False)
|
|
325
|
+
if not response.ok or not response.content:
|
|
326
|
+
return []
|
|
327
|
+
|
|
328
|
+
base = f"https://supp.apa.org/psycarticles/supplemental/{code}/"
|
|
329
|
+
files = []
|
|
330
|
+
for index, href in enumerate(_unique(_APA_HREF_RE.findall(response.text))):
|
|
331
|
+
if href.startswith("#") or href.lower().startswith(("mailto:", "javascript:")):
|
|
332
|
+
continue
|
|
333
|
+
name = href.split("?")[0].rsplit("/", 1)[-1]
|
|
334
|
+
if not name or "." not in name or name.lower().endswith((".html", ".htm", ".css", ".js")):
|
|
335
|
+
continue
|
|
336
|
+
url = href if href.startswith("http") else base + href.lstrip("/")
|
|
337
|
+
files.append(SupplementFile(
|
|
338
|
+
name=name,
|
|
339
|
+
url=url,
|
|
340
|
+
provider="apa_supplemental",
|
|
341
|
+
listing_index=index,
|
|
342
|
+
role=ROLE_SUPPLEMENT,
|
|
343
|
+
origin_doi=ids.doi,
|
|
344
|
+
polite=False,
|
|
345
|
+
))
|
|
346
|
+
if len(files) >= _MAX_PER_PUBLISHER:
|
|
347
|
+
break
|
|
348
|
+
return files
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
# -- shared -----------------------------------------------------------------
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _has_prefix(doi: Optional[str], prefixes) -> bool:
|
|
355
|
+
return bool(doi) and any(str(doi).startswith(p) for p in prefixes)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _unique(values) -> List[str]:
|
|
359
|
+
"""Order-preserving dedupe: a page lists the same href more than once."""
|
|
360
|
+
seen = set()
|
|
361
|
+
out = []
|
|
362
|
+
for value in values:
|
|
363
|
+
if value not in seen:
|
|
364
|
+
seen.add(value)
|
|
365
|
+
out.append(value)
|
|
366
|
+
return out
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _as_int(value) -> Optional[int]:
|
|
370
|
+
try:
|
|
371
|
+
parsed = int(value)
|
|
372
|
+
except (TypeError, ValueError):
|
|
373
|
+
return None
|
|
374
|
+
return parsed if parsed > 0 else None
|