litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""HTML extractor for Nature/Springer articles."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from bs4 import BeautifulSoup
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def can_handle(url: str) -> bool:
|
|
10
|
+
"""Check if this adapter can handle the given URL."""
|
|
11
|
+
return any(
|
|
12
|
+
domain in url.lower()
|
|
13
|
+
for domain in ["nature.com", "springer.com", "springerlink.com"]
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def extract(html: str, url: str = "") -> dict:
|
|
18
|
+
"""Extract paper content from Nature/Springer HTML."""
|
|
19
|
+
soup = BeautifulSoup(html, "lxml")
|
|
20
|
+
|
|
21
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
22
|
+
tag.decompose()
|
|
23
|
+
|
|
24
|
+
title = _extract_title(soup)
|
|
25
|
+
authors = _extract_authors(soup)
|
|
26
|
+
abstract = _extract_abstract(soup)
|
|
27
|
+
full_text = _extract_body(soup)
|
|
28
|
+
figures = _extract_figures(soup)
|
|
29
|
+
references = _extract_references(soup)
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
"title": title,
|
|
33
|
+
"authors": authors,
|
|
34
|
+
"abstract": abstract,
|
|
35
|
+
"full_text": full_text,
|
|
36
|
+
"figures": figures,
|
|
37
|
+
"references": references,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
42
|
+
for selector in [
|
|
43
|
+
"h1.c-article-title",
|
|
44
|
+
"h1.article-item__title",
|
|
45
|
+
"meta[name='citation_title']",
|
|
46
|
+
"meta[property='og:title']",
|
|
47
|
+
]:
|
|
48
|
+
el = soup.select_one(selector)
|
|
49
|
+
if el:
|
|
50
|
+
if el.name == "meta":
|
|
51
|
+
return el.get("content", "").strip()
|
|
52
|
+
return el.get_text(strip=True)
|
|
53
|
+
return ""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
57
|
+
"""Extract authors using three-layer pipeline: meta -> JSON-LD -> DOM."""
|
|
58
|
+
# Layer 1: meta tags (most reliable)
|
|
59
|
+
authors = []
|
|
60
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
61
|
+
name = meta.get("content", "").strip()
|
|
62
|
+
if name:
|
|
63
|
+
authors.append(name)
|
|
64
|
+
if authors:
|
|
65
|
+
return authors
|
|
66
|
+
|
|
67
|
+
# Layer 2: JSON-LD
|
|
68
|
+
authors = _authors_from_jsonld(soup)
|
|
69
|
+
if authors:
|
|
70
|
+
return authors
|
|
71
|
+
|
|
72
|
+
# Layer 3: DOM selectors
|
|
73
|
+
for el in soup.select("li.c-article-author-list__item a"):
|
|
74
|
+
name = el.get_text(strip=True)
|
|
75
|
+
if name:
|
|
76
|
+
authors.append(name)
|
|
77
|
+
if not authors:
|
|
78
|
+
for el in soup.select("[data-test='author-name']"):
|
|
79
|
+
name = el.get_text(strip=True)
|
|
80
|
+
if name:
|
|
81
|
+
authors.append(name)
|
|
82
|
+
return authors
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _authors_from_jsonld(soup: BeautifulSoup) -> list[str]:
|
|
86
|
+
"""Extract authors from JSON-LD structured data."""
|
|
87
|
+
for script in soup.select("script[type='application/ld+json']"):
|
|
88
|
+
try:
|
|
89
|
+
data = json.loads(script.string or "")
|
|
90
|
+
except (json.JSONDecodeError, TypeError):
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
# Handle both single object and list
|
|
94
|
+
items = data if isinstance(data, list) else [data]
|
|
95
|
+
for item in items:
|
|
96
|
+
author = item.get("author", [])
|
|
97
|
+
if isinstance(author, dict):
|
|
98
|
+
author = [author]
|
|
99
|
+
names = []
|
|
100
|
+
for a in author:
|
|
101
|
+
if isinstance(a, dict):
|
|
102
|
+
name = a.get("name", "")
|
|
103
|
+
if name:
|
|
104
|
+
names.append(name)
|
|
105
|
+
if names:
|
|
106
|
+
return names
|
|
107
|
+
return []
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
111
|
+
for selector in [
|
|
112
|
+
"#Abs1-content",
|
|
113
|
+
"div.c-article-section__content[id*='Abs']",
|
|
114
|
+
"#abstract",
|
|
115
|
+
"section[data-title='Abstract']",
|
|
116
|
+
"div.article__body p.article__teaser",
|
|
117
|
+
"meta[name='description']",
|
|
118
|
+
"meta[property='og:description']",
|
|
119
|
+
]:
|
|
120
|
+
el = soup.select_one(selector)
|
|
121
|
+
if el:
|
|
122
|
+
if el.name == "meta":
|
|
123
|
+
return el.get("content", "").strip()
|
|
124
|
+
return _clean(el.get_text())
|
|
125
|
+
return ""
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
129
|
+
parts = []
|
|
130
|
+
|
|
131
|
+
# Nature uses section[data-title] for article sections
|
|
132
|
+
sections = soup.select("section[data-title]")
|
|
133
|
+
if sections:
|
|
134
|
+
for section in sections:
|
|
135
|
+
title = section.get("data-title", "")
|
|
136
|
+
if title.lower() in ("abstract", "references", "supplementary information", "data availability"):
|
|
137
|
+
continue
|
|
138
|
+
content_div = section.select_one(".c-article-section__content")
|
|
139
|
+
if content_div:
|
|
140
|
+
# Extract text and tables
|
|
141
|
+
text_parts = []
|
|
142
|
+
for child in content_div.children:
|
|
143
|
+
if child.name == "table":
|
|
144
|
+
table_text = _extract_table(child)
|
|
145
|
+
if table_text:
|
|
146
|
+
text_parts.append(table_text)
|
|
147
|
+
elif child.name:
|
|
148
|
+
text = _clean(child.get_text())
|
|
149
|
+
if text:
|
|
150
|
+
text_parts.append(text)
|
|
151
|
+
content = "\n\n".join(text_parts)
|
|
152
|
+
if content:
|
|
153
|
+
parts.append(f"## {title}\n\n{content}")
|
|
154
|
+
|
|
155
|
+
# Fallback: try article body div
|
|
156
|
+
if not parts:
|
|
157
|
+
body = soup.select_one("div.c-article-body") or soup.select_one("div.article__body")
|
|
158
|
+
if body:
|
|
159
|
+
parts.append(_clean(body.get_text()))
|
|
160
|
+
|
|
161
|
+
return "\n\n".join(parts)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _extract_table(table) -> str:
|
|
165
|
+
"""Extract table content as structured text."""
|
|
166
|
+
rows = []
|
|
167
|
+
for tr in table.find_all("tr"):
|
|
168
|
+
cells = []
|
|
169
|
+
for td in tr.find_all(["td", "th"]):
|
|
170
|
+
cells.append(_clean(td.get_text()))
|
|
171
|
+
if any(cells):
|
|
172
|
+
rows.append(" | ".join(cells))
|
|
173
|
+
return "\n".join(rows)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
177
|
+
captions = []
|
|
178
|
+
for fig in soup.select("figure"):
|
|
179
|
+
cap = fig.select_one("figcaption")
|
|
180
|
+
if cap:
|
|
181
|
+
text = _clean(cap.get_text())
|
|
182
|
+
if text and len(text) > 10:
|
|
183
|
+
captions.append(text)
|
|
184
|
+
return captions
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
188
|
+
refs = []
|
|
189
|
+
ref_section = soup.select_one("#Bib1") or soup.select_one("#references")
|
|
190
|
+
if ref_section:
|
|
191
|
+
for li in ref_section.select("li.c-article-references__item"):
|
|
192
|
+
text = _clean(li.get_text())
|
|
193
|
+
if text:
|
|
194
|
+
refs.append(text)
|
|
195
|
+
if not refs:
|
|
196
|
+
for li in ref_section.find_all("li"):
|
|
197
|
+
text = _clean(li.get_text())
|
|
198
|
+
if text and len(text) > 20:
|
|
199
|
+
refs.append(text)
|
|
200
|
+
return refs
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _clean(text: str) -> str:
|
|
204
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""HTML extractor for RSC (Royal Society of Chemistry) articles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def can_handle(url: str) -> bool:
|
|
9
|
+
"""Check if this adapter can handle the given URL."""
|
|
10
|
+
return "pubs.rsc.org" in url.lower()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract(html: str, url: str = "") -> dict:
|
|
14
|
+
"""Extract paper content from RSC Publications HTML."""
|
|
15
|
+
soup = BeautifulSoup(html, "lxml")
|
|
16
|
+
|
|
17
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
18
|
+
tag.decompose()
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
"title": _extract_title(soup),
|
|
22
|
+
"authors": _extract_authors(soup),
|
|
23
|
+
"abstract": _extract_abstract(soup),
|
|
24
|
+
"full_text": _extract_body(soup),
|
|
25
|
+
"figures": _extract_figures(soup),
|
|
26
|
+
"references": _extract_references(soup),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
31
|
+
for selector in [
|
|
32
|
+
"h1.headline",
|
|
33
|
+
"h1.title",
|
|
34
|
+
"meta[name='citation_title']",
|
|
35
|
+
"meta[property='og:title']",
|
|
36
|
+
]:
|
|
37
|
+
el = soup.select_one(selector)
|
|
38
|
+
if el:
|
|
39
|
+
if el.name == "meta":
|
|
40
|
+
return el.get("content", "").strip()
|
|
41
|
+
return el.get_text(strip=True)
|
|
42
|
+
return ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
46
|
+
authors = []
|
|
47
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
48
|
+
name = meta.get("content", "").strip()
|
|
49
|
+
if name:
|
|
50
|
+
authors.append(name)
|
|
51
|
+
if not authors:
|
|
52
|
+
for el in soup.select(".article__authors a, .authors-list a"):
|
|
53
|
+
name = el.get_text(strip=True)
|
|
54
|
+
if name and len(name) > 1:
|
|
55
|
+
authors.append(name)
|
|
56
|
+
return authors
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
60
|
+
for selector in [
|
|
61
|
+
"div.article__abstract",
|
|
62
|
+
"div.abstract",
|
|
63
|
+
"#abstract",
|
|
64
|
+
"meta[name='description']",
|
|
65
|
+
]:
|
|
66
|
+
el = soup.select_one(selector)
|
|
67
|
+
if el:
|
|
68
|
+
if el.name == "meta":
|
|
69
|
+
return el.get("content", "").strip()
|
|
70
|
+
return _clean(el.get_text())
|
|
71
|
+
return ""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
75
|
+
parts = []
|
|
76
|
+
|
|
77
|
+
body = soup.select_one("div#article-body") or soup.select_one("div.article__body")
|
|
78
|
+
if body:
|
|
79
|
+
for section in body.find_all(["section", "div"], recursive=False):
|
|
80
|
+
heading = section.find(re.compile(r"h[2-4]"))
|
|
81
|
+
heading_text = heading.get_text(strip=True) if heading else ""
|
|
82
|
+
level = int(heading.name[1]) if heading else 2
|
|
83
|
+
|
|
84
|
+
if heading_text.lower() in ("abstract", "references", "notes and references"):
|
|
85
|
+
continue
|
|
86
|
+
|
|
87
|
+
text_parts = []
|
|
88
|
+
for child in section.children:
|
|
89
|
+
if child.name:
|
|
90
|
+
if child.name == "table":
|
|
91
|
+
table_text = _extract_table(child)
|
|
92
|
+
if table_text:
|
|
93
|
+
text_parts.append(table_text)
|
|
94
|
+
else:
|
|
95
|
+
text = _clean(child.get_text())
|
|
96
|
+
if text:
|
|
97
|
+
text_parts.append(text)
|
|
98
|
+
|
|
99
|
+
content = "\n\n".join(text_parts)
|
|
100
|
+
if heading_text and content:
|
|
101
|
+
prefix = "#" * min(level, 4)
|
|
102
|
+
parts.append(f"{prefix} {heading_text}\n\n{content}")
|
|
103
|
+
elif content:
|
|
104
|
+
parts.append(content)
|
|
105
|
+
|
|
106
|
+
if not parts and body:
|
|
107
|
+
parts.append(_clean(body.get_text()))
|
|
108
|
+
|
|
109
|
+
if not parts:
|
|
110
|
+
article = soup.select_one("article")
|
|
111
|
+
if article:
|
|
112
|
+
parts.append(_clean(article.get_text()))
|
|
113
|
+
|
|
114
|
+
return "\n\n".join(parts)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _extract_table(table) -> str:
|
|
118
|
+
rows = []
|
|
119
|
+
for tr in table.find_all("tr"):
|
|
120
|
+
cells = []
|
|
121
|
+
for td in tr.find_all(["td", "th"]):
|
|
122
|
+
cells.append(_clean(td.get_text()))
|
|
123
|
+
if any(cells):
|
|
124
|
+
rows.append(" | ".join(cells))
|
|
125
|
+
return "\n".join(rows)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
129
|
+
captions = []
|
|
130
|
+
for fig in soup.select("figure"):
|
|
131
|
+
cap = fig.select_one("figcaption")
|
|
132
|
+
if cap:
|
|
133
|
+
text = _clean(cap.get_text())
|
|
134
|
+
if text and len(text) > 10:
|
|
135
|
+
captions.append(text)
|
|
136
|
+
return captions
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
140
|
+
refs = []
|
|
141
|
+
for selector in ["div.references", "section.references", "#references"]:
|
|
142
|
+
ref_section = soup.select_one(selector)
|
|
143
|
+
if ref_section:
|
|
144
|
+
for li in ref_section.find_all("li"):
|
|
145
|
+
text = _clean(li.get_text())
|
|
146
|
+
if text and len(text) > 20:
|
|
147
|
+
refs.append(text)
|
|
148
|
+
if refs:
|
|
149
|
+
return refs
|
|
150
|
+
return refs
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _clean(text: str) -> str:
|
|
154
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""HTML extractor for Taylor & Francis articles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def can_handle(url: str) -> bool:
|
|
9
|
+
"""Check if this adapter can handle the given URL."""
|
|
10
|
+
return "tandfonline.com" in url.lower()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract(html: str, url: str = "") -> dict:
|
|
14
|
+
"""Extract paper content from Taylor & Francis HTML."""
|
|
15
|
+
soup = BeautifulSoup(html, "lxml")
|
|
16
|
+
|
|
17
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
18
|
+
tag.decompose()
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
"title": _extract_title(soup),
|
|
22
|
+
"authors": _extract_authors(soup),
|
|
23
|
+
"abstract": _extract_abstract(soup),
|
|
24
|
+
"full_text": _extract_body(soup),
|
|
25
|
+
"figures": _extract_figures(soup),
|
|
26
|
+
"references": _extract_references(soup),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
31
|
+
for selector in [
|
|
32
|
+
"h1.article-header__title",
|
|
33
|
+
"h1.title",
|
|
34
|
+
"meta[name='citation_title']",
|
|
35
|
+
"meta[property='og:title']",
|
|
36
|
+
]:
|
|
37
|
+
el = soup.select_one(selector)
|
|
38
|
+
if el:
|
|
39
|
+
if el.name == "meta":
|
|
40
|
+
return el.get("content", "").strip()
|
|
41
|
+
return el.get_text(strip=True)
|
|
42
|
+
return ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
46
|
+
authors = []
|
|
47
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
48
|
+
name = meta.get("content", "").strip()
|
|
49
|
+
if name:
|
|
50
|
+
authors.append(name)
|
|
51
|
+
if not authors:
|
|
52
|
+
for el in soup.select(".contrib-group .contrib, .author-name, a.author"):
|
|
53
|
+
name = el.get_text(strip=True)
|
|
54
|
+
if name and len(name) > 1:
|
|
55
|
+
authors.append(name)
|
|
56
|
+
return authors
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
60
|
+
for selector in [
|
|
61
|
+
"div.abstract",
|
|
62
|
+
"div.abstractSection",
|
|
63
|
+
"#abstract",
|
|
64
|
+
"meta[name='description']",
|
|
65
|
+
]:
|
|
66
|
+
el = soup.select_one(selector)
|
|
67
|
+
if el:
|
|
68
|
+
if el.name == "meta":
|
|
69
|
+
return el.get("content", "").strip()
|
|
70
|
+
return _clean(el.get_text())
|
|
71
|
+
return ""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
75
|
+
parts = []
|
|
76
|
+
|
|
77
|
+
body = soup.select_one("div.article-body") or soup.select_one("div.article_body")
|
|
78
|
+
if body:
|
|
79
|
+
for section in body.find_all(["section", "div"], recursive=False):
|
|
80
|
+
heading = section.find(re.compile(r"h[2-4]"))
|
|
81
|
+
heading_text = heading.get_text(strip=True) if heading else ""
|
|
82
|
+
level = int(heading.name[1]) if heading else 2
|
|
83
|
+
|
|
84
|
+
if heading_text.lower() in ("abstract", "references", "acknowledgments", "acknowledgements"):
|
|
85
|
+
continue
|
|
86
|
+
|
|
87
|
+
text_parts = []
|
|
88
|
+
for child in section.children:
|
|
89
|
+
if child.name:
|
|
90
|
+
if child.name == "table":
|
|
91
|
+
table_text = _extract_table(child)
|
|
92
|
+
if table_text:
|
|
93
|
+
text_parts.append(table_text)
|
|
94
|
+
else:
|
|
95
|
+
text = _clean(child.get_text())
|
|
96
|
+
if text:
|
|
97
|
+
text_parts.append(text)
|
|
98
|
+
|
|
99
|
+
content = "\n\n".join(text_parts)
|
|
100
|
+
if heading_text and content:
|
|
101
|
+
prefix = "#" * min(level, 4)
|
|
102
|
+
parts.append(f"{prefix} {heading_text}\n\n{content}")
|
|
103
|
+
elif content:
|
|
104
|
+
parts.append(content)
|
|
105
|
+
|
|
106
|
+
if not parts and body:
|
|
107
|
+
parts.append(_clean(body.get_text()))
|
|
108
|
+
|
|
109
|
+
if not parts:
|
|
110
|
+
article = soup.select_one("article")
|
|
111
|
+
if article:
|
|
112
|
+
parts.append(_clean(article.get_text()))
|
|
113
|
+
|
|
114
|
+
return "\n\n".join(parts)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _extract_table(table) -> str:
|
|
118
|
+
rows = []
|
|
119
|
+
for tr in table.find_all("tr"):
|
|
120
|
+
cells = []
|
|
121
|
+
for td in tr.find_all(["td", "th"]):
|
|
122
|
+
cells.append(_clean(td.get_text()))
|
|
123
|
+
if any(cells):
|
|
124
|
+
rows.append(" | ".join(cells))
|
|
125
|
+
return "\n".join(rows)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
129
|
+
captions = []
|
|
130
|
+
for fig in soup.select("figure"):
|
|
131
|
+
cap = fig.select_one("figcaption")
|
|
132
|
+
if cap:
|
|
133
|
+
text = _clean(cap.get_text())
|
|
134
|
+
if text and len(text) > 10:
|
|
135
|
+
captions.append(text)
|
|
136
|
+
return captions
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
140
|
+
refs = []
|
|
141
|
+
for selector in ["div.ref-list", "section.references", "#references", "div.references"]:
|
|
142
|
+
ref_section = soup.select_one(selector)
|
|
143
|
+
if ref_section:
|
|
144
|
+
for li in ref_section.find_all("li"):
|
|
145
|
+
text = _clean(li.get_text())
|
|
146
|
+
if text and len(text) > 20:
|
|
147
|
+
refs.append(text)
|
|
148
|
+
if refs:
|
|
149
|
+
return refs
|
|
150
|
+
return refs
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _clean(text: str) -> str:
|
|
154
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""HTML extractor for Wiley Online Library articles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def can_handle(url: str) -> bool:
|
|
9
|
+
"""Check if this adapter can handle the given URL."""
|
|
10
|
+
return "wiley.com" in url.lower() or "onlinelibrary.wiley" in url.lower()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract(html: str, url: str = "") -> dict:
|
|
14
|
+
"""Extract paper content from Wiley Online Library HTML."""
|
|
15
|
+
soup = BeautifulSoup(html, "lxml")
|
|
16
|
+
|
|
17
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
18
|
+
tag.decompose()
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
"title": _extract_title(soup),
|
|
22
|
+
"authors": _extract_authors(soup),
|
|
23
|
+
"abstract": _extract_abstract(soup),
|
|
24
|
+
"full_text": _extract_body(soup),
|
|
25
|
+
"figures": _extract_figures(soup),
|
|
26
|
+
"references": _extract_references(soup),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
31
|
+
for selector in [
|
|
32
|
+
"h1.citation__title",
|
|
33
|
+
".article-header__title",
|
|
34
|
+
"meta[name='citation_title']",
|
|
35
|
+
"meta[property='og:title']",
|
|
36
|
+
]:
|
|
37
|
+
el = soup.select_one(selector)
|
|
38
|
+
if el:
|
|
39
|
+
if el.name == "meta":
|
|
40
|
+
return el.get("content", "").strip()
|
|
41
|
+
return el.get_text(strip=True)
|
|
42
|
+
return ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
46
|
+
"""Extract authors using meta tags first, then DOM."""
|
|
47
|
+
authors = []
|
|
48
|
+
# Meta tags (most reliable)
|
|
49
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
50
|
+
name = meta.get("content", "").strip()
|
|
51
|
+
if name:
|
|
52
|
+
authors.append(name)
|
|
53
|
+
if authors:
|
|
54
|
+
return authors
|
|
55
|
+
|
|
56
|
+
# DOM: multiple possible selectors
|
|
57
|
+
for selector in [
|
|
58
|
+
".loa-authors-trunc a.author-name",
|
|
59
|
+
".loa-authors .author-name span",
|
|
60
|
+
"a.author-name",
|
|
61
|
+
"[data-test='author-name']",
|
|
62
|
+
]:
|
|
63
|
+
for el in soup.select(selector):
|
|
64
|
+
name = el.get_text(strip=True)
|
|
65
|
+
if name:
|
|
66
|
+
authors.append(name)
|
|
67
|
+
if authors:
|
|
68
|
+
return authors
|
|
69
|
+
return authors
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
73
|
+
for selector in [
|
|
74
|
+
"section.article-section__abstract",
|
|
75
|
+
"div.abstract-group",
|
|
76
|
+
"#abstract",
|
|
77
|
+
"meta[name='description']",
|
|
78
|
+
]:
|
|
79
|
+
el = soup.select_one(selector)
|
|
80
|
+
if el:
|
|
81
|
+
if el.name == "meta":
|
|
82
|
+
return el.get("content", "").strip()
|
|
83
|
+
return _clean(el.get_text())
|
|
84
|
+
return ""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
88
|
+
parts = []
|
|
89
|
+
|
|
90
|
+
# Wiley uses section.article-section__content divs
|
|
91
|
+
for section in soup.select("section.article-section__content"):
|
|
92
|
+
heading = section.find_previous("h2")
|
|
93
|
+
heading_text = heading.get_text(strip=True) if heading else ""
|
|
94
|
+
|
|
95
|
+
if heading_text.lower() in ("abstract", "references", "supporting information", "data availability"):
|
|
96
|
+
continue
|
|
97
|
+
|
|
98
|
+
# Extract text with table support
|
|
99
|
+
text_parts = []
|
|
100
|
+
for child in section.children:
|
|
101
|
+
if child.name == "table":
|
|
102
|
+
table_text = _extract_table(child)
|
|
103
|
+
if table_text:
|
|
104
|
+
text_parts.append(table_text)
|
|
105
|
+
elif child.name:
|
|
106
|
+
text = _clean(child.get_text())
|
|
107
|
+
if text:
|
|
108
|
+
text_parts.append(text)
|
|
109
|
+
content = "\n\n".join(text_parts)
|
|
110
|
+
|
|
111
|
+
if heading_text and content:
|
|
112
|
+
parts.append(f"## {heading_text}\n\n{content}")
|
|
113
|
+
elif content:
|
|
114
|
+
parts.append(content)
|
|
115
|
+
|
|
116
|
+
# Fallback
|
|
117
|
+
if not parts:
|
|
118
|
+
body = soup.select_one("article.article__body") or soup.select_one(".article-body-section")
|
|
119
|
+
if body:
|
|
120
|
+
parts.append(_clean(body.get_text()))
|
|
121
|
+
|
|
122
|
+
return "\n\n".join(parts)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _extract_table(table) -> str:
|
|
126
|
+
"""Extract table content as structured text."""
|
|
127
|
+
rows = []
|
|
128
|
+
for tr in table.find_all("tr"):
|
|
129
|
+
cells = []
|
|
130
|
+
for td in tr.find_all(["td", "th"]):
|
|
131
|
+
cells.append(_clean(td.get_text()))
|
|
132
|
+
if any(cells):
|
|
133
|
+
rows.append(" | ".join(cells))
|
|
134
|
+
return "\n".join(rows)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
138
|
+
captions = []
|
|
139
|
+
for fig in soup.select("figure"):
|
|
140
|
+
cap = fig.select_one("figcaption")
|
|
141
|
+
if cap:
|
|
142
|
+
text = _clean(cap.get_text())
|
|
143
|
+
if text and len(text) > 10:
|
|
144
|
+
captions.append(text)
|
|
145
|
+
return captions
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
149
|
+
refs = []
|
|
150
|
+
ref_section = soup.select_one("section#references-section")
|
|
151
|
+
if ref_section:
|
|
152
|
+
for li in ref_section.find_all("li"):
|
|
153
|
+
text = _clean(li.get_text())
|
|
154
|
+
if text and len(text) > 20:
|
|
155
|
+
refs.append(text)
|
|
156
|
+
return refs
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _clean(text: str) -> str:
|
|
160
|
+
return re.sub(r"\s+", " ", text).strip()
|