litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""PDF text extraction using pymupdf."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pymupdf
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
# Patterns for figure captions
|
|
12
|
+
FIGURE_PATTERN = re.compile(
|
|
13
|
+
r"^(?:Fig(?:ure|\.)\s*\d+[.:]\s*.+)$",
|
|
14
|
+
re.MULTILINE | re.IGNORECASE,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def extract_text(pdf_path: str | Path) -> str:
|
|
19
|
+
"""Extract text content from a PDF file.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
pdf_path: Path to the PDF file.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
Extracted text content.
|
|
26
|
+
"""
|
|
27
|
+
pdf_path = Path(pdf_path)
|
|
28
|
+
if not pdf_path.exists():
|
|
29
|
+
logger.error("PDF file not found: %s", pdf_path)
|
|
30
|
+
return ""
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
doc = pymupdf.open(str(pdf_path))
|
|
34
|
+
except Exception as e:
|
|
35
|
+
logger.error("Failed to open PDF %s: %s", pdf_path, e)
|
|
36
|
+
return ""
|
|
37
|
+
|
|
38
|
+
text_parts = []
|
|
39
|
+
for page in doc:
|
|
40
|
+
text = page.get_text("text")
|
|
41
|
+
if text:
|
|
42
|
+
text_parts.append(text)
|
|
43
|
+
|
|
44
|
+
doc.close()
|
|
45
|
+
full_text = "\n\n".join(text_parts)
|
|
46
|
+
|
|
47
|
+
# Clean up common PDF artifacts
|
|
48
|
+
full_text = _clean_text(full_text)
|
|
49
|
+
return full_text
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def extract_figures(pdf_path: str | Path) -> list[str]:
|
|
53
|
+
"""Extract figure captions from a PDF file.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
pdf_path: Path to the PDF file.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
List of figure caption strings.
|
|
60
|
+
"""
|
|
61
|
+
text = extract_text(pdf_path)
|
|
62
|
+
if not text:
|
|
63
|
+
return []
|
|
64
|
+
|
|
65
|
+
captions = []
|
|
66
|
+
for match in FIGURE_PATTERN.finditer(text):
|
|
67
|
+
caption = match.group(0).strip()
|
|
68
|
+
if len(caption) > 10: # Filter out too-short matches
|
|
69
|
+
captions.append(caption)
|
|
70
|
+
|
|
71
|
+
return captions
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def extract_figures_from_text(text: str) -> list[str]:
|
|
75
|
+
"""Extract figure captions from already-extracted text.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
text: Extracted text content from a PDF.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
List of figure caption strings.
|
|
82
|
+
"""
|
|
83
|
+
if not text:
|
|
84
|
+
return []
|
|
85
|
+
|
|
86
|
+
captions = []
|
|
87
|
+
for match in FIGURE_PATTERN.finditer(text):
|
|
88
|
+
caption = match.group(0).strip()
|
|
89
|
+
if len(caption) > 10:
|
|
90
|
+
captions.append(caption)
|
|
91
|
+
|
|
92
|
+
return captions
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def extract_from_bytes(pdf_bytes: bytes) -> str:
|
|
96
|
+
"""Extract text from PDF bytes (e.g., from a download).
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
pdf_bytes: Raw PDF content.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
Extracted text content.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
doc = pymupdf.open(stream=pdf_bytes, filetype="pdf")
|
|
106
|
+
except Exception as e:
|
|
107
|
+
logger.error("Failed to open PDF from bytes: %s", e)
|
|
108
|
+
return ""
|
|
109
|
+
|
|
110
|
+
text_parts = []
|
|
111
|
+
for page in doc:
|
|
112
|
+
text = page.get_text("text")
|
|
113
|
+
if text:
|
|
114
|
+
text_parts.append(text)
|
|
115
|
+
|
|
116
|
+
doc.close()
|
|
117
|
+
return _clean_text("\n\n".join(text_parts))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _clean_text(text: str) -> str:
|
|
121
|
+
"""Clean up common PDF extraction artifacts."""
|
|
122
|
+
# Remove excessive whitespace but preserve paragraph breaks
|
|
123
|
+
text = re.sub(r"[ \t]+", " ", text)
|
|
124
|
+
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
125
|
+
# Remove hyphenation at line breaks
|
|
126
|
+
text = re.sub(r"(\w)-\n(\w)", r"\1\2", text)
|
|
127
|
+
# Fix broken lines within paragraphs (single newline -> space)
|
|
128
|
+
# But preserve double newlines (paragraph breaks)
|
|
129
|
+
_SENTENCE_ENDS = (".", ":", "?", "!", "。", ";", "!", "?")
|
|
130
|
+
_LIST_PREFIX = re.compile(r"^\s*(?:\d+[\.\)]\s|[-*•]\s)")
|
|
131
|
+
lines = text.split("\n")
|
|
132
|
+
result = []
|
|
133
|
+
for i, line in enumerate(lines):
|
|
134
|
+
stripped = line.strip()
|
|
135
|
+
if not stripped:
|
|
136
|
+
# Keep the paragraph break: a bare "" would vanish when the lines
|
|
137
|
+
# are joined back together.
|
|
138
|
+
result.append("\n")
|
|
139
|
+
elif (
|
|
140
|
+
i + 1 < len(lines)
|
|
141
|
+
and lines[i + 1].strip()
|
|
142
|
+
and not stripped.endswith(_SENTENCE_ENDS)
|
|
143
|
+
and len(stripped) > 20
|
|
144
|
+
and not _LIST_PREFIX.match(stripped)
|
|
145
|
+
and not _LIST_PREFIX.match(lines[i + 1].strip())
|
|
146
|
+
):
|
|
147
|
+
result.append(stripped + " ")
|
|
148
|
+
else:
|
|
149
|
+
result.append(stripped + "\n")
|
|
150
|
+
|
|
151
|
+
return "".join(result).strip()
|
|
File without changes
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""HTML extractor for ACS Publications articles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def can_handle(url: str) -> bool:
|
|
9
|
+
"""Check if this adapter can handle the given URL."""
|
|
10
|
+
return "pubs.acs.org" in url.lower()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract(html: str, url: str = "") -> dict:
|
|
14
|
+
"""Extract paper content from ACS Publications HTML."""
|
|
15
|
+
soup = BeautifulSoup(html, "lxml")
|
|
16
|
+
|
|
17
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
18
|
+
tag.decompose()
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
"title": _extract_title(soup),
|
|
22
|
+
"authors": _extract_authors(soup),
|
|
23
|
+
"abstract": _extract_abstract(soup),
|
|
24
|
+
"full_text": _extract_body(soup),
|
|
25
|
+
"figures": _extract_figures(soup),
|
|
26
|
+
"references": _extract_references(soup),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
31
|
+
for selector in [
|
|
32
|
+
"h1.article_header-title",
|
|
33
|
+
".article-title",
|
|
34
|
+
"meta[name='citation_title']",
|
|
35
|
+
"meta[property='og:title']",
|
|
36
|
+
]:
|
|
37
|
+
el = soup.select_one(selector)
|
|
38
|
+
if el:
|
|
39
|
+
if el.name == "meta":
|
|
40
|
+
return el.get("content", "").strip()
|
|
41
|
+
return el.get_text(strip=True)
|
|
42
|
+
return ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
46
|
+
"""Extract authors using meta tags first, then DOM."""
|
|
47
|
+
authors = []
|
|
48
|
+
# Meta tags
|
|
49
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
50
|
+
name = meta.get("content", "").strip()
|
|
51
|
+
if name:
|
|
52
|
+
authors.append(name)
|
|
53
|
+
if authors:
|
|
54
|
+
return authors
|
|
55
|
+
|
|
56
|
+
# DOM selectors
|
|
57
|
+
for selector in [
|
|
58
|
+
".loa li .hlFld-ContribAuthor",
|
|
59
|
+
".contrib-group .contrib",
|
|
60
|
+
"a.author-name",
|
|
61
|
+
]:
|
|
62
|
+
for el in soup.select(selector):
|
|
63
|
+
name = el.get_text(strip=True)
|
|
64
|
+
if name:
|
|
65
|
+
authors.append(name)
|
|
66
|
+
if authors:
|
|
67
|
+
return authors
|
|
68
|
+
return authors
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
72
|
+
for selector in [
|
|
73
|
+
"div.article_abstract-content",
|
|
74
|
+
"#abstractBox",
|
|
75
|
+
"p.articleBody_abstractText",
|
|
76
|
+
"meta[name='description']",
|
|
77
|
+
]:
|
|
78
|
+
el = soup.select_one(selector)
|
|
79
|
+
if el:
|
|
80
|
+
if el.name == "meta":
|
|
81
|
+
return el.get("content", "").strip()
|
|
82
|
+
return _clean(el.get_text())
|
|
83
|
+
return ""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
87
|
+
parts = []
|
|
88
|
+
|
|
89
|
+
# ACS uses div.article_content with .NLM_sec sections
|
|
90
|
+
article_content = soup.select_one("div.article_content")
|
|
91
|
+
if article_content:
|
|
92
|
+
for section in article_content.select(".NLM_sec"):
|
|
93
|
+
heading = section.find(re.compile(r"h[2-4]"))
|
|
94
|
+
heading_text = heading.get_text(strip=True) if heading else ""
|
|
95
|
+
level = int(heading.name[1]) if heading else 2
|
|
96
|
+
|
|
97
|
+
if heading_text.lower() in ("abstract", "references", "supporting information"):
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
# Extract text and tables from section
|
|
101
|
+
text_parts = []
|
|
102
|
+
for child in section.children:
|
|
103
|
+
if child.name and child != heading:
|
|
104
|
+
if child.name == "table":
|
|
105
|
+
table_text = _extract_table(child)
|
|
106
|
+
if table_text:
|
|
107
|
+
text_parts.append(table_text)
|
|
108
|
+
elif not child.select(".NLM_sec"): # Skip nested sections
|
|
109
|
+
text = _clean(child.get_text())
|
|
110
|
+
if text:
|
|
111
|
+
text_parts.append(text)
|
|
112
|
+
|
|
113
|
+
content = "\n\n".join(text_parts)
|
|
114
|
+
if heading_text and content:
|
|
115
|
+
prefix = "#" * min(level, 4)
|
|
116
|
+
parts.append(f"{prefix} {heading_text}\n\n{content}")
|
|
117
|
+
elif content:
|
|
118
|
+
parts.append(content)
|
|
119
|
+
|
|
120
|
+
if not parts and article_content:
|
|
121
|
+
parts.append(_clean(article_content.get_text()))
|
|
122
|
+
|
|
123
|
+
# Fallback
|
|
124
|
+
if not parts:
|
|
125
|
+
body = soup.select_one("article") or soup.select_one("#article-body")
|
|
126
|
+
if body:
|
|
127
|
+
parts.append(_clean(body.get_text()))
|
|
128
|
+
|
|
129
|
+
return "\n\n".join(parts)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _extract_table(table) -> str:
|
|
133
|
+
"""Extract table content as structured text."""
|
|
134
|
+
rows = []
|
|
135
|
+
for tr in table.find_all("tr"):
|
|
136
|
+
cells = []
|
|
137
|
+
for td in tr.find_all(["td", "th"]):
|
|
138
|
+
cells.append(_clean(td.get_text()))
|
|
139
|
+
if any(cells):
|
|
140
|
+
rows.append(" | ".join(cells))
|
|
141
|
+
return "\n".join(rows)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
145
|
+
captions = []
|
|
146
|
+
for fig in soup.select("figure, .article_figure"):
|
|
147
|
+
cap = fig.select_one("figcaption, .article_figure-caption")
|
|
148
|
+
if cap:
|
|
149
|
+
text = _clean(cap.get_text())
|
|
150
|
+
if text and len(text) > 10:
|
|
151
|
+
captions.append(text)
|
|
152
|
+
return captions
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
156
|
+
refs = []
|
|
157
|
+
ref_section = soup.select_one("#references") or soup.select_one(".article_references")
|
|
158
|
+
if ref_section:
|
|
159
|
+
for li in ref_section.find_all("li"):
|
|
160
|
+
text = _clean(li.get_text())
|
|
161
|
+
if text and len(text) > 20:
|
|
162
|
+
refs.append(text)
|
|
163
|
+
return refs
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _clean(text: str) -> str:
|
|
167
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""HTML extractor for Elsevier/ScienceDirect articles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def can_handle(url: str) -> bool:
|
|
9
|
+
"""Check if this adapter can handle the given URL."""
|
|
10
|
+
return any(
|
|
11
|
+
domain in url.lower()
|
|
12
|
+
for domain in ["sciencedirect.com", "elsevier.com"]
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def extract(html: str, url: str = "") -> dict:
|
|
17
|
+
"""Extract paper content from Elsevier/ScienceDirect HTML."""
|
|
18
|
+
soup = BeautifulSoup(html, "lxml")
|
|
19
|
+
|
|
20
|
+
for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
|
|
21
|
+
tag.decompose()
|
|
22
|
+
|
|
23
|
+
return {
|
|
24
|
+
"title": _extract_title(soup),
|
|
25
|
+
"authors": _extract_authors(soup),
|
|
26
|
+
"abstract": _extract_abstract(soup),
|
|
27
|
+
"full_text": _extract_body(soup),
|
|
28
|
+
"figures": _extract_figures(soup),
|
|
29
|
+
"references": _extract_references(soup),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
34
|
+
for selector in [
|
|
35
|
+
"span.title-text",
|
|
36
|
+
"h1.article-header__title",
|
|
37
|
+
"meta[name='citation_title']",
|
|
38
|
+
"meta[property='og:title']",
|
|
39
|
+
"meta[name='dc.title']",
|
|
40
|
+
]:
|
|
41
|
+
el = soup.select_one(selector)
|
|
42
|
+
if el:
|
|
43
|
+
if el.name == "meta":
|
|
44
|
+
return el.get("content", "").strip()
|
|
45
|
+
return el.get_text(strip=True)
|
|
46
|
+
return ""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
50
|
+
"""Extract authors using meta tags first, then DOM."""
|
|
51
|
+
authors = []
|
|
52
|
+
# Meta tags (most reliable for ScienceDirect)
|
|
53
|
+
for meta in soup.select("meta[name='citation_author']"):
|
|
54
|
+
name = meta.get("content", "").strip()
|
|
55
|
+
if name:
|
|
56
|
+
authors.append(name)
|
|
57
|
+
if not authors:
|
|
58
|
+
for meta in soup.select("meta[name='dc.creator']"):
|
|
59
|
+
name = meta.get("content", "").strip()
|
|
60
|
+
if name:
|
|
61
|
+
authors.append(name)
|
|
62
|
+
if authors:
|
|
63
|
+
return authors
|
|
64
|
+
|
|
65
|
+
# DOM selectors
|
|
66
|
+
for selector in [
|
|
67
|
+
".author-group .author span.content",
|
|
68
|
+
"a.author-name",
|
|
69
|
+
"[data-test='author-name']",
|
|
70
|
+
]:
|
|
71
|
+
for el in soup.select(selector):
|
|
72
|
+
name = el.get_text(strip=True)
|
|
73
|
+
if name:
|
|
74
|
+
authors.append(name)
|
|
75
|
+
if authors:
|
|
76
|
+
return authors
|
|
77
|
+
return authors
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
81
|
+
for selector in [
|
|
82
|
+
"div.abstract",
|
|
83
|
+
"#abstracts",
|
|
84
|
+
"div.Abstracts",
|
|
85
|
+
"section#abstract",
|
|
86
|
+
"meta[name='description']",
|
|
87
|
+
"meta[property='og:description']",
|
|
88
|
+
]:
|
|
89
|
+
el = soup.select_one(selector)
|
|
90
|
+
if el:
|
|
91
|
+
if el.name == "meta":
|
|
92
|
+
return el.get("content", "").strip()
|
|
93
|
+
return _clean(el.get_text())
|
|
94
|
+
return ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
98
|
+
parts = []
|
|
99
|
+
|
|
100
|
+
# ScienceDirect article body
|
|
101
|
+
body = soup.select_one("div#body") or soup.select_one("div.Body")
|
|
102
|
+
if body:
|
|
103
|
+
for section in body.find_all("section", recursive=False):
|
|
104
|
+
heading = section.find(re.compile(r"h[2-4]"))
|
|
105
|
+
heading_text = heading.get_text(strip=True) if heading else ""
|
|
106
|
+
level = int(heading.name[1]) if heading else 2
|
|
107
|
+
|
|
108
|
+
if heading_text.lower() in ("abstract", "references", "supplementary material"):
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
# Extract text with tables
|
|
112
|
+
text_parts = []
|
|
113
|
+
for child in section.children:
|
|
114
|
+
if child.name:
|
|
115
|
+
if child.name == "table":
|
|
116
|
+
table_text = _extract_table(child)
|
|
117
|
+
if table_text:
|
|
118
|
+
text_parts.append(table_text)
|
|
119
|
+
elif not child.find("section"): # Skip nested sections
|
|
120
|
+
text = _clean(child.get_text())
|
|
121
|
+
if text:
|
|
122
|
+
text_parts.append(text)
|
|
123
|
+
|
|
124
|
+
content = "\n\n".join(text_parts)
|
|
125
|
+
if heading_text and content:
|
|
126
|
+
prefix = "#" * min(level, 4)
|
|
127
|
+
parts.append(f"{prefix} {heading_text}\n\n{content}")
|
|
128
|
+
elif content:
|
|
129
|
+
parts.append(content)
|
|
130
|
+
|
|
131
|
+
if not parts and body:
|
|
132
|
+
parts.append(_clean(body.get_text()))
|
|
133
|
+
|
|
134
|
+
# Fallback
|
|
135
|
+
if not parts:
|
|
136
|
+
article = soup.select_one("article") or soup.select_one("#main-content")
|
|
137
|
+
if article:
|
|
138
|
+
parts.append(_clean(article.get_text()))
|
|
139
|
+
|
|
140
|
+
return "\n\n".join(parts)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _extract_table(table) -> str:
|
|
144
|
+
"""Extract table content as structured text."""
|
|
145
|
+
rows = []
|
|
146
|
+
for tr in table.find_all("tr"):
|
|
147
|
+
cells = []
|
|
148
|
+
for td in tr.find_all(["td", "th"]):
|
|
149
|
+
cells.append(_clean(td.get_text()))
|
|
150
|
+
if any(cells):
|
|
151
|
+
rows.append(" | ".join(cells))
|
|
152
|
+
return "\n".join(rows)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
156
|
+
captions = []
|
|
157
|
+
for fig in soup.select("figure, .figure"):
|
|
158
|
+
cap = fig.select_one("figcaption, .caption")
|
|
159
|
+
if cap:
|
|
160
|
+
text = _clean(cap.get_text())
|
|
161
|
+
if text and len(text) > 10:
|
|
162
|
+
captions.append(text)
|
|
163
|
+
return captions
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
167
|
+
refs = []
|
|
168
|
+
ref_section = soup.select_one("#bibliography") or soup.select_one("section.bibliography")
|
|
169
|
+
if ref_section:
|
|
170
|
+
for li in ref_section.find_all("li"):
|
|
171
|
+
text = _clean(li.get_text())
|
|
172
|
+
if text and len(text) > 20:
|
|
173
|
+
refs.append(text)
|
|
174
|
+
return refs
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _clean(text: str) -> str:
|
|
178
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Generic HTML extractor - fallback for unknown publishers."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from bs4 import BeautifulSoup, Tag
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def extract(html: str, url: str = "") -> dict:
|
|
9
|
+
"""Extract paper content from generic HTML.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
html: Raw HTML content.
|
|
13
|
+
url: The URL the HTML was fetched from.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
Dict with title, abstract, full_text, figures, references.
|
|
17
|
+
"""
|
|
18
|
+
soup = BeautifulSoup(html, "lxml")
|
|
19
|
+
|
|
20
|
+
# Remove script, style, nav, header, footer elements
|
|
21
|
+
for tag in soup.find_all(["script", "style", "nav", "header", "footer", "aside"]):
|
|
22
|
+
tag.decompose()
|
|
23
|
+
|
|
24
|
+
title = _extract_title(soup)
|
|
25
|
+
abstract = _extract_abstract(soup)
|
|
26
|
+
full_text = _extract_body(soup)
|
|
27
|
+
figures = _extract_figures(soup)
|
|
28
|
+
references = _extract_references(soup)
|
|
29
|
+
authors = _extract_authors(soup)
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
"title": title,
|
|
33
|
+
"authors": authors,
|
|
34
|
+
"abstract": abstract,
|
|
35
|
+
"full_text": full_text,
|
|
36
|
+
"figures": figures,
|
|
37
|
+
"references": references,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _extract_title(soup: BeautifulSoup) -> str:
|
|
42
|
+
"""Extract article title."""
|
|
43
|
+
# Try common title selectors
|
|
44
|
+
for selector in [
|
|
45
|
+
"h1.article-title",
|
|
46
|
+
"h1.c-article-title",
|
|
47
|
+
"h1#title",
|
|
48
|
+
".article-header h1",
|
|
49
|
+
"article h1",
|
|
50
|
+
"h1",
|
|
51
|
+
]:
|
|
52
|
+
el = soup.select_one(selector)
|
|
53
|
+
if el and el.get_text(strip=True):
|
|
54
|
+
return el.get_text(strip=True)
|
|
55
|
+
|
|
56
|
+
# Fallback to <title> tag
|
|
57
|
+
title_tag = soup.find("title")
|
|
58
|
+
if title_tag:
|
|
59
|
+
return title_tag.get_text(strip=True)
|
|
60
|
+
return ""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _extract_abstract(soup: BeautifulSoup) -> str:
|
|
64
|
+
"""Extract abstract."""
|
|
65
|
+
for selector in [
|
|
66
|
+
"#abstract",
|
|
67
|
+
".abstract",
|
|
68
|
+
"[data-title='Abstract']",
|
|
69
|
+
"section.abstract",
|
|
70
|
+
"div.abstractSection",
|
|
71
|
+
]:
|
|
72
|
+
el = soup.select_one(selector)
|
|
73
|
+
if el:
|
|
74
|
+
return _clean_text(el.get_text())
|
|
75
|
+
|
|
76
|
+
# Try meta tag
|
|
77
|
+
meta = soup.find("meta", attrs={"name": "description"})
|
|
78
|
+
if meta and meta.get("content"):
|
|
79
|
+
return meta["content"].strip()
|
|
80
|
+
|
|
81
|
+
return ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _extract_body(soup: BeautifulSoup) -> str:
|
|
85
|
+
"""Extract main body text."""
|
|
86
|
+
# Try common article body selectors
|
|
87
|
+
for selector in [
|
|
88
|
+
"article",
|
|
89
|
+
"[role='main']",
|
|
90
|
+
"main",
|
|
91
|
+
".article-body",
|
|
92
|
+
".article-content",
|
|
93
|
+
"#body",
|
|
94
|
+
".body",
|
|
95
|
+
]:
|
|
96
|
+
el = soup.select_one(selector)
|
|
97
|
+
if el:
|
|
98
|
+
text = _clean_text(el.get_text())
|
|
99
|
+
if len(text) > 500: # Sanity check - body should be substantial
|
|
100
|
+
return text
|
|
101
|
+
|
|
102
|
+
# Fallback: find the largest text block
|
|
103
|
+
best = ""
|
|
104
|
+
for tag in soup.find_all(["div", "section"]):
|
|
105
|
+
text = _clean_text(tag.get_text())
|
|
106
|
+
if len(text) > len(best):
|
|
107
|
+
best = text
|
|
108
|
+
|
|
109
|
+
return best
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _extract_figures(soup: BeautifulSoup) -> list[str]:
|
|
113
|
+
"""Extract figure captions."""
|
|
114
|
+
captions = []
|
|
115
|
+
for fig in soup.find_all("figure"):
|
|
116
|
+
cap = fig.find("figcaption")
|
|
117
|
+
if cap:
|
|
118
|
+
text = _clean_text(cap.get_text())
|
|
119
|
+
if text:
|
|
120
|
+
captions.append(text)
|
|
121
|
+
|
|
122
|
+
# Also try common caption classes
|
|
123
|
+
if not captions:
|
|
124
|
+
for selector in [".figure-caption", ".caption", ".fig-caption"]:
|
|
125
|
+
for el in soup.select(selector):
|
|
126
|
+
text = _clean_text(el.get_text())
|
|
127
|
+
if text and len(text) > 10:
|
|
128
|
+
captions.append(text)
|
|
129
|
+
|
|
130
|
+
return captions
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _extract_references(soup: BeautifulSoup) -> list[str]:
|
|
134
|
+
"""Extract reference list."""
|
|
135
|
+
refs = []
|
|
136
|
+
|
|
137
|
+
# Try common reference section selectors
|
|
138
|
+
ref_section = None
|
|
139
|
+
for selector in [
|
|
140
|
+
"#references",
|
|
141
|
+
".references",
|
|
142
|
+
"#bibliography",
|
|
143
|
+
"[data-title='References']",
|
|
144
|
+
"section.ref-list",
|
|
145
|
+
]:
|
|
146
|
+
ref_section = soup.select_one(selector)
|
|
147
|
+
if ref_section:
|
|
148
|
+
break
|
|
149
|
+
|
|
150
|
+
if ref_section:
|
|
151
|
+
for li in ref_section.find_all("li"):
|
|
152
|
+
text = _clean_text(li.get_text())
|
|
153
|
+
if text and len(text) > 20:
|
|
154
|
+
refs.append(text)
|
|
155
|
+
|
|
156
|
+
# If no <li>, try <p> or <div> items
|
|
157
|
+
if not refs:
|
|
158
|
+
for el in ref_section.find_all(["p", "div"], class_=re.compile(r"ref|citation")):
|
|
159
|
+
text = _clean_text(el.get_text())
|
|
160
|
+
if text and len(text) > 20:
|
|
161
|
+
refs.append(text)
|
|
162
|
+
|
|
163
|
+
return refs
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _extract_authors(soup: BeautifulSoup) -> list[str]:
|
|
167
|
+
"""Extract author names."""
|
|
168
|
+
authors = []
|
|
169
|
+
for selector in [
|
|
170
|
+
"meta[name='citation_author']",
|
|
171
|
+
"meta[name='dc.creator']",
|
|
172
|
+
]:
|
|
173
|
+
for meta in soup.select(selector):
|
|
174
|
+
name = meta.get("content", "").strip()
|
|
175
|
+
if name:
|
|
176
|
+
authors.append(name)
|
|
177
|
+
|
|
178
|
+
if not authors:
|
|
179
|
+
for selector in [".author-name", ".authors a", ".contrib-author"]:
|
|
180
|
+
for el in soup.select(selector):
|
|
181
|
+
name = el.get_text(strip=True)
|
|
182
|
+
if name:
|
|
183
|
+
authors.append(name)
|
|
184
|
+
|
|
185
|
+
return authors
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _clean_text(text: str) -> str:
|
|
189
|
+
"""Clean extracted text."""
|
|
190
|
+
text = re.sub(r"\s+", " ", text)
|
|
191
|
+
return text.strip()
|