litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,204 @@
1
+ """HTML extractor for Nature/Springer articles."""
2
+
3
+ import json
4
+ import re
5
+
6
+ from bs4 import BeautifulSoup
7
+
8
+
9
+ def can_handle(url: str) -> bool:
10
+ """Check if this adapter can handle the given URL."""
11
+ return any(
12
+ domain in url.lower()
13
+ for domain in ["nature.com", "springer.com", "springerlink.com"]
14
+ )
15
+
16
+
17
+ def extract(html: str, url: str = "") -> dict:
18
+ """Extract paper content from Nature/Springer HTML."""
19
+ soup = BeautifulSoup(html, "lxml")
20
+
21
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
22
+ tag.decompose()
23
+
24
+ title = _extract_title(soup)
25
+ authors = _extract_authors(soup)
26
+ abstract = _extract_abstract(soup)
27
+ full_text = _extract_body(soup)
28
+ figures = _extract_figures(soup)
29
+ references = _extract_references(soup)
30
+
31
+ return {
32
+ "title": title,
33
+ "authors": authors,
34
+ "abstract": abstract,
35
+ "full_text": full_text,
36
+ "figures": figures,
37
+ "references": references,
38
+ }
39
+
40
+
41
+ def _extract_title(soup: BeautifulSoup) -> str:
42
+ for selector in [
43
+ "h1.c-article-title",
44
+ "h1.article-item__title",
45
+ "meta[name='citation_title']",
46
+ "meta[property='og:title']",
47
+ ]:
48
+ el = soup.select_one(selector)
49
+ if el:
50
+ if el.name == "meta":
51
+ return el.get("content", "").strip()
52
+ return el.get_text(strip=True)
53
+ return ""
54
+
55
+
56
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
57
+ """Extract authors using three-layer pipeline: meta -> JSON-LD -> DOM."""
58
+ # Layer 1: meta tags (most reliable)
59
+ authors = []
60
+ for meta in soup.select("meta[name='citation_author']"):
61
+ name = meta.get("content", "").strip()
62
+ if name:
63
+ authors.append(name)
64
+ if authors:
65
+ return authors
66
+
67
+ # Layer 2: JSON-LD
68
+ authors = _authors_from_jsonld(soup)
69
+ if authors:
70
+ return authors
71
+
72
+ # Layer 3: DOM selectors
73
+ for el in soup.select("li.c-article-author-list__item a"):
74
+ name = el.get_text(strip=True)
75
+ if name:
76
+ authors.append(name)
77
+ if not authors:
78
+ for el in soup.select("[data-test='author-name']"):
79
+ name = el.get_text(strip=True)
80
+ if name:
81
+ authors.append(name)
82
+ return authors
83
+
84
+
85
+ def _authors_from_jsonld(soup: BeautifulSoup) -> list[str]:
86
+ """Extract authors from JSON-LD structured data."""
87
+ for script in soup.select("script[type='application/ld+json']"):
88
+ try:
89
+ data = json.loads(script.string or "")
90
+ except (json.JSONDecodeError, TypeError):
91
+ continue
92
+
93
+ # Handle both single object and list
94
+ items = data if isinstance(data, list) else [data]
95
+ for item in items:
96
+ author = item.get("author", [])
97
+ if isinstance(author, dict):
98
+ author = [author]
99
+ names = []
100
+ for a in author:
101
+ if isinstance(a, dict):
102
+ name = a.get("name", "")
103
+ if name:
104
+ names.append(name)
105
+ if names:
106
+ return names
107
+ return []
108
+
109
+
110
+ def _extract_abstract(soup: BeautifulSoup) -> str:
111
+ for selector in [
112
+ "#Abs1-content",
113
+ "div.c-article-section__content[id*='Abs']",
114
+ "#abstract",
115
+ "section[data-title='Abstract']",
116
+ "div.article__body p.article__teaser",
117
+ "meta[name='description']",
118
+ "meta[property='og:description']",
119
+ ]:
120
+ el = soup.select_one(selector)
121
+ if el:
122
+ if el.name == "meta":
123
+ return el.get("content", "").strip()
124
+ return _clean(el.get_text())
125
+ return ""
126
+
127
+
128
+ def _extract_body(soup: BeautifulSoup) -> str:
129
+ parts = []
130
+
131
+ # Nature uses section[data-title] for article sections
132
+ sections = soup.select("section[data-title]")
133
+ if sections:
134
+ for section in sections:
135
+ title = section.get("data-title", "")
136
+ if title.lower() in ("abstract", "references", "supplementary information", "data availability"):
137
+ continue
138
+ content_div = section.select_one(".c-article-section__content")
139
+ if content_div:
140
+ # Extract text and tables
141
+ text_parts = []
142
+ for child in content_div.children:
143
+ if child.name == "table":
144
+ table_text = _extract_table(child)
145
+ if table_text:
146
+ text_parts.append(table_text)
147
+ elif child.name:
148
+ text = _clean(child.get_text())
149
+ if text:
150
+ text_parts.append(text)
151
+ content = "\n\n".join(text_parts)
152
+ if content:
153
+ parts.append(f"## {title}\n\n{content}")
154
+
155
+ # Fallback: try article body div
156
+ if not parts:
157
+ body = soup.select_one("div.c-article-body") or soup.select_one("div.article__body")
158
+ if body:
159
+ parts.append(_clean(body.get_text()))
160
+
161
+ return "\n\n".join(parts)
162
+
163
+
164
+ def _extract_table(table) -> str:
165
+ """Extract table content as structured text."""
166
+ rows = []
167
+ for tr in table.find_all("tr"):
168
+ cells = []
169
+ for td in tr.find_all(["td", "th"]):
170
+ cells.append(_clean(td.get_text()))
171
+ if any(cells):
172
+ rows.append(" | ".join(cells))
173
+ return "\n".join(rows)
174
+
175
+
176
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
177
+ captions = []
178
+ for fig in soup.select("figure"):
179
+ cap = fig.select_one("figcaption")
180
+ if cap:
181
+ text = _clean(cap.get_text())
182
+ if text and len(text) > 10:
183
+ captions.append(text)
184
+ return captions
185
+
186
+
187
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
188
+ refs = []
189
+ ref_section = soup.select_one("#Bib1") or soup.select_one("#references")
190
+ if ref_section:
191
+ for li in ref_section.select("li.c-article-references__item"):
192
+ text = _clean(li.get_text())
193
+ if text:
194
+ refs.append(text)
195
+ if not refs:
196
+ for li in ref_section.find_all("li"):
197
+ text = _clean(li.get_text())
198
+ if text and len(text) > 20:
199
+ refs.append(text)
200
+ return refs
201
+
202
+
203
+ def _clean(text: str) -> str:
204
+ return re.sub(r"\s+", " ", text).strip()
@@ -0,0 +1,154 @@
1
+ """HTML extractor for RSC (Royal Society of Chemistry) articles."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ def can_handle(url: str) -> bool:
9
+ """Check if this adapter can handle the given URL."""
10
+ return "pubs.rsc.org" in url.lower()
11
+
12
+
13
+ def extract(html: str, url: str = "") -> dict:
14
+ """Extract paper content from RSC Publications HTML."""
15
+ soup = BeautifulSoup(html, "lxml")
16
+
17
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
18
+ tag.decompose()
19
+
20
+ return {
21
+ "title": _extract_title(soup),
22
+ "authors": _extract_authors(soup),
23
+ "abstract": _extract_abstract(soup),
24
+ "full_text": _extract_body(soup),
25
+ "figures": _extract_figures(soup),
26
+ "references": _extract_references(soup),
27
+ }
28
+
29
+
30
+ def _extract_title(soup: BeautifulSoup) -> str:
31
+ for selector in [
32
+ "h1.headline",
33
+ "h1.title",
34
+ "meta[name='citation_title']",
35
+ "meta[property='og:title']",
36
+ ]:
37
+ el = soup.select_one(selector)
38
+ if el:
39
+ if el.name == "meta":
40
+ return el.get("content", "").strip()
41
+ return el.get_text(strip=True)
42
+ return ""
43
+
44
+
45
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
46
+ authors = []
47
+ for meta in soup.select("meta[name='citation_author']"):
48
+ name = meta.get("content", "").strip()
49
+ if name:
50
+ authors.append(name)
51
+ if not authors:
52
+ for el in soup.select(".article__authors a, .authors-list a"):
53
+ name = el.get_text(strip=True)
54
+ if name and len(name) > 1:
55
+ authors.append(name)
56
+ return authors
57
+
58
+
59
+ def _extract_abstract(soup: BeautifulSoup) -> str:
60
+ for selector in [
61
+ "div.article__abstract",
62
+ "div.abstract",
63
+ "#abstract",
64
+ "meta[name='description']",
65
+ ]:
66
+ el = soup.select_one(selector)
67
+ if el:
68
+ if el.name == "meta":
69
+ return el.get("content", "").strip()
70
+ return _clean(el.get_text())
71
+ return ""
72
+
73
+
74
+ def _extract_body(soup: BeautifulSoup) -> str:
75
+ parts = []
76
+
77
+ body = soup.select_one("div#article-body") or soup.select_one("div.article__body")
78
+ if body:
79
+ for section in body.find_all(["section", "div"], recursive=False):
80
+ heading = section.find(re.compile(r"h[2-4]"))
81
+ heading_text = heading.get_text(strip=True) if heading else ""
82
+ level = int(heading.name[1]) if heading else 2
83
+
84
+ if heading_text.lower() in ("abstract", "references", "notes and references"):
85
+ continue
86
+
87
+ text_parts = []
88
+ for child in section.children:
89
+ if child.name:
90
+ if child.name == "table":
91
+ table_text = _extract_table(child)
92
+ if table_text:
93
+ text_parts.append(table_text)
94
+ else:
95
+ text = _clean(child.get_text())
96
+ if text:
97
+ text_parts.append(text)
98
+
99
+ content = "\n\n".join(text_parts)
100
+ if heading_text and content:
101
+ prefix = "#" * min(level, 4)
102
+ parts.append(f"{prefix} {heading_text}\n\n{content}")
103
+ elif content:
104
+ parts.append(content)
105
+
106
+ if not parts and body:
107
+ parts.append(_clean(body.get_text()))
108
+
109
+ if not parts:
110
+ article = soup.select_one("article")
111
+ if article:
112
+ parts.append(_clean(article.get_text()))
113
+
114
+ return "\n\n".join(parts)
115
+
116
+
117
+ def _extract_table(table) -> str:
118
+ rows = []
119
+ for tr in table.find_all("tr"):
120
+ cells = []
121
+ for td in tr.find_all(["td", "th"]):
122
+ cells.append(_clean(td.get_text()))
123
+ if any(cells):
124
+ rows.append(" | ".join(cells))
125
+ return "\n".join(rows)
126
+
127
+
128
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
129
+ captions = []
130
+ for fig in soup.select("figure"):
131
+ cap = fig.select_one("figcaption")
132
+ if cap:
133
+ text = _clean(cap.get_text())
134
+ if text and len(text) > 10:
135
+ captions.append(text)
136
+ return captions
137
+
138
+
139
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
140
+ refs = []
141
+ for selector in ["div.references", "section.references", "#references"]:
142
+ ref_section = soup.select_one(selector)
143
+ if ref_section:
144
+ for li in ref_section.find_all("li"):
145
+ text = _clean(li.get_text())
146
+ if text and len(text) > 20:
147
+ refs.append(text)
148
+ if refs:
149
+ return refs
150
+ return refs
151
+
152
+
153
+ def _clean(text: str) -> str:
154
+ return re.sub(r"\s+", " ", text).strip()
@@ -0,0 +1,154 @@
1
+ """HTML extractor for Taylor & Francis articles."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ def can_handle(url: str) -> bool:
9
+ """Check if this adapter can handle the given URL."""
10
+ return "tandfonline.com" in url.lower()
11
+
12
+
13
+ def extract(html: str, url: str = "") -> dict:
14
+ """Extract paper content from Taylor & Francis HTML."""
15
+ soup = BeautifulSoup(html, "lxml")
16
+
17
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
18
+ tag.decompose()
19
+
20
+ return {
21
+ "title": _extract_title(soup),
22
+ "authors": _extract_authors(soup),
23
+ "abstract": _extract_abstract(soup),
24
+ "full_text": _extract_body(soup),
25
+ "figures": _extract_figures(soup),
26
+ "references": _extract_references(soup),
27
+ }
28
+
29
+
30
+ def _extract_title(soup: BeautifulSoup) -> str:
31
+ for selector in [
32
+ "h1.article-header__title",
33
+ "h1.title",
34
+ "meta[name='citation_title']",
35
+ "meta[property='og:title']",
36
+ ]:
37
+ el = soup.select_one(selector)
38
+ if el:
39
+ if el.name == "meta":
40
+ return el.get("content", "").strip()
41
+ return el.get_text(strip=True)
42
+ return ""
43
+
44
+
45
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
46
+ authors = []
47
+ for meta in soup.select("meta[name='citation_author']"):
48
+ name = meta.get("content", "").strip()
49
+ if name:
50
+ authors.append(name)
51
+ if not authors:
52
+ for el in soup.select(".contrib-group .contrib, .author-name, a.author"):
53
+ name = el.get_text(strip=True)
54
+ if name and len(name) > 1:
55
+ authors.append(name)
56
+ return authors
57
+
58
+
59
+ def _extract_abstract(soup: BeautifulSoup) -> str:
60
+ for selector in [
61
+ "div.abstract",
62
+ "div.abstractSection",
63
+ "#abstract",
64
+ "meta[name='description']",
65
+ ]:
66
+ el = soup.select_one(selector)
67
+ if el:
68
+ if el.name == "meta":
69
+ return el.get("content", "").strip()
70
+ return _clean(el.get_text())
71
+ return ""
72
+
73
+
74
+ def _extract_body(soup: BeautifulSoup) -> str:
75
+ parts = []
76
+
77
+ body = soup.select_one("div.article-body") or soup.select_one("div.article_body")
78
+ if body:
79
+ for section in body.find_all(["section", "div"], recursive=False):
80
+ heading = section.find(re.compile(r"h[2-4]"))
81
+ heading_text = heading.get_text(strip=True) if heading else ""
82
+ level = int(heading.name[1]) if heading else 2
83
+
84
+ if heading_text.lower() in ("abstract", "references", "acknowledgments", "acknowledgements"):
85
+ continue
86
+
87
+ text_parts = []
88
+ for child in section.children:
89
+ if child.name:
90
+ if child.name == "table":
91
+ table_text = _extract_table(child)
92
+ if table_text:
93
+ text_parts.append(table_text)
94
+ else:
95
+ text = _clean(child.get_text())
96
+ if text:
97
+ text_parts.append(text)
98
+
99
+ content = "\n\n".join(text_parts)
100
+ if heading_text and content:
101
+ prefix = "#" * min(level, 4)
102
+ parts.append(f"{prefix} {heading_text}\n\n{content}")
103
+ elif content:
104
+ parts.append(content)
105
+
106
+ if not parts and body:
107
+ parts.append(_clean(body.get_text()))
108
+
109
+ if not parts:
110
+ article = soup.select_one("article")
111
+ if article:
112
+ parts.append(_clean(article.get_text()))
113
+
114
+ return "\n\n".join(parts)
115
+
116
+
117
+ def _extract_table(table) -> str:
118
+ rows = []
119
+ for tr in table.find_all("tr"):
120
+ cells = []
121
+ for td in tr.find_all(["td", "th"]):
122
+ cells.append(_clean(td.get_text()))
123
+ if any(cells):
124
+ rows.append(" | ".join(cells))
125
+ return "\n".join(rows)
126
+
127
+
128
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
129
+ captions = []
130
+ for fig in soup.select("figure"):
131
+ cap = fig.select_one("figcaption")
132
+ if cap:
133
+ text = _clean(cap.get_text())
134
+ if text and len(text) > 10:
135
+ captions.append(text)
136
+ return captions
137
+
138
+
139
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
140
+ refs = []
141
+ for selector in ["div.ref-list", "section.references", "#references", "div.references"]:
142
+ ref_section = soup.select_one(selector)
143
+ if ref_section:
144
+ for li in ref_section.find_all("li"):
145
+ text = _clean(li.get_text())
146
+ if text and len(text) > 20:
147
+ refs.append(text)
148
+ if refs:
149
+ return refs
150
+ return refs
151
+
152
+
153
+ def _clean(text: str) -> str:
154
+ return re.sub(r"\s+", " ", text).strip()
@@ -0,0 +1,160 @@
1
+ """HTML extractor for Wiley Online Library articles."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ def can_handle(url: str) -> bool:
9
+ """Check if this adapter can handle the given URL."""
10
+ return "wiley.com" in url.lower() or "onlinelibrary.wiley" in url.lower()
11
+
12
+
13
+ def extract(html: str, url: str = "") -> dict:
14
+ """Extract paper content from Wiley Online Library HTML."""
15
+ soup = BeautifulSoup(html, "lxml")
16
+
17
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
18
+ tag.decompose()
19
+
20
+ return {
21
+ "title": _extract_title(soup),
22
+ "authors": _extract_authors(soup),
23
+ "abstract": _extract_abstract(soup),
24
+ "full_text": _extract_body(soup),
25
+ "figures": _extract_figures(soup),
26
+ "references": _extract_references(soup),
27
+ }
28
+
29
+
30
+ def _extract_title(soup: BeautifulSoup) -> str:
31
+ for selector in [
32
+ "h1.citation__title",
33
+ ".article-header__title",
34
+ "meta[name='citation_title']",
35
+ "meta[property='og:title']",
36
+ ]:
37
+ el = soup.select_one(selector)
38
+ if el:
39
+ if el.name == "meta":
40
+ return el.get("content", "").strip()
41
+ return el.get_text(strip=True)
42
+ return ""
43
+
44
+
45
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
46
+ """Extract authors using meta tags first, then DOM."""
47
+ authors = []
48
+ # Meta tags (most reliable)
49
+ for meta in soup.select("meta[name='citation_author']"):
50
+ name = meta.get("content", "").strip()
51
+ if name:
52
+ authors.append(name)
53
+ if authors:
54
+ return authors
55
+
56
+ # DOM: multiple possible selectors
57
+ for selector in [
58
+ ".loa-authors-trunc a.author-name",
59
+ ".loa-authors .author-name span",
60
+ "a.author-name",
61
+ "[data-test='author-name']",
62
+ ]:
63
+ for el in soup.select(selector):
64
+ name = el.get_text(strip=True)
65
+ if name:
66
+ authors.append(name)
67
+ if authors:
68
+ return authors
69
+ return authors
70
+
71
+
72
+ def _extract_abstract(soup: BeautifulSoup) -> str:
73
+ for selector in [
74
+ "section.article-section__abstract",
75
+ "div.abstract-group",
76
+ "#abstract",
77
+ "meta[name='description']",
78
+ ]:
79
+ el = soup.select_one(selector)
80
+ if el:
81
+ if el.name == "meta":
82
+ return el.get("content", "").strip()
83
+ return _clean(el.get_text())
84
+ return ""
85
+
86
+
87
+ def _extract_body(soup: BeautifulSoup) -> str:
88
+ parts = []
89
+
90
+ # Wiley uses section.article-section__content divs
91
+ for section in soup.select("section.article-section__content"):
92
+ heading = section.find_previous("h2")
93
+ heading_text = heading.get_text(strip=True) if heading else ""
94
+
95
+ if heading_text.lower() in ("abstract", "references", "supporting information", "data availability"):
96
+ continue
97
+
98
+ # Extract text with table support
99
+ text_parts = []
100
+ for child in section.children:
101
+ if child.name == "table":
102
+ table_text = _extract_table(child)
103
+ if table_text:
104
+ text_parts.append(table_text)
105
+ elif child.name:
106
+ text = _clean(child.get_text())
107
+ if text:
108
+ text_parts.append(text)
109
+ content = "\n\n".join(text_parts)
110
+
111
+ if heading_text and content:
112
+ parts.append(f"## {heading_text}\n\n{content}")
113
+ elif content:
114
+ parts.append(content)
115
+
116
+ # Fallback
117
+ if not parts:
118
+ body = soup.select_one("article.article__body") or soup.select_one(".article-body-section")
119
+ if body:
120
+ parts.append(_clean(body.get_text()))
121
+
122
+ return "\n\n".join(parts)
123
+
124
+
125
+ def _extract_table(table) -> str:
126
+ """Extract table content as structured text."""
127
+ rows = []
128
+ for tr in table.find_all("tr"):
129
+ cells = []
130
+ for td in tr.find_all(["td", "th"]):
131
+ cells.append(_clean(td.get_text()))
132
+ if any(cells):
133
+ rows.append(" | ".join(cells))
134
+ return "\n".join(rows)
135
+
136
+
137
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
138
+ captions = []
139
+ for fig in soup.select("figure"):
140
+ cap = fig.select_one("figcaption")
141
+ if cap:
142
+ text = _clean(cap.get_text())
143
+ if text and len(text) > 10:
144
+ captions.append(text)
145
+ return captions
146
+
147
+
148
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
149
+ refs = []
150
+ ref_section = soup.select_one("section#references-section")
151
+ if ref_section:
152
+ for li in ref_section.find_all("li"):
153
+ text = _clean(li.get_text())
154
+ if text and len(text) > 20:
155
+ refs.append(text)
156
+ return refs
157
+
158
+
159
+ def _clean(text: str) -> str:
160
+ return re.sub(r"\s+", " ", text).strip()