litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,151 @@
1
+ """PDF text extraction using pymupdf."""
2
+
3
+ import logging
4
+ import re
5
+ from pathlib import Path
6
+
7
+ import pymupdf
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ # Patterns for figure captions
12
+ FIGURE_PATTERN = re.compile(
13
+ r"^(?:Fig(?:ure|\.)\s*\d+[.:]\s*.+)$",
14
+ re.MULTILINE | re.IGNORECASE,
15
+ )
16
+
17
+
18
+ def extract_text(pdf_path: str | Path) -> str:
19
+ """Extract text content from a PDF file.
20
+
21
+ Args:
22
+ pdf_path: Path to the PDF file.
23
+
24
+ Returns:
25
+ Extracted text content.
26
+ """
27
+ pdf_path = Path(pdf_path)
28
+ if not pdf_path.exists():
29
+ logger.error("PDF file not found: %s", pdf_path)
30
+ return ""
31
+
32
+ try:
33
+ doc = pymupdf.open(str(pdf_path))
34
+ except Exception as e:
35
+ logger.error("Failed to open PDF %s: %s", pdf_path, e)
36
+ return ""
37
+
38
+ text_parts = []
39
+ for page in doc:
40
+ text = page.get_text("text")
41
+ if text:
42
+ text_parts.append(text)
43
+
44
+ doc.close()
45
+ full_text = "\n\n".join(text_parts)
46
+
47
+ # Clean up common PDF artifacts
48
+ full_text = _clean_text(full_text)
49
+ return full_text
50
+
51
+
52
+ def extract_figures(pdf_path: str | Path) -> list[str]:
53
+ """Extract figure captions from a PDF file.
54
+
55
+ Args:
56
+ pdf_path: Path to the PDF file.
57
+
58
+ Returns:
59
+ List of figure caption strings.
60
+ """
61
+ text = extract_text(pdf_path)
62
+ if not text:
63
+ return []
64
+
65
+ captions = []
66
+ for match in FIGURE_PATTERN.finditer(text):
67
+ caption = match.group(0).strip()
68
+ if len(caption) > 10: # Filter out too-short matches
69
+ captions.append(caption)
70
+
71
+ return captions
72
+
73
+
74
+ def extract_figures_from_text(text: str) -> list[str]:
75
+ """Extract figure captions from already-extracted text.
76
+
77
+ Args:
78
+ text: Extracted text content from a PDF.
79
+
80
+ Returns:
81
+ List of figure caption strings.
82
+ """
83
+ if not text:
84
+ return []
85
+
86
+ captions = []
87
+ for match in FIGURE_PATTERN.finditer(text):
88
+ caption = match.group(0).strip()
89
+ if len(caption) > 10:
90
+ captions.append(caption)
91
+
92
+ return captions
93
+
94
+
95
+ def extract_from_bytes(pdf_bytes: bytes) -> str:
96
+ """Extract text from PDF bytes (e.g., from a download).
97
+
98
+ Args:
99
+ pdf_bytes: Raw PDF content.
100
+
101
+ Returns:
102
+ Extracted text content.
103
+ """
104
+ try:
105
+ doc = pymupdf.open(stream=pdf_bytes, filetype="pdf")
106
+ except Exception as e:
107
+ logger.error("Failed to open PDF from bytes: %s", e)
108
+ return ""
109
+
110
+ text_parts = []
111
+ for page in doc:
112
+ text = page.get_text("text")
113
+ if text:
114
+ text_parts.append(text)
115
+
116
+ doc.close()
117
+ return _clean_text("\n\n".join(text_parts))
118
+
119
+
120
+ def _clean_text(text: str) -> str:
121
+ """Clean up common PDF extraction artifacts."""
122
+ # Remove excessive whitespace but preserve paragraph breaks
123
+ text = re.sub(r"[ \t]+", " ", text)
124
+ text = re.sub(r"\n{3,}", "\n\n", text)
125
+ # Remove hyphenation at line breaks
126
+ text = re.sub(r"(\w)-\n(\w)", r"\1\2", text)
127
+ # Fix broken lines within paragraphs (single newline -> space)
128
+ # But preserve double newlines (paragraph breaks)
129
+ _SENTENCE_ENDS = (".", ":", "?", "!", "。", ";", "!", "?")
130
+ _LIST_PREFIX = re.compile(r"^\s*(?:\d+[\.\)]\s|[-*•]\s)")
131
+ lines = text.split("\n")
132
+ result = []
133
+ for i, line in enumerate(lines):
134
+ stripped = line.strip()
135
+ if not stripped:
136
+ # Keep the paragraph break: a bare "" would vanish when the lines
137
+ # are joined back together.
138
+ result.append("\n")
139
+ elif (
140
+ i + 1 < len(lines)
141
+ and lines[i + 1].strip()
142
+ and not stripped.endswith(_SENTENCE_ENDS)
143
+ and len(stripped) > 20
144
+ and not _LIST_PREFIX.match(stripped)
145
+ and not _LIST_PREFIX.match(lines[i + 1].strip())
146
+ ):
147
+ result.append(stripped + " ")
148
+ else:
149
+ result.append(stripped + "\n")
150
+
151
+ return "".join(result).strip()
File without changes
@@ -0,0 +1,167 @@
1
+ """HTML extractor for ACS Publications articles."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ def can_handle(url: str) -> bool:
9
+ """Check if this adapter can handle the given URL."""
10
+ return "pubs.acs.org" in url.lower()
11
+
12
+
13
+ def extract(html: str, url: str = "") -> dict:
14
+ """Extract paper content from ACS Publications HTML."""
15
+ soup = BeautifulSoup(html, "lxml")
16
+
17
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
18
+ tag.decompose()
19
+
20
+ return {
21
+ "title": _extract_title(soup),
22
+ "authors": _extract_authors(soup),
23
+ "abstract": _extract_abstract(soup),
24
+ "full_text": _extract_body(soup),
25
+ "figures": _extract_figures(soup),
26
+ "references": _extract_references(soup),
27
+ }
28
+
29
+
30
+ def _extract_title(soup: BeautifulSoup) -> str:
31
+ for selector in [
32
+ "h1.article_header-title",
33
+ ".article-title",
34
+ "meta[name='citation_title']",
35
+ "meta[property='og:title']",
36
+ ]:
37
+ el = soup.select_one(selector)
38
+ if el:
39
+ if el.name == "meta":
40
+ return el.get("content", "").strip()
41
+ return el.get_text(strip=True)
42
+ return ""
43
+
44
+
45
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
46
+ """Extract authors using meta tags first, then DOM."""
47
+ authors = []
48
+ # Meta tags
49
+ for meta in soup.select("meta[name='citation_author']"):
50
+ name = meta.get("content", "").strip()
51
+ if name:
52
+ authors.append(name)
53
+ if authors:
54
+ return authors
55
+
56
+ # DOM selectors
57
+ for selector in [
58
+ ".loa li .hlFld-ContribAuthor",
59
+ ".contrib-group .contrib",
60
+ "a.author-name",
61
+ ]:
62
+ for el in soup.select(selector):
63
+ name = el.get_text(strip=True)
64
+ if name:
65
+ authors.append(name)
66
+ if authors:
67
+ return authors
68
+ return authors
69
+
70
+
71
+ def _extract_abstract(soup: BeautifulSoup) -> str:
72
+ for selector in [
73
+ "div.article_abstract-content",
74
+ "#abstractBox",
75
+ "p.articleBody_abstractText",
76
+ "meta[name='description']",
77
+ ]:
78
+ el = soup.select_one(selector)
79
+ if el:
80
+ if el.name == "meta":
81
+ return el.get("content", "").strip()
82
+ return _clean(el.get_text())
83
+ return ""
84
+
85
+
86
+ def _extract_body(soup: BeautifulSoup) -> str:
87
+ parts = []
88
+
89
+ # ACS uses div.article_content with .NLM_sec sections
90
+ article_content = soup.select_one("div.article_content")
91
+ if article_content:
92
+ for section in article_content.select(".NLM_sec"):
93
+ heading = section.find(re.compile(r"h[2-4]"))
94
+ heading_text = heading.get_text(strip=True) if heading else ""
95
+ level = int(heading.name[1]) if heading else 2
96
+
97
+ if heading_text.lower() in ("abstract", "references", "supporting information"):
98
+ continue
99
+
100
+ # Extract text and tables from section
101
+ text_parts = []
102
+ for child in section.children:
103
+ if child.name and child != heading:
104
+ if child.name == "table":
105
+ table_text = _extract_table(child)
106
+ if table_text:
107
+ text_parts.append(table_text)
108
+ elif not child.select(".NLM_sec"): # Skip nested sections
109
+ text = _clean(child.get_text())
110
+ if text:
111
+ text_parts.append(text)
112
+
113
+ content = "\n\n".join(text_parts)
114
+ if heading_text and content:
115
+ prefix = "#" * min(level, 4)
116
+ parts.append(f"{prefix} {heading_text}\n\n{content}")
117
+ elif content:
118
+ parts.append(content)
119
+
120
+ if not parts and article_content:
121
+ parts.append(_clean(article_content.get_text()))
122
+
123
+ # Fallback
124
+ if not parts:
125
+ body = soup.select_one("article") or soup.select_one("#article-body")
126
+ if body:
127
+ parts.append(_clean(body.get_text()))
128
+
129
+ return "\n\n".join(parts)
130
+
131
+
132
+ def _extract_table(table) -> str:
133
+ """Extract table content as structured text."""
134
+ rows = []
135
+ for tr in table.find_all("tr"):
136
+ cells = []
137
+ for td in tr.find_all(["td", "th"]):
138
+ cells.append(_clean(td.get_text()))
139
+ if any(cells):
140
+ rows.append(" | ".join(cells))
141
+ return "\n".join(rows)
142
+
143
+
144
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
145
+ captions = []
146
+ for fig in soup.select("figure, .article_figure"):
147
+ cap = fig.select_one("figcaption, .article_figure-caption")
148
+ if cap:
149
+ text = _clean(cap.get_text())
150
+ if text and len(text) > 10:
151
+ captions.append(text)
152
+ return captions
153
+
154
+
155
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
156
+ refs = []
157
+ ref_section = soup.select_one("#references") or soup.select_one(".article_references")
158
+ if ref_section:
159
+ for li in ref_section.find_all("li"):
160
+ text = _clean(li.get_text())
161
+ if text and len(text) > 20:
162
+ refs.append(text)
163
+ return refs
164
+
165
+
166
+ def _clean(text: str) -> str:
167
+ return re.sub(r"\s+", " ", text).strip()
@@ -0,0 +1,178 @@
1
+ """HTML extractor for Elsevier/ScienceDirect articles."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup
6
+
7
+
8
+ def can_handle(url: str) -> bool:
9
+ """Check if this adapter can handle the given URL."""
10
+ return any(
11
+ domain in url.lower()
12
+ for domain in ["sciencedirect.com", "elsevier.com"]
13
+ )
14
+
15
+
16
+ def extract(html: str, url: str = "") -> dict:
17
+ """Extract paper content from Elsevier/ScienceDirect HTML."""
18
+ soup = BeautifulSoup(html, "lxml")
19
+
20
+ for tag in soup.find_all(["script", "style", "nav", "footer", "header"]):
21
+ tag.decompose()
22
+
23
+ return {
24
+ "title": _extract_title(soup),
25
+ "authors": _extract_authors(soup),
26
+ "abstract": _extract_abstract(soup),
27
+ "full_text": _extract_body(soup),
28
+ "figures": _extract_figures(soup),
29
+ "references": _extract_references(soup),
30
+ }
31
+
32
+
33
+ def _extract_title(soup: BeautifulSoup) -> str:
34
+ for selector in [
35
+ "span.title-text",
36
+ "h1.article-header__title",
37
+ "meta[name='citation_title']",
38
+ "meta[property='og:title']",
39
+ "meta[name='dc.title']",
40
+ ]:
41
+ el = soup.select_one(selector)
42
+ if el:
43
+ if el.name == "meta":
44
+ return el.get("content", "").strip()
45
+ return el.get_text(strip=True)
46
+ return ""
47
+
48
+
49
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
50
+ """Extract authors using meta tags first, then DOM."""
51
+ authors = []
52
+ # Meta tags (most reliable for ScienceDirect)
53
+ for meta in soup.select("meta[name='citation_author']"):
54
+ name = meta.get("content", "").strip()
55
+ if name:
56
+ authors.append(name)
57
+ if not authors:
58
+ for meta in soup.select("meta[name='dc.creator']"):
59
+ name = meta.get("content", "").strip()
60
+ if name:
61
+ authors.append(name)
62
+ if authors:
63
+ return authors
64
+
65
+ # DOM selectors
66
+ for selector in [
67
+ ".author-group .author span.content",
68
+ "a.author-name",
69
+ "[data-test='author-name']",
70
+ ]:
71
+ for el in soup.select(selector):
72
+ name = el.get_text(strip=True)
73
+ if name:
74
+ authors.append(name)
75
+ if authors:
76
+ return authors
77
+ return authors
78
+
79
+
80
+ def _extract_abstract(soup: BeautifulSoup) -> str:
81
+ for selector in [
82
+ "div.abstract",
83
+ "#abstracts",
84
+ "div.Abstracts",
85
+ "section#abstract",
86
+ "meta[name='description']",
87
+ "meta[property='og:description']",
88
+ ]:
89
+ el = soup.select_one(selector)
90
+ if el:
91
+ if el.name == "meta":
92
+ return el.get("content", "").strip()
93
+ return _clean(el.get_text())
94
+ return ""
95
+
96
+
97
+ def _extract_body(soup: BeautifulSoup) -> str:
98
+ parts = []
99
+
100
+ # ScienceDirect article body
101
+ body = soup.select_one("div#body") or soup.select_one("div.Body")
102
+ if body:
103
+ for section in body.find_all("section", recursive=False):
104
+ heading = section.find(re.compile(r"h[2-4]"))
105
+ heading_text = heading.get_text(strip=True) if heading else ""
106
+ level = int(heading.name[1]) if heading else 2
107
+
108
+ if heading_text.lower() in ("abstract", "references", "supplementary material"):
109
+ continue
110
+
111
+ # Extract text with tables
112
+ text_parts = []
113
+ for child in section.children:
114
+ if child.name:
115
+ if child.name == "table":
116
+ table_text = _extract_table(child)
117
+ if table_text:
118
+ text_parts.append(table_text)
119
+ elif not child.find("section"): # Skip nested sections
120
+ text = _clean(child.get_text())
121
+ if text:
122
+ text_parts.append(text)
123
+
124
+ content = "\n\n".join(text_parts)
125
+ if heading_text and content:
126
+ prefix = "#" * min(level, 4)
127
+ parts.append(f"{prefix} {heading_text}\n\n{content}")
128
+ elif content:
129
+ parts.append(content)
130
+
131
+ if not parts and body:
132
+ parts.append(_clean(body.get_text()))
133
+
134
+ # Fallback
135
+ if not parts:
136
+ article = soup.select_one("article") or soup.select_one("#main-content")
137
+ if article:
138
+ parts.append(_clean(article.get_text()))
139
+
140
+ return "\n\n".join(parts)
141
+
142
+
143
+ def _extract_table(table) -> str:
144
+ """Extract table content as structured text."""
145
+ rows = []
146
+ for tr in table.find_all("tr"):
147
+ cells = []
148
+ for td in tr.find_all(["td", "th"]):
149
+ cells.append(_clean(td.get_text()))
150
+ if any(cells):
151
+ rows.append(" | ".join(cells))
152
+ return "\n".join(rows)
153
+
154
+
155
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
156
+ captions = []
157
+ for fig in soup.select("figure, .figure"):
158
+ cap = fig.select_one("figcaption, .caption")
159
+ if cap:
160
+ text = _clean(cap.get_text())
161
+ if text and len(text) > 10:
162
+ captions.append(text)
163
+ return captions
164
+
165
+
166
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
167
+ refs = []
168
+ ref_section = soup.select_one("#bibliography") or soup.select_one("section.bibliography")
169
+ if ref_section:
170
+ for li in ref_section.find_all("li"):
171
+ text = _clean(li.get_text())
172
+ if text and len(text) > 20:
173
+ refs.append(text)
174
+ return refs
175
+
176
+
177
+ def _clean(text: str) -> str:
178
+ return re.sub(r"\s+", " ", text).strip()
@@ -0,0 +1,191 @@
1
+ """Generic HTML extractor - fallback for unknown publishers."""
2
+
3
+ import re
4
+
5
+ from bs4 import BeautifulSoup, Tag
6
+
7
+
8
+ def extract(html: str, url: str = "") -> dict:
9
+ """Extract paper content from generic HTML.
10
+
11
+ Args:
12
+ html: Raw HTML content.
13
+ url: The URL the HTML was fetched from.
14
+
15
+ Returns:
16
+ Dict with title, abstract, full_text, figures, references.
17
+ """
18
+ soup = BeautifulSoup(html, "lxml")
19
+
20
+ # Remove script, style, nav, header, footer elements
21
+ for tag in soup.find_all(["script", "style", "nav", "header", "footer", "aside"]):
22
+ tag.decompose()
23
+
24
+ title = _extract_title(soup)
25
+ abstract = _extract_abstract(soup)
26
+ full_text = _extract_body(soup)
27
+ figures = _extract_figures(soup)
28
+ references = _extract_references(soup)
29
+ authors = _extract_authors(soup)
30
+
31
+ return {
32
+ "title": title,
33
+ "authors": authors,
34
+ "abstract": abstract,
35
+ "full_text": full_text,
36
+ "figures": figures,
37
+ "references": references,
38
+ }
39
+
40
+
41
+ def _extract_title(soup: BeautifulSoup) -> str:
42
+ """Extract article title."""
43
+ # Try common title selectors
44
+ for selector in [
45
+ "h1.article-title",
46
+ "h1.c-article-title",
47
+ "h1#title",
48
+ ".article-header h1",
49
+ "article h1",
50
+ "h1",
51
+ ]:
52
+ el = soup.select_one(selector)
53
+ if el and el.get_text(strip=True):
54
+ return el.get_text(strip=True)
55
+
56
+ # Fallback to <title> tag
57
+ title_tag = soup.find("title")
58
+ if title_tag:
59
+ return title_tag.get_text(strip=True)
60
+ return ""
61
+
62
+
63
+ def _extract_abstract(soup: BeautifulSoup) -> str:
64
+ """Extract abstract."""
65
+ for selector in [
66
+ "#abstract",
67
+ ".abstract",
68
+ "[data-title='Abstract']",
69
+ "section.abstract",
70
+ "div.abstractSection",
71
+ ]:
72
+ el = soup.select_one(selector)
73
+ if el:
74
+ return _clean_text(el.get_text())
75
+
76
+ # Try meta tag
77
+ meta = soup.find("meta", attrs={"name": "description"})
78
+ if meta and meta.get("content"):
79
+ return meta["content"].strip()
80
+
81
+ return ""
82
+
83
+
84
+ def _extract_body(soup: BeautifulSoup) -> str:
85
+ """Extract main body text."""
86
+ # Try common article body selectors
87
+ for selector in [
88
+ "article",
89
+ "[role='main']",
90
+ "main",
91
+ ".article-body",
92
+ ".article-content",
93
+ "#body",
94
+ ".body",
95
+ ]:
96
+ el = soup.select_one(selector)
97
+ if el:
98
+ text = _clean_text(el.get_text())
99
+ if len(text) > 500: # Sanity check - body should be substantial
100
+ return text
101
+
102
+ # Fallback: find the largest text block
103
+ best = ""
104
+ for tag in soup.find_all(["div", "section"]):
105
+ text = _clean_text(tag.get_text())
106
+ if len(text) > len(best):
107
+ best = text
108
+
109
+ return best
110
+
111
+
112
+ def _extract_figures(soup: BeautifulSoup) -> list[str]:
113
+ """Extract figure captions."""
114
+ captions = []
115
+ for fig in soup.find_all("figure"):
116
+ cap = fig.find("figcaption")
117
+ if cap:
118
+ text = _clean_text(cap.get_text())
119
+ if text:
120
+ captions.append(text)
121
+
122
+ # Also try common caption classes
123
+ if not captions:
124
+ for selector in [".figure-caption", ".caption", ".fig-caption"]:
125
+ for el in soup.select(selector):
126
+ text = _clean_text(el.get_text())
127
+ if text and len(text) > 10:
128
+ captions.append(text)
129
+
130
+ return captions
131
+
132
+
133
+ def _extract_references(soup: BeautifulSoup) -> list[str]:
134
+ """Extract reference list."""
135
+ refs = []
136
+
137
+ # Try common reference section selectors
138
+ ref_section = None
139
+ for selector in [
140
+ "#references",
141
+ ".references",
142
+ "#bibliography",
143
+ "[data-title='References']",
144
+ "section.ref-list",
145
+ ]:
146
+ ref_section = soup.select_one(selector)
147
+ if ref_section:
148
+ break
149
+
150
+ if ref_section:
151
+ for li in ref_section.find_all("li"):
152
+ text = _clean_text(li.get_text())
153
+ if text and len(text) > 20:
154
+ refs.append(text)
155
+
156
+ # If no <li>, try <p> or <div> items
157
+ if not refs:
158
+ for el in ref_section.find_all(["p", "div"], class_=re.compile(r"ref|citation")):
159
+ text = _clean_text(el.get_text())
160
+ if text and len(text) > 20:
161
+ refs.append(text)
162
+
163
+ return refs
164
+
165
+
166
+ def _extract_authors(soup: BeautifulSoup) -> list[str]:
167
+ """Extract author names."""
168
+ authors = []
169
+ for selector in [
170
+ "meta[name='citation_author']",
171
+ "meta[name='dc.creator']",
172
+ ]:
173
+ for meta in soup.select(selector):
174
+ name = meta.get("content", "").strip()
175
+ if name:
176
+ authors.append(name)
177
+
178
+ if not authors:
179
+ for selector in [".author-name", ".authors a", ".contrib-author"]:
180
+ for el in soup.select(selector):
181
+ name = el.get_text(strip=True)
182
+ if name:
183
+ authors.append(name)
184
+
185
+ return authors
186
+
187
+
188
+ def _clean_text(text: str) -> str:
189
+ """Clean extracted text."""
190
+ text = re.sub(r"\s+", " ", text)
191
+ return text.strip()