litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,163 @@
1
+ """Title-to-DOI resolver via OpenAlex search with similarity matching."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import time
7
+ from difflib import SequenceMatcher
8
+ from typing import Any
9
+
10
+ from ..citations.paper_list import PaperEntry
11
+
12
+ log = logging.getLogger(__name__)
13
+
14
+ TITLE_SIMILARITY_THRESHOLD = 0.75
15
+
16
+
17
+ def _words(text: str) -> list[str]:
18
+ import re
19
+ return [w for w in re.findall(r"[a-z0-9]+", text.lower()) if len(w) > 2]
20
+
21
+
22
+ def resolve_reference(line: str, config: dict[str, Any], timeout: float = 15.0) -> dict[str, Any] | None:
23
+ """The paper a reference line (or a bare title) names, by Crossref's
24
+ ``query.bibliographic`` (built for matching whole citations; free, and in
25
+ the polite pool with the contact email). Accepted only when nearly all of
26
+ the found title's words are in the line, so a near miss never becomes a
27
+ download. Returns {"doi", "title", "year"} or None."""
28
+ import requests
29
+
30
+ from .. import quota
31
+ line = " ".join((line or "").split())
32
+ if len(line) < 15:
33
+ return None
34
+ email = str(config.get("email") or "").strip()
35
+ provider = "crossref_polite" if email else "crossref"
36
+ params = {"query.bibliographic": line[:600], "rows": 3, "select": "DOI,title,issued"}
37
+ if email:
38
+ params["mailto"] = email
39
+ try:
40
+ with quota.slot(provider):
41
+ r = requests.get("https://api.crossref.org/works", params=params, timeout=timeout)
42
+ quota.record("crossref", r)
43
+ if r.status_code != 200:
44
+ return None
45
+ items = r.json().get("message", {}).get("items", [])
46
+ except (requests.RequestException, ValueError):
47
+ return None
48
+ have = set(_words(line))
49
+ for it in items:
50
+ title = ((it.get("title") or [""])[0] or "").strip()
51
+ words = _words(title)
52
+ if len(words) < 3 or not it.get("DOI"):
53
+ continue
54
+ if sum(1 for w in words if w in have) / len(words) >= 0.85:
55
+ parts = (it.get("issued") or {}).get("date-parts") or [[None]]
56
+ return {"doi": str(it["DOI"]).lower(), "title": title, "year": parts[0][0] if parts and parts[0] else None}
57
+ return None
58
+
59
+
60
+ def resolve_references(lines: list[str], config: dict[str, Any], budget_s: float = 60.0) -> list[dict[str, Any]]:
61
+ """resolve_reference for each line, three at a time (the polite pool's
62
+ limit), within ``budget_s``; [{"line", "doi", "title", "year"} | {"line"}]."""
63
+ from concurrent.futures import ThreadPoolExecutor, wait
64
+ ex = ThreadPoolExecutor(max_workers=3)
65
+ futs = [ex.submit(resolve_reference, ln, config) for ln in lines]
66
+ wait(futs, timeout=budget_s)
67
+ ex.shutdown(wait=False, cancel_futures=True)
68
+ out = []
69
+ for ln, f in zip(lines, futs):
70
+ hit = f.result() if f.done() and not f.cancelled() and f.exception() is None else None
71
+ out.append({"line": ln, **hit} if hit else {"line": ln})
72
+ return out
73
+
74
+
75
+ def resolve_title_to_doi(title: str, config: dict[str, Any]) -> str | None:
76
+ """Search OpenAlex by title and return best-matching DOI."""
77
+ from ..network import _get_session, request_timeout
78
+ from .quick import _reconstruct_abstract
79
+
80
+ if not title or len(title) < 10:
81
+ return None
82
+
83
+ try:
84
+ session = _get_session(config)
85
+ resp = session.get(
86
+ "https://api.openalex.org/works",
87
+ params={"search": title, "per_page": 5},
88
+ timeout=request_timeout(config),
89
+ )
90
+ if resp.status_code != 200:
91
+ return None
92
+ data = resp.json()
93
+ except Exception as exc:
94
+ log.debug("Title search failed for '%s': %s", title[:50], exc)
95
+ return None
96
+
97
+ best_doi = None
98
+ best_score = 0.0
99
+ title_lower = title.lower().strip()
100
+
101
+ for work in data.get("results", []):
102
+ result_title = (work.get("title") or "").lower().strip()
103
+ if not result_title:
104
+ continue
105
+ score = SequenceMatcher(None, title_lower, result_title).ratio()
106
+ if score > best_score:
107
+ best_score = score
108
+ doi_raw = work.get("doi", "") or ""
109
+ best_doi = doi_raw.replace("https://doi.org/", "") if doi_raw else None
110
+
111
+ if best_score >= TITLE_SIMILARITY_THRESHOLD and best_doi:
112
+ log.info("Resolved title '%s...' → %s (score=%.2f)", title[:40], best_doi, best_score)
113
+ return best_doi
114
+
115
+ log.debug("No match for '%s...' (best score=%.2f)", title[:40], best_score)
116
+ return None
117
+
118
+
119
+ def batch_resolve(
120
+ entries: list[PaperEntry],
121
+ config: dict[str, Any],
122
+ delay: float = 0.3,
123
+ ) -> dict[str, Any]:
124
+ """Batch resolve: fix DOI format + search by title for missing DOIs.
125
+
126
+ Returns dict with 'entries' (resolved list) and 'stats'.
127
+ """
128
+ total = len(entries)
129
+ fixed_unicode = 0
130
+ resolved_by_title = 0
131
+ already_has_doi = 0
132
+ unresolvable = 0
133
+
134
+ for i, entry in enumerate(entries):
135
+ if entry.doi:
136
+ already_has_doi += 1
137
+ continue
138
+
139
+ if not entry.title:
140
+ unresolvable += 1
141
+ continue
142
+
143
+ # Search by title
144
+ doi = resolve_title_to_doi(entry.title, config)
145
+ if doi:
146
+ entry.doi = doi
147
+ resolved_by_title += 1
148
+ else:
149
+ unresolvable += 1
150
+
151
+ # Rate limit for OpenAlex API
152
+ if delay > 0 and i < total - 1:
153
+ time.sleep(delay)
154
+
155
+ return {
156
+ "entries": entries,
157
+ "stats": {
158
+ "total": total,
159
+ "already_has_doi": already_has_doi,
160
+ "resolved_by_title": resolved_by_title,
161
+ "unresolvable": unresolvable,
162
+ },
163
+ }
@@ -0,0 +1,124 @@
1
+ """Saved searches (定期追踪): run one again and report only papers not seen before.
2
+
3
+ A watch is a question plus the query that answers it, remembered from a screening
4
+ session. Checking one runs the same keyword search, drops every paper already in
5
+ that session or in an earlier check, and keeps the rest so the next check starts
6
+ from there. Nothing here knows about scheduling; the app decides when to ask.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import threading
12
+ import time
13
+ import uuid
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .. import config, db
18
+ from ..identifiers import canonical_doi
19
+
20
+ MAX_SEEN = 5000 # own value: enough history for years of one topic
21
+ NEW_LIMIT = 50 # own value: how many hits one check pulls before dedupe
22
+
23
+
24
+ def path() -> Path:
25
+ """Legacy JSON path, kept for backward compatibility and one-time import."""
26
+ return Path(config.DATA_DIR) / "watches.json"
27
+
28
+
29
+ def _ensure_legacy_imported() -> None:
30
+ legacy = path()
31
+ if legacy.exists():
32
+ db.import_legacy_watches(legacy_path=legacy)
33
+
34
+
35
+ def key(paper: dict[str, Any]) -> str:
36
+ """A stable identity for one paper: its DOI, else its title."""
37
+ doi = canonical_doi(str(paper.get("doi") or ""))
38
+ if doi:
39
+ return f"doi:{doi.lower()}"
40
+ return "title:" + " ".join(str(paper.get("title") or "").lower().split())
41
+
42
+
43
+ def list_watches() -> list[dict[str, Any]]:
44
+ """Newest first; the seen list stays in the database (it can be thousands long)."""
45
+ _ensure_legacy_imported()
46
+ return db.list_watches()
47
+
48
+
49
+ def get(wid: str) -> dict[str, Any] | None:
50
+ _ensure_legacy_imported()
51
+ return db.get_watch(wid)
52
+
53
+
54
+ def add(question: str, query: str, core_terms: list[str] | None = None,
55
+ seen: list[dict[str, Any]] | None = None, session: str = "") -> dict[str, Any]:
56
+ """Remember a search; papers passed in ``seen`` are not reported as new later."""
57
+ _ensure_legacy_imported()
58
+ query = " ".join(str(query or "").split())
59
+ if not query:
60
+ raise ValueError("无法保存追踪:检索式是空的")
61
+ keys = [key(p) for p in seen or []][:MAX_SEEN]
62
+ return db.add_watch(
63
+ wid=uuid.uuid4().hex[:8],
64
+ question=" ".join(str(question or "").split()),
65
+ query=query,
66
+ core_terms=list(core_terms or []),
67
+ seen_keys=keys,
68
+ session=session,
69
+ )
70
+
71
+
72
+ def remove(wid: str) -> bool:
73
+ _ensure_legacy_imported()
74
+ return db.remove_watch(wid)
75
+
76
+
77
+ def check(wid: str, config: dict[str, Any] | None = None) -> dict[str, Any]:
78
+ """Run the saved search once; return only the papers this watch has not seen."""
79
+ from . import quick
80
+ _ensure_legacy_imported()
81
+ watch = db.get_watch(wid)
82
+ if not watch:
83
+ raise ValueError("没有这个追踪")
84
+ hits = quick.search_papers_v2(watch["query"], limit=NEW_LIMIT, sort="date", use_cache=False) or []
85
+ seen = set(watch.get("seen") or [])
86
+ fresh = [p for p in hits if key(p) not in seen]
87
+ now = time.strftime("%Y-%m-%d %H:%M")
88
+ keys = list(watch.get("seen") or []) + [key(x) for x in hits]
89
+ merged_seen = list(dict.fromkeys(keys))[-MAX_SEEN:]
90
+ db.update_watch_check(wid, seen_keys=merged_seen, checked=now, last_new=len(fresh))
91
+ return {"id": wid, "checked": now, "found": len(hits), "new": fresh}
92
+
93
+
94
+ _DUE_LOCK = threading.Lock()
95
+
96
+
97
+ def check_due(config: dict[str, Any] | None = None) -> dict[str, Any]:
98
+ """Check the watches whose last check is older than `watch_check_days`.
99
+
100
+ Called once when the app opens, so a followed topic does not need a click;
101
+ the default (7 days) keeps the OpenAlex credits spent on this small. Returns
102
+ the per-watch outcome and how many new papers came back in total.
103
+ """
104
+ from ..config import load_config
105
+ config = config or load_config()
106
+ days = float(config.get("watch_check_days") or 7)
107
+ if not _DUE_LOCK.acquire(blocking=False):
108
+ return {"checked": [], "new_total": 0, "busy": True}
109
+ try:
110
+ _ensure_legacy_imported()
111
+ done: list[dict[str, Any]] = []
112
+ total = 0
113
+ for w in db.due_watches(days):
114
+ try:
115
+ r = check(str(w["id"]), config)
116
+ except Exception as exc: # one broken search must not stop the rest
117
+ done.append({"id": w["id"], "question": w.get("question") or w.get("query"), "error": str(exc)})
118
+ continue
119
+ done.append({"id": r["id"], "question": w.get("question") or w.get("query"),
120
+ "found": r["found"], "new": len(r["new"])})
121
+ total += len(r["new"])
122
+ return {"checked": done, "new_total": total}
123
+ finally:
124
+ _DUE_LOCK.release()