litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/db.py ADDED
@@ -0,0 +1,1922 @@
1
+ """SQLite persistence layer for LitLoop (sessions, pool items, decisions, watches).
2
+
3
+ Phase 1 introduces the schema, WAL-mode connections with explicit closing,
4
+ versioned migrations via ``PRAGMA user_version`` + ``schema_migrations``,
5
+ idempotent legacy JSON import, and watch table operations.
6
+ Phase 2 adds screening session & decision operations with automatic on-demand
7
+ import of legacy ``<session>.json`` files.
8
+ Phase 3 adds deep-search candidate pool & summary operations with automatic
9
+ on-demand import of legacy ``<session>.pool.json`` / ``<session>.summary.json`` files.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import re
16
+ import sqlite3
17
+ import time
18
+ from contextlib import contextmanager
19
+ from datetime import datetime, timedelta
20
+ from pathlib import Path
21
+ from typing import Any, Callable, Iterator
22
+
23
+ from . import config, store
24
+ from .log import get_logger
25
+ from .terms.text import paper_key
26
+
27
+ logger = get_logger()
28
+
29
+ DB_SCHEMA = 1
30
+
31
+
32
+ def db_path() -> Path:
33
+ """Return the SQLite database path under ``config.DATA_DIR``, creating parents."""
34
+ p = Path(config.DATA_DIR) / "litloop.db"
35
+ p.parent.mkdir(parents=True, exist_ok=True)
36
+ return p
37
+
38
+
39
+ def _resolve_db_file(data_dir: Path | str | None) -> Path | None:
40
+ if data_dir is None:
41
+ return None
42
+ return Path(data_dir) / "litloop.db"
43
+
44
+
45
+ def _now_iso() -> str:
46
+ return time.strftime("%Y-%m-%d %H:%M:%S")
47
+
48
+
49
+ def _migrate_v1(conn: sqlite3.Connection) -> None:
50
+ statements = [
51
+ """
52
+ CREATE TABLE IF NOT EXISTS sessions (
53
+ id TEXT PRIMARY KEY,
54
+ schema_version INTEGER NOT NULL DEFAULT 1,
55
+ created TEXT NOT NULL DEFAULT '',
56
+ updated TEXT NOT NULL DEFAULT '',
57
+ question TEXT NOT NULL DEFAULT '',
58
+ mode TEXT NOT NULL DEFAULT 'quick',
59
+ criteria_json TEXT NOT NULL DEFAULT '[]',
60
+ searches_json TEXT NOT NULL DEFAULT '[]',
61
+ summary_json TEXT,
62
+ pool_state_json TEXT
63
+ )
64
+ """,
65
+ """
66
+ CREATE TABLE IF NOT EXISTS pool_items (
67
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
68
+ paper_key TEXT NOT NULL,
69
+ doi TEXT,
70
+ openalex TEXT,
71
+ title TEXT NOT NULL DEFAULT '',
72
+ year INTEGER,
73
+ journal TEXT NOT NULL DEFAULT '',
74
+ cited_by_count INTEGER NOT NULL DEFAULT 0,
75
+ abstract TEXT NOT NULL DEFAULT '',
76
+ score REAL,
77
+ data_json TEXT NOT NULL DEFAULT '{}',
78
+ PRIMARY KEY (session_id, paper_key)
79
+ )
80
+ """,
81
+ "CREATE INDEX IF NOT EXISTS idx_pool_items_session_score ON pool_items(session_id, score DESC)",
82
+ "CREATE INDEX IF NOT EXISTS idx_pool_items_doi ON pool_items(doi)",
83
+ """
84
+ CREATE TABLE IF NOT EXISTS decisions (
85
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
86
+ paper_key TEXT NOT NULL,
87
+ n INTEGER,
88
+ doi TEXT,
89
+ title TEXT NOT NULL DEFAULT '',
90
+ year INTEGER,
91
+ journal TEXT NOT NULL DEFAULT '',
92
+ journal_rank TEXT NOT NULL DEFAULT '',
93
+ authors_json TEXT NOT NULL DEFAULT '[]',
94
+ cited_by_count INTEGER NOT NULL DEFAULT 0,
95
+ abstract TEXT NOT NULL DEFAULT '',
96
+ abstract_zh TEXT,
97
+ abstract_missing INTEGER,
98
+ retracted INTEGER,
99
+ concern INTEGER,
100
+ relevance TEXT NOT NULL DEFAULT '',
101
+ reason TEXT NOT NULL DEFAULT '',
102
+ user_relevance TEXT,
103
+ decision TEXT,
104
+ reject_reason TEXT,
105
+ data_json TEXT NOT NULL DEFAULT '{}',
106
+ PRIMARY KEY (session_id, paper_key)
107
+ )
108
+ """,
109
+ "CREATE INDEX IF NOT EXISTS idx_decisions_session_n ON decisions(session_id, n)",
110
+ "CREATE INDEX IF NOT EXISTS idx_decisions_doi ON decisions(doi)",
111
+ """
112
+ CREATE TABLE IF NOT EXISTS watches (
113
+ id TEXT PRIMARY KEY,
114
+ question TEXT NOT NULL DEFAULT '',
115
+ query TEXT NOT NULL,
116
+ core_terms_json TEXT NOT NULL DEFAULT '[]',
117
+ session TEXT NOT NULL DEFAULT '',
118
+ created TEXT NOT NULL DEFAULT '',
119
+ checked TEXT NOT NULL DEFAULT '',
120
+ last_new INTEGER NOT NULL DEFAULT 0,
121
+ seen_json TEXT NOT NULL DEFAULT '[]'
122
+ )
123
+ """,
124
+ "CREATE INDEX IF NOT EXISTS idx_watches_checked ON watches(checked)",
125
+ ]
126
+ for stmt in statements:
127
+ conn.execute(stmt)
128
+
129
+
130
+ MIGRATIONS: list[tuple[int, str, Callable[[sqlite3.Connection], None]]] = [
131
+ (1, "initial schema", _migrate_v1),
132
+ ]
133
+
134
+
135
+ def _apply_pragmas(conn: sqlite3.Connection) -> None:
136
+ conn.execute("PRAGMA journal_mode = WAL")
137
+ conn.execute("PRAGMA busy_timeout = 30000")
138
+ conn.execute("PRAGMA foreign_keys = ON")
139
+ conn.execute("PRAGMA synchronous = NORMAL")
140
+ conn.row_factory = sqlite3.Row
141
+
142
+
143
+ def _run_migrations(conn: sqlite3.Connection) -> None:
144
+ ver = int(conn.execute("PRAGMA user_version").fetchone()[0])
145
+ if ver > DB_SCHEMA:
146
+ conn.close()
147
+ raise store.SchemaTooNew(
148
+ f"litloop.db was written by a newer LitLoop (schema {ver}, "
149
+ f"this copy knows {DB_SCHEMA}); update LitLoop"
150
+ )
151
+ if ver >= DB_SCHEMA:
152
+ return
153
+
154
+ conn.execute("BEGIN IMMEDIATE")
155
+ try:
156
+ ver = int(conn.execute("PRAGMA user_version").fetchone()[0])
157
+ if ver > DB_SCHEMA:
158
+ raise store.SchemaTooNew(
159
+ f"litloop.db was written by a newer LitLoop (schema {ver}, "
160
+ f"this copy knows {DB_SCHEMA}); update LitLoop"
161
+ )
162
+ conn.execute(
163
+ """
164
+ CREATE TABLE IF NOT EXISTS schema_migrations (
165
+ version INTEGER PRIMARY KEY,
166
+ applied_at TEXT NOT NULL,
167
+ note TEXT NOT NULL
168
+ )
169
+ """
170
+ )
171
+ for m_ver, m_note, m_fn in sorted(MIGRATIONS, key=lambda item: item[0]):
172
+ if ver < m_ver <= DB_SCHEMA:
173
+ m_fn(conn)
174
+ conn.execute(
175
+ "INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
176
+ "VALUES (?, ?, ?)",
177
+ (m_ver, _now_iso(), m_note),
178
+ )
179
+ conn.execute(f"PRAGMA user_version = {int(m_ver)}")
180
+ ver = m_ver
181
+ if conn.in_transaction:
182
+ conn.execute("COMMIT")
183
+ except BaseException:
184
+ if conn.in_transaction:
185
+ try:
186
+ conn.execute("ROLLBACK")
187
+ except sqlite3.Error:
188
+ pass
189
+ conn.close()
190
+ raise
191
+
192
+
193
+ def _open_connection(db_file: Path | str | None = None) -> sqlite3.Connection:
194
+ target = Path(db_file) if db_file is not None else db_path()
195
+ target.parent.mkdir(parents=True, exist_ok=True)
196
+ conn = sqlite3.connect(str(target), timeout=30.0, isolation_level=None)
197
+ try:
198
+ _apply_pragmas(conn)
199
+ _run_migrations(conn)
200
+ except BaseException:
201
+ try:
202
+ conn.close()
203
+ except sqlite3.Error:
204
+ pass
205
+ raise
206
+ return conn
207
+
208
+
209
+ @contextmanager
210
+ def connect(db_file: Path | str | None = None) -> Iterator[sqlite3.Connection]:
211
+ """Open a WAL-mode SQLite connection and always close it on exit."""
212
+ conn = _open_connection(db_file)
213
+ try:
214
+ yield conn
215
+ finally:
216
+ conn.close()
217
+
218
+
219
+ @contextmanager
220
+ def transaction(db_file: Path | str | None = None) -> Iterator[sqlite3.Connection]:
221
+ """Open a connection wrapped in an IMMEDIATE transaction, closed on exit."""
222
+ with connect(db_file) as conn:
223
+ conn.execute("BEGIN IMMEDIATE")
224
+ try:
225
+ yield conn
226
+ except BaseException:
227
+ if conn.in_transaction:
228
+ conn.execute("ROLLBACK")
229
+ raise
230
+ else:
231
+ if conn.in_transaction:
232
+ conn.execute("COMMIT")
233
+
234
+
235
+ # --------------------------------------------------------------------------- watches
236
+
237
+
238
+ def _watch_row_to_dict(row: sqlite3.Row, *, include_seen: bool) -> dict[str, Any]:
239
+ seen = json.loads(row["seen_json"] or "[]")
240
+ core_terms = json.loads(row["core_terms_json"] or "[]")
241
+ out: dict[str, Any] = {
242
+ "id": row["id"],
243
+ "question": row["question"],
244
+ "query": row["query"],
245
+ "core_terms": core_terms,
246
+ "session": row["session"],
247
+ "created": row["created"],
248
+ "checked": row["checked"],
249
+ "last_new": row["last_new"],
250
+ }
251
+ if include_seen:
252
+ out["seen"] = seen
253
+ else:
254
+ out["seen_count"] = len(seen)
255
+ return out
256
+
257
+
258
+ def list_watches(db_file: Path | str | None = None) -> list[dict[str, Any]]:
259
+ """Return watches newest-first without the full ``seen`` list."""
260
+ with connect(db_file) as conn:
261
+ rows = conn.execute("SELECT * FROM watches").fetchall()
262
+ items = [_watch_row_to_dict(r, include_seen=False) for r in rows]
263
+ return sorted(items, key=lambda w: str(w.get("checked") or w.get("created") or ""), reverse=True)
264
+
265
+
266
+ def get_watch(wid: str, db_file: Path | str | None = None) -> dict[str, Any] | None:
267
+ """Return the full watch dict (including ``seen``), or ``None`` if absent."""
268
+ with connect(db_file) as conn:
269
+ row = conn.execute("SELECT * FROM watches WHERE id = ?", (wid,)).fetchone()
270
+ return _watch_row_to_dict(row, include_seen=True) if row is not None else None
271
+
272
+
273
+ def add_watch(
274
+ wid: str,
275
+ question: str,
276
+ query: str,
277
+ core_terms: list[str] | None = None,
278
+ seen_keys: list[str] | None = None,
279
+ session: str = "",
280
+ created: str | None = None,
281
+ checked: str = "",
282
+ last_new: int = 0,
283
+ db_file: Path | str | None = None,
284
+ ) -> dict[str, Any]:
285
+ """Insert a watch row and return its summary dict."""
286
+ created_str = created if created is not None else time.strftime("%Y-%m-%d %H:%M")
287
+ core_list = list(core_terms or [])
288
+ seen_list = list(seen_keys or [])
289
+ with transaction(db_file) as conn:
290
+ conn.execute(
291
+ """
292
+ INSERT INTO watches (
293
+ id, question, query, core_terms_json, session,
294
+ created, checked, last_new, seen_json
295
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
296
+ """,
297
+ (
298
+ wid,
299
+ question,
300
+ query,
301
+ json.dumps(core_list, ensure_ascii=False),
302
+ session,
303
+ created_str,
304
+ checked,
305
+ int(last_new),
306
+ json.dumps(seen_list, ensure_ascii=False),
307
+ ),
308
+ )
309
+ return {
310
+ "id": wid,
311
+ "question": question,
312
+ "query": query,
313
+ "core_terms": core_list,
314
+ "session": session,
315
+ "created": created_str,
316
+ "checked": checked,
317
+ "last_new": int(last_new),
318
+ "seen_count": len(seen_list),
319
+ }
320
+
321
+
322
+ def remove_watch(wid: str, db_file: Path | str | None = None) -> bool:
323
+ """Delete a watch by id; return ``True`` if a row was removed."""
324
+ with transaction(db_file) as conn:
325
+ cur = conn.execute("DELETE FROM watches WHERE id = ?", (wid,))
326
+ return cur.rowcount > 0
327
+
328
+
329
+ def update_watch_check(
330
+ wid: str,
331
+ *,
332
+ seen_keys: list[str],
333
+ checked: str,
334
+ last_new: int,
335
+ db_file: Path | str | None = None,
336
+ ) -> bool:
337
+ """Update ``seen_json``, ``checked`` and ``last_new`` after a check run."""
338
+ with transaction(db_file) as conn:
339
+ cur = conn.execute(
340
+ """
341
+ UPDATE watches
342
+ SET seen_json = ?, checked = ?, last_new = ?
343
+ WHERE id = ?
344
+ """,
345
+ (json.dumps(list(seen_keys), ensure_ascii=False), checked, int(last_new), wid),
346
+ )
347
+ return cur.rowcount > 0
348
+
349
+
350
+ def due_watches(
351
+ days: float,
352
+ *,
353
+ now: datetime | None = None,
354
+ db_file: Path | str | None = None,
355
+ ) -> list[dict[str, Any]]:
356
+ """Return watches never checked or checked at least ``days`` ago."""
357
+ ref = now or datetime.now()
358
+ threshold = timedelta(days=float(days))
359
+ with connect(db_file) as conn:
360
+ rows = conn.execute("SELECT * FROM watches").fetchall()
361
+ due: list[dict[str, Any]] = []
362
+ for row in rows:
363
+ checked = str(row["checked"] or "")
364
+ if checked:
365
+ try:
366
+ if ref - datetime.fromisoformat(checked) < threshold:
367
+ continue
368
+ except ValueError:
369
+ pass
370
+ due.append(_watch_row_to_dict(row, include_seen=True))
371
+ return due
372
+
373
+
374
+ # --------------------------------------------------------------------------- legacy JSON import
375
+
376
+
377
+ def _read_json_safe(path: Path) -> Any:
378
+ try:
379
+ return json.loads(path.read_text(encoding="utf-8"))
380
+ except (OSError, ValueError, TypeError) as exc:
381
+ logger.warning("db: skipping unreadable legacy JSON %s (%s)", path, exc)
382
+ return None
383
+
384
+
385
+ def import_legacy_watches(
386
+ data_dir: Path | str | None = None,
387
+ legacy_path: Path | str | None = None,
388
+ ) -> int:
389
+ """Idempotently import ``watches.json`` into SQLite without deleting the file."""
390
+ base = Path(data_dir) if data_dir is not None else Path(config.DATA_DIR)
391
+ src = Path(legacy_path) if legacy_path is not None else base / "watches.json"
392
+ if not src.exists():
393
+ return 0
394
+ data = _read_json_safe(src)
395
+ if not isinstance(data, dict):
396
+ return 0
397
+ raw_watches = data.get("watches")
398
+ if not isinstance(raw_watches, list):
399
+ return 0
400
+
401
+ db_file = _resolve_db_file(data_dir)
402
+ inserted = 0
403
+ with transaction(db_file) as conn:
404
+ for w in raw_watches:
405
+ if not isinstance(w, dict):
406
+ continue
407
+ wid = str(w.get("id") or "").strip()
408
+ query = str(w.get("query") or "").strip()
409
+ if not wid or not query:
410
+ continue
411
+ core_terms = w.get("core_terms") if isinstance(w.get("core_terms"), list) else []
412
+ seen = w.get("seen") if isinstance(w.get("seen"), list) else []
413
+ cur = conn.execute(
414
+ """
415
+ INSERT OR IGNORE INTO watches (
416
+ id, question, query, core_terms_json, session,
417
+ created, checked, last_new, seen_json
418
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
419
+ """,
420
+ (
421
+ wid,
422
+ str(w.get("question") or ""),
423
+ query,
424
+ json.dumps(core_terms, ensure_ascii=False),
425
+ str(w.get("session") or ""),
426
+ str(w.get("created") or ""),
427
+ str(w.get("checked") or ""),
428
+ int(w.get("last_new") or 0),
429
+ json.dumps(seen, ensure_ascii=False),
430
+ ),
431
+ )
432
+ if cur.rowcount > 0:
433
+ inserted += 1
434
+ return inserted
435
+
436
+
437
+ _DECISION_KNOWN_KEYS = frozenset(
438
+ {
439
+ "n",
440
+ "doi",
441
+ "title",
442
+ "year",
443
+ "journal",
444
+ "journal_rank",
445
+ "authors",
446
+ "cited_by_count",
447
+ "abstract",
448
+ "abstract_zh",
449
+ "abstract_missing",
450
+ "retracted",
451
+ "concern",
452
+ "relevance",
453
+ "reason",
454
+ "user_relevance",
455
+ "decision",
456
+ "reject_reason",
457
+ }
458
+ )
459
+
460
+
461
+ def _opt_bool_int(value: Any) -> int | None:
462
+ if value is None:
463
+ return None
464
+ return 1 if bool(value) else 0
465
+
466
+
467
+ def _opt_int(value: Any) -> int | None:
468
+ if value is None or isinstance(value, bool):
469
+ return None
470
+ try:
471
+ return int(value)
472
+ except (TypeError, ValueError):
473
+ return None
474
+
475
+
476
+ def _opt_float(value: Any) -> float | None:
477
+ if value is None or isinstance(value, bool):
478
+ return None
479
+ try:
480
+ return float(value)
481
+ except (TypeError, ValueError):
482
+ return None
483
+
484
+
485
+ def _collect_session_ids(sdir: Path) -> list[str]:
486
+ sids: set[str] = set()
487
+ for p in sdir.iterdir():
488
+ if not p.is_file():
489
+ continue
490
+ name = p.name
491
+ if name.startswith(".") or not name.endswith(".json"):
492
+ continue
493
+ if name.endswith(".progress.json"):
494
+ continue
495
+ if name.endswith(".summary.json"):
496
+ sids.add(name[: -len(".summary.json")])
497
+ elif name.endswith(".pool.json"):
498
+ sids.add(name[: -len(".pool.json")])
499
+ else:
500
+ sids.add(name[: -len(".json")])
501
+ return sorted(s for s in sids if s)
502
+
503
+
504
+ def _import_session_file(conn: sqlite3.Connection, sid: str, sdata: dict[str, Any], counts: dict[str, int]) -> None:
505
+ criteria = sdata.get("criteria") if isinstance(sdata.get("criteria"), list) else []
506
+ searches = sdata.get("searches") if isinstance(sdata.get("searches"), list) else []
507
+ cur = conn.execute(
508
+ """
509
+ INSERT OR IGNORE INTO sessions (
510
+ id, schema_version, created, updated, question, mode,
511
+ criteria_json, searches_json
512
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
513
+ """,
514
+ (
515
+ sid,
516
+ _opt_int(sdata.get("schema")) or 1,
517
+ str(sdata.get("created") or ""),
518
+ str(sdata.get("updated") or ""),
519
+ str(sdata.get("question") or ""),
520
+ str(sdata.get("mode") or "quick"),
521
+ json.dumps(criteria, ensure_ascii=False),
522
+ json.dumps(searches, ensure_ascii=False),
523
+ ),
524
+ )
525
+ if cur.rowcount > 0:
526
+ counts["sessions"] += 1
527
+
528
+ items = sdata.get("items")
529
+ if not isinstance(items, list):
530
+ return
531
+ for item in items:
532
+ if not isinstance(item, dict):
533
+ continue
534
+ pkey = paper_key(item)
535
+ authors = item.get("authors") if isinstance(item.get("authors"), list) else []
536
+ extra = {k: v for k, v in item.items() if k not in _DECISION_KNOWN_KEYS}
537
+ dcur = conn.execute(
538
+ """
539
+ INSERT OR IGNORE INTO decisions (
540
+ session_id, paper_key, n, doi, title, year, journal,
541
+ journal_rank, authors_json, cited_by_count, abstract,
542
+ abstract_zh, abstract_missing, retracted, concern,
543
+ relevance, reason, user_relevance, decision, reject_reason,
544
+ data_json
545
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
546
+ """,
547
+ (
548
+ sid,
549
+ pkey,
550
+ _opt_int(item.get("n")),
551
+ str(item["doi"]) if item.get("doi") is not None else None,
552
+ str(item.get("title") or ""),
553
+ _opt_int(item.get("year")),
554
+ str(item.get("journal") or ""),
555
+ str(item.get("journal_rank") or ""),
556
+ json.dumps(authors, ensure_ascii=False),
557
+ _opt_int(item.get("cited_by_count")) or 0,
558
+ str(item.get("abstract") or ""),
559
+ str(item["abstract_zh"]) if item.get("abstract_zh") is not None else None,
560
+ _opt_bool_int(item.get("abstract_missing")),
561
+ _opt_bool_int(item.get("retracted")),
562
+ _opt_bool_int(item.get("concern")),
563
+ str(item.get("relevance") or ""),
564
+ str(item.get("reason") or ""),
565
+ str(item["user_relevance"]) if item.get("user_relevance") is not None else None,
566
+ str(item["decision"]) if item.get("decision") is not None else None,
567
+ str(item["reject_reason"]) if item.get("reject_reason") is not None else None,
568
+ json.dumps(extra, ensure_ascii=False),
569
+ ),
570
+ )
571
+ if dcur.rowcount > 0:
572
+ counts["decisions"] += 1
573
+
574
+
575
+ def _import_summary_file(conn: sqlite3.Connection, sid: str, summary: dict[str, Any], counts: dict[str, int]) -> None:
576
+ cur = conn.execute(
577
+ "INSERT OR IGNORE INTO sessions (id, question, mode, updated) VALUES (?, ?, ?, ?)",
578
+ (
579
+ sid,
580
+ str(summary.get("question") or ""),
581
+ str(summary.get("mode") or "deep"),
582
+ str(summary.get("updated") or ""),
583
+ ),
584
+ )
585
+ if cur.rowcount > 0:
586
+ counts["sessions"] += 1
587
+ conn.execute(
588
+ "UPDATE sessions SET summary_json = ? WHERE id = ? AND summary_json IS NULL",
589
+ (json.dumps(summary, ensure_ascii=False), sid),
590
+ )
591
+
592
+
593
+ def _import_pool_file(conn: sqlite3.Connection, sid: str, pdata: dict[str, Any], counts: dict[str, int]) -> None:
594
+ cur = conn.execute(
595
+ "INSERT OR IGNORE INTO sessions (id, question, mode) VALUES (?, ?, ?)",
596
+ (sid, str(pdata.get("question") or ""), "deep"),
597
+ )
598
+ if cur.rowcount > 0:
599
+ counts["sessions"] += 1
600
+
601
+ pool_meta = {k: v for k, v in pdata.items() if k != "pool"}
602
+ conn.execute(
603
+ "UPDATE sessions SET pool_state_json = ? WHERE id = ? AND pool_state_json IS NULL",
604
+ (json.dumps(pool_meta, ensure_ascii=False), sid),
605
+ )
606
+
607
+ pool = pdata.get("pool")
608
+ if not isinstance(pool, dict):
609
+ return
610
+ for pkey, rec in pool.items():
611
+ if not isinstance(rec, dict):
612
+ continue
613
+ pcur = conn.execute(
614
+ """
615
+ INSERT OR IGNORE INTO pool_items (
616
+ session_id, paper_key, doi, openalex, title, year, journal,
617
+ cited_by_count, abstract, score, data_json
618
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
619
+ """,
620
+ (
621
+ sid,
622
+ str(pkey),
623
+ str(rec["doi"]) if rec.get("doi") is not None else None,
624
+ str(rec["openalex"]) if rec.get("openalex") is not None else None,
625
+ str(rec.get("title") or ""),
626
+ _opt_int(rec.get("year")),
627
+ str(rec.get("journal") or ""),
628
+ _opt_int(rec.get("cited_by_count")) or 0,
629
+ str(rec.get("abstract") or ""),
630
+ _opt_float(rec.get("score")),
631
+ json.dumps(rec, ensure_ascii=False),
632
+ ),
633
+ )
634
+ if pcur.rowcount > 0:
635
+ counts["pool_items"] += 1
636
+
637
+
638
+ def import_legacy_data(data_dir: Path | str | None = None) -> dict[str, int]:
639
+ """Idempotently import legacy JSON files (watches + screening sessions/summaries/pools).
640
+
641
+ Never deletes any source JSON file and skips malformed JSON without crashing.
642
+ """
643
+ base = Path(data_dir) if data_dir is not None else Path(config.DATA_DIR)
644
+ counts: dict[str, int] = {
645
+ "sessions": 0,
646
+ "decisions": 0,
647
+ "pool_items": 0,
648
+ "watches": import_legacy_watches(data_dir=data_dir),
649
+ }
650
+ sdir = base / "screening"
651
+ if not sdir.is_dir():
652
+ return counts
653
+
654
+ db_file = _resolve_db_file(data_dir)
655
+ sids = _collect_session_ids(sdir)
656
+ with transaction(db_file) as conn:
657
+ for sid in sids:
658
+ main_file = sdir / f"{sid}.json"
659
+ if main_file.is_file():
660
+ sdata = _read_json_safe(main_file)
661
+ if isinstance(sdata, dict):
662
+ _import_session_file(conn, sid, sdata, counts)
663
+
664
+ summary_file = sdir / f"{sid}.summary.json"
665
+ if summary_file.is_file():
666
+ summary = _read_json_safe(summary_file)
667
+ if isinstance(summary, dict):
668
+ _import_summary_file(conn, sid, summary, counts)
669
+
670
+ pool_file = sdir / f"{sid}.pool.json"
671
+ if pool_file.is_file():
672
+ pdata = _read_json_safe(pool_file)
673
+ if isinstance(pdata, dict):
674
+ _import_pool_file(conn, sid, pdata, counts)
675
+
676
+ return counts
677
+
678
+
679
+ # --------------------------------------------------------------------------- screening sessions & decisions
680
+
681
+ RELEVANCE = ("高", "中", "低", "不相关")
682
+ ABSTRACT_CHARS = 1200
683
+ _RANK = {r: i for i, r in enumerate(RELEVANCE)}
684
+ _EN_RELEVANCE = {"high": "高", "medium": "中", "low": "低", "irrelevant": "不相关", "none": "不相关"}
685
+ _SAFE_SESSION_RE = re.compile(r"^[\w\-. \u4e00-\u9fff]+$")
686
+
687
+
688
+ def _norm_relevance(value: Any) -> str:
689
+ v = str(value or "").strip()
690
+ v = _EN_RELEVANCE.get(v.lower(), v)
691
+ if v not in _RANK:
692
+ raise ValueError(f"relevance must be one of {RELEVANCE}, got {value!r}")
693
+ return v
694
+
695
+
696
+ def _slug(text: str) -> str:
697
+ s = re.sub(r"[^\w\u4e00-\u9fff]+", "-", text.strip().lower()).strip("-")
698
+ return s[:40] or "search"
699
+
700
+
701
+ def _screening_dir(db_file: Path | str | None = None) -> Path:
702
+ if db_file is not None:
703
+ return Path(db_file).parent / "screening"
704
+ from .search.screening import screening_dir
705
+ return screening_dir()
706
+
707
+
708
+ def _decision_brief(it: dict[str, Any]) -> dict[str, Any]:
709
+ return {
710
+ "n": it["n"],
711
+ "relevance": it["relevance"],
712
+ "title": str(it.get("title") or "")[:90],
713
+ "doi": it.get("doi"),
714
+ }
715
+
716
+
717
+ def _decision_row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
718
+ item: dict[str, Any] = {}
719
+ if row["n"] is not None:
720
+ item["n"] = int(row["n"])
721
+ if row["doi"]:
722
+ item["doi"] = row["doi"]
723
+ if row["title"]:
724
+ item["title"] = row["title"]
725
+ if row["year"] is not None:
726
+ item["year"] = int(row["year"])
727
+ if row["journal"]:
728
+ item["journal"] = row["journal"]
729
+ if row["journal_rank"]:
730
+ item["journal_rank"] = row["journal_rank"]
731
+ authors = json.loads(row["authors_json"] or "[]")
732
+ if authors:
733
+ item["authors"] = authors
734
+ if row["cited_by_count"] is not None:
735
+ item["cited_by_count"] = int(row["cited_by_count"])
736
+ if row["abstract"]:
737
+ item["abstract"] = row["abstract"]
738
+ if row["abstract_zh"]:
739
+ item["abstract_zh"] = row["abstract_zh"]
740
+ if row["abstract_missing"] is not None:
741
+ item["abstract_missing"] = bool(row["abstract_missing"])
742
+ if row["retracted"] is not None:
743
+ item["retracted"] = bool(row["retracted"])
744
+ if row["concern"] is not None:
745
+ item["concern"] = bool(row["concern"])
746
+ item["relevance"] = row["relevance"]
747
+ item["reason"] = row["reason"]
748
+ if row["user_relevance"] is not None:
749
+ item["user_relevance"] = row["user_relevance"]
750
+ item["decision"] = row["decision"]
751
+ if row["reject_reason"] is not None:
752
+ item["reject_reason"] = row["reject_reason"]
753
+ extra = json.loads(row["data_json"] or "{}")
754
+ if isinstance(extra, dict) and extra:
755
+ for k, v in extra.items():
756
+ if k not in item:
757
+ item[k] = v
758
+ return item
759
+
760
+
761
+ def _load_session_record(conn: sqlite3.Connection, session: str) -> dict[str, Any]:
762
+ srow = conn.execute("SELECT * FROM sessions WHERE id = ?", (session,)).fetchone()
763
+ if srow is None:
764
+ raise FileNotFoundError(f"no screening session {session!r}")
765
+ ver = int(srow["schema_version"] or 1)
766
+ if ver > store.SCHEMA:
767
+ raise store.SchemaTooNew(
768
+ f"{session} was written by a newer LitLoop (schema {ver}, this copy knows {store.SCHEMA}); update LitLoop"
769
+ )
770
+ drows = conn.execute(
771
+ "SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
772
+ (session,),
773
+ ).fetchall()
774
+ return {
775
+ "schema": ver,
776
+ "id": srow["id"],
777
+ "created": srow["created"],
778
+ "updated": srow["updated"],
779
+ "question": srow["question"],
780
+ "mode": srow["mode"],
781
+ "criteria": json.loads(srow["criteria_json"] or "[]"),
782
+ "searches": json.loads(srow["searches_json"] or "[]"),
783
+ "items": [_decision_row_to_dict(r) for r in drows],
784
+ }
785
+
786
+
787
+ def _legacy_session_file(session: str, sdir: Path) -> Path | None:
788
+ if not session or session.startswith(".") or "/" in session or "\\" in session:
789
+ return None
790
+ if not _SAFE_SESSION_RE.match(session):
791
+ return None
792
+ p = sdir / f"{session}.json"
793
+ return p if p.is_file() else None
794
+
795
+
796
+ def _read_legacy_session(path: Path) -> dict[str, Any] | None:
797
+ sdata = _read_json_safe(path)
798
+ if not isinstance(sdata, dict):
799
+ return None
800
+ ver = _opt_int(sdata.get("schema")) or 1
801
+ if ver > store.SCHEMA:
802
+ raise store.SchemaTooNew(
803
+ f"{path.name} was written by a newer LitLoop (schema {ver}, this copy knows {store.SCHEMA}); update LitLoop"
804
+ )
805
+ return sdata
806
+
807
+
808
+ def _ensure_session_in_conn(
809
+ conn: sqlite3.Connection,
810
+ session: str,
811
+ *,
812
+ sdir: Path,
813
+ in_write_tx: bool,
814
+ ) -> bool:
815
+ row = conn.execute("SELECT 1 FROM sessions WHERE id = ?", (session,)).fetchone()
816
+ if row is not None:
817
+ return True
818
+ legacy = _legacy_session_file(session, sdir)
819
+ if legacy is None:
820
+ return False
821
+ sdata = _read_legacy_session(legacy)
822
+ if sdata is None:
823
+ return False
824
+ counts = {"sessions": 0, "decisions": 0}
825
+ if not in_write_tx:
826
+ conn.execute("BEGIN IMMEDIATE")
827
+ try:
828
+ _import_session_file(conn, session, sdata, counts)
829
+ if conn.in_transaction:
830
+ conn.execute("COMMIT")
831
+ except BaseException:
832
+ if conn.in_transaction:
833
+ conn.execute("ROLLBACK")
834
+ raise
835
+ else:
836
+ _import_session_file(conn, session, sdata, counts)
837
+ return True
838
+
839
+
840
+ def _sync_missing_session_jsons(conn: sqlite3.Connection, *, sdir: Path) -> None:
841
+ if not sdir.is_dir():
842
+ return
843
+ disk_files = [
844
+ p for p in sdir.glob("*.json")
845
+ if p.name.count(".") == 1 and not p.name.startswith(".")
846
+ ]
847
+ if not disk_files:
848
+ return
849
+ existing = {r[0] for r in conn.execute("SELECT id FROM sessions").fetchall()}
850
+ missing = [(p.stem, p) for p in disk_files if p.stem not in existing]
851
+ if not missing:
852
+ return
853
+ counts = {"sessions": 0, "decisions": 0}
854
+ conn.execute("BEGIN IMMEDIATE")
855
+ try:
856
+ for sid, path in missing:
857
+ sdata = _read_legacy_session(path)
858
+ if sdata is not None:
859
+ _import_session_file(conn, sid, sdata, counts)
860
+ if conn.in_transaction:
861
+ conn.execute("COMMIT")
862
+ except BaseException:
863
+ if conn.in_transaction:
864
+ conn.execute("ROLLBACK")
865
+ raise
866
+
867
+
868
+ def _upsert_decision(
869
+ conn: sqlite3.Connection,
870
+ sid: str,
871
+ item: dict[str, Any],
872
+ pkey: str | None = None,
873
+ ) -> None:
874
+ key = pkey or paper_key(item)
875
+ authors = item.get("authors") if isinstance(item.get("authors"), list) else []
876
+ extra = {k: v for k, v in item.items() if k not in _DECISION_KNOWN_KEYS}
877
+ conn.execute(
878
+ """
879
+ INSERT INTO decisions (
880
+ session_id, paper_key, n, doi, title, year, journal,
881
+ journal_rank, authors_json, cited_by_count, abstract,
882
+ abstract_zh, abstract_missing, retracted, concern,
883
+ relevance, reason, user_relevance, decision, reject_reason,
884
+ data_json
885
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
886
+ ON CONFLICT(session_id, paper_key) DO UPDATE SET
887
+ n = excluded.n,
888
+ doi = excluded.doi,
889
+ title = excluded.title,
890
+ year = excluded.year,
891
+ journal = excluded.journal,
892
+ journal_rank = excluded.journal_rank,
893
+ authors_json = excluded.authors_json,
894
+ cited_by_count = excluded.cited_by_count,
895
+ abstract = excluded.abstract,
896
+ abstract_zh = excluded.abstract_zh,
897
+ abstract_missing = excluded.abstract_missing,
898
+ retracted = excluded.retracted,
899
+ concern = excluded.concern,
900
+ relevance = excluded.relevance,
901
+ reason = excluded.reason,
902
+ user_relevance = excluded.user_relevance,
903
+ decision = excluded.decision,
904
+ reject_reason = excluded.reject_reason,
905
+ data_json = excluded.data_json
906
+ """,
907
+ (
908
+ sid,
909
+ key,
910
+ _opt_int(item.get("n")),
911
+ str(item["doi"]) if item.get("doi") else None,
912
+ str(item.get("title") or ""),
913
+ _opt_int(item.get("year")),
914
+ str(item.get("journal") or ""),
915
+ str(item.get("journal_rank") or ""),
916
+ json.dumps(authors, ensure_ascii=False),
917
+ _opt_int(item.get("cited_by_count")) or 0,
918
+ str(item.get("abstract") or ""),
919
+ str(item["abstract_zh"]) if item.get("abstract_zh") else None,
920
+ _opt_bool_int(item.get("abstract_missing")),
921
+ _opt_bool_int(item.get("retracted")),
922
+ _opt_bool_int(item.get("concern")),
923
+ str(item.get("relevance") or ""),
924
+ str(item.get("reason") or ""),
925
+ str(item["user_relevance"]) if item.get("user_relevance") is not None else None,
926
+ str(item["decision"]) if item.get("decision") is not None else None,
927
+ str(item["reject_reason"]) if item.get("reject_reason") is not None else None,
928
+ json.dumps(extra, ensure_ascii=False),
929
+ ),
930
+ )
931
+
932
+
933
+ def session_exists(session: str, *, db_file: Path | str | None = None) -> bool:
934
+ """Return True if SESSION exists in SQLite or can be imported from legacy JSON."""
935
+ sdir = _screening_dir(db_file)
936
+ with connect(db_file) as conn:
937
+ return _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=False)
938
+
939
+
940
+ def load_screening(session: str, *, db_file: Path | str | None = None) -> dict[str, Any]:
941
+ """Load one screening session record (auto-importing legacy JSON if present)."""
942
+ sdir = _screening_dir(db_file)
943
+ with connect(db_file) as conn:
944
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=False):
945
+ raise FileNotFoundError(f"no screening session {session!r}")
946
+ return _load_session_record(conn, session)
947
+
948
+
949
+ def save_screening(
950
+ question: str,
951
+ items: list[dict[str, Any]],
952
+ criteria: list[str] | str | None = None,
953
+ query: str = "",
954
+ mode: str = "quick",
955
+ session: str | None = None,
956
+ search_params: dict[str, Any] | None = None,
957
+ *,
958
+ db_file: Path | str | None = None,
959
+ ) -> dict[str, Any]:
960
+ """Create or merge a screening session in SQLite and return the full record + summary."""
961
+ sid = session or time.strftime("%Y%m%d-%H%M%S") + "-" + _slug(question or query)
962
+ sdir = _screening_dir(db_file)
963
+ now = _now_iso()
964
+ with transaction(db_file) as conn:
965
+ if session:
966
+ if not _ensure_session_in_conn(conn, sid, sdir=sdir, in_write_tx=True):
967
+ raise FileNotFoundError(f"no screening session {sid!r}")
968
+ data = _load_session_record(conn, sid)
969
+ else:
970
+ data = {
971
+ "schema": store.SCHEMA,
972
+ "id": sid,
973
+ "created": now,
974
+ "updated": now,
975
+ "question": question,
976
+ "mode": mode,
977
+ "criteria": [],
978
+ "searches": [],
979
+ "items": [],
980
+ }
981
+ if criteria:
982
+ data["criteria"] = [criteria] if isinstance(criteria, str) else list(criteria)
983
+
984
+ by_key = {paper_key(it): it for it in data["items"]}
985
+ added = 0
986
+ for raw in items:
987
+ item = {
988
+ k: raw.get(k)
989
+ for k in (
990
+ "doi",
991
+ "title",
992
+ "year",
993
+ "journal",
994
+ "journal_rank",
995
+ "authors",
996
+ "cited_by_count",
997
+ "abstract",
998
+ "abstract_missing",
999
+ "retracted",
1000
+ "concern",
1001
+ )
1002
+ if raw.get(k) not in (None, "", [])
1003
+ }
1004
+ if item.get("abstract"):
1005
+ item["abstract"] = str(item["abstract"])[:ABSTRACT_CHARS]
1006
+ if isinstance(item.get("authors"), list):
1007
+ item["authors"] = item["authors"][:3]
1008
+ if raw.get("abstract_zh"):
1009
+ item["abstract_zh"] = str(raw["abstract_zh"]).strip()[:ABSTRACT_CHARS]
1010
+ item["relevance"] = _norm_relevance(raw.get("relevance"))
1011
+ item["reason"] = str(raw.get("reason") or "").strip()
1012
+
1013
+ key = paper_key(item)
1014
+ old = by_key.get(key)
1015
+ if old:
1016
+ old.update(item)
1017
+ else:
1018
+ item["decision"] = None
1019
+ data["items"].append(item)
1020
+ by_key[key] = item
1021
+ added += 1
1022
+
1023
+ data["items"].sort(key=lambda it: (_RANK[it["relevance"]], -(it.get("cited_by_count") or 0)))
1024
+ for n, it in enumerate(data["items"], 1):
1025
+ it["n"] = n
1026
+
1027
+ if query:
1028
+ data["searches"].append(
1029
+ {
1030
+ "query": query,
1031
+ "time": now,
1032
+ "params": search_params or {},
1033
+ "results": len(items),
1034
+ "new": added,
1035
+ }
1036
+ )
1037
+ data["updated"] = now
1038
+
1039
+ conn.execute(
1040
+ """
1041
+ INSERT INTO sessions (
1042
+ id, schema_version, created, updated, question, mode,
1043
+ criteria_json, searches_json
1044
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1045
+ ON CONFLICT(id) DO UPDATE SET
1046
+ schema_version = excluded.schema_version,
1047
+ updated = excluded.updated,
1048
+ question = excluded.question,
1049
+ mode = excluded.mode,
1050
+ criteria_json = excluded.criteria_json,
1051
+ searches_json = excluded.searches_json
1052
+ """,
1053
+ (
1054
+ data["id"],
1055
+ int(data.get("schema") or store.SCHEMA),
1056
+ data["created"],
1057
+ data["updated"],
1058
+ data["question"],
1059
+ data["mode"],
1060
+ json.dumps(data["criteria"], ensure_ascii=False),
1061
+ json.dumps(data["searches"], ensure_ascii=False),
1062
+ ),
1063
+ )
1064
+ for it in data["items"]:
1065
+ _upsert_decision(conn, data["id"], it)
1066
+
1067
+ return {
1068
+ "session": data["id"],
1069
+ "added": added,
1070
+ "total": len(data["items"]),
1071
+ "by_relevance": {r: sum(1 for it in data["items"] if it["relevance"] == r) for r in RELEVANCE},
1072
+ "numbering": [_decision_brief(it) for it in data["items"]],
1073
+ "record": data,
1074
+ }
1075
+
1076
+
1077
+ def list_sessions(limit: int = 20, *, db_file: Path | str | None = None) -> list[dict[str, Any]]:
1078
+ """Return screening sessions newest-first with item and selected counts."""
1079
+ sdir = _screening_dir(db_file)
1080
+ with connect(db_file) as conn:
1081
+ _sync_missing_session_jsons(conn, sdir=sdir)
1082
+ rows = conn.execute(
1083
+ """
1084
+ SELECT s.id, s.question, s.updated,
1085
+ COUNT(d.paper_key) AS items,
1086
+ COALESCE(SUM(CASE WHEN d.decision = 'selected' THEN 1 ELSE 0 END), 0) AS selected
1087
+ FROM sessions s
1088
+ LEFT JOIN decisions d ON d.session_id = s.id
1089
+ GROUP BY s.id
1090
+ ORDER BY s.id DESC
1091
+ LIMIT ?
1092
+ """,
1093
+ (int(limit),),
1094
+ ).fetchall()
1095
+ return [
1096
+ {
1097
+ "session": r["id"],
1098
+ "question": r["question"],
1099
+ "updated": r["updated"],
1100
+ "items": int(r["items"]),
1101
+ "selected": int(r["selected"]),
1102
+ }
1103
+ for r in rows
1104
+ ]
1105
+
1106
+
1107
+ def history(limit: int = 500, *, db_file: Path | str | None = None) -> list[dict[str, Any]]:
1108
+ """Return screening session history with relevance counts and selected picks, newest-first."""
1109
+ sdir = _screening_dir(db_file)
1110
+ with connect(db_file) as conn:
1111
+ _sync_missing_session_jsons(conn, sdir=sdir)
1112
+ srows = conn.execute(
1113
+ "SELECT * FROM sessions ORDER BY id DESC LIMIT ?",
1114
+ (int(limit),),
1115
+ ).fetchall()
1116
+ out: list[dict[str, Any]] = []
1117
+ for srow in srows:
1118
+ sid = srow["id"]
1119
+ drows = conn.execute(
1120
+ "SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
1121
+ (sid,),
1122
+ ).fetchall()
1123
+ items = [_decision_row_to_dict(r) for r in drows]
1124
+ out.append(
1125
+ {
1126
+ "session": sid,
1127
+ "question": srow["question"],
1128
+ "mode": srow["mode"],
1129
+ "created": srow["created"],
1130
+ "updated": srow["updated"],
1131
+ "items": len(items),
1132
+ "by_relevance": {
1133
+ r: sum(1 for it in items if (it.get("user_relevance") or it["relevance"]) == r)
1134
+ for r in RELEVANCE
1135
+ },
1136
+ "selected": [
1137
+ {k: it.get(k) for k in ("doi", "title", "year", "journal", "journal_rank")}
1138
+ for it in items
1139
+ if it.get("decision") == "selected"
1140
+ ],
1141
+ }
1142
+ )
1143
+ return out
1144
+
1145
+
1146
+ def decide_screening(
1147
+ session: str,
1148
+ selected: list[int | str],
1149
+ rejected: list[int | str] | None = None,
1150
+ *,
1151
+ db_file: Path | str | None = None,
1152
+ ) -> dict[str, Any]:
1153
+ """Record selected/rejected picks by item number or DOI, replacing prior decisions."""
1154
+ sdir = _screening_dir(db_file)
1155
+ now = _now_iso()
1156
+ with transaction(db_file) as conn:
1157
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1158
+ raise FileNotFoundError(f"no screening session {session!r}")
1159
+ data = _load_session_record(conn, session)
1160
+ index: dict[str, dict[str, Any]] = {}
1161
+ for it in data["items"]:
1162
+ index[str(it["n"])] = it
1163
+ if it.get("doi"):
1164
+ index[str(it["doi"]).lower()] = it
1165
+ refs = [(ref, "selected") for ref in selected or []] + [
1166
+ (ref, "rejected") for ref in rejected or []
1167
+ ]
1168
+ unknown = [ref for ref, _ in refs if str(ref).strip().lower() not in index]
1169
+ if unknown:
1170
+ return {
1171
+ "session": session,
1172
+ "error": "unknown item numbers/DOIs; nothing recorded",
1173
+ "unknown": unknown,
1174
+ "numbering": [_decision_brief(it) for it in data["items"]],
1175
+ }
1176
+
1177
+ for it in data["items"]:
1178
+ it["decision"] = None
1179
+ it.pop("reject_reason", None)
1180
+ for ref, decision in refs:
1181
+ index[str(ref).strip().lower()]["decision"] = decision
1182
+
1183
+ conn.execute(
1184
+ "UPDATE decisions SET decision = NULL, reject_reason = NULL WHERE session_id = ?",
1185
+ (session,),
1186
+ )
1187
+ for ref, decision in refs:
1188
+ it = index[str(ref).strip().lower()]
1189
+ conn.execute(
1190
+ "UPDATE decisions SET decision = ? WHERE session_id = ? AND paper_key = ?",
1191
+ (decision, session, paper_key(it)),
1192
+ )
1193
+ data["updated"] = now
1194
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1195
+
1196
+ chosen = [it for it in data["items"] if it.get("decision") == "selected"]
1197
+ return {
1198
+ "session": session,
1199
+ "selected": len(chosen),
1200
+ "identifiers": [it["doi"] for it in chosen if it.get("doi")],
1201
+ "chosen": [_decision_brief(it) for it in chosen],
1202
+ "rejected": [_decision_brief(it) for it in data["items"] if it.get("decision") == "rejected"],
1203
+ "unknown": unknown,
1204
+ "record": data,
1205
+ }
1206
+
1207
+
1208
+ def translate_screening(
1209
+ session: str,
1210
+ items: list[dict[str, Any]],
1211
+ *,
1212
+ db_file: Path | str | None = None,
1213
+ ) -> dict[str, Any]:
1214
+ """Store Chinese abstracts in SQLite without altering ratings or decisions."""
1215
+ sdir = _screening_dir(db_file)
1216
+ now = _now_iso()
1217
+ with transaction(db_file) as conn:
1218
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1219
+ raise FileNotFoundError(f"no screening session {session!r}")
1220
+ data = _load_session_record(conn, session)
1221
+ by_doi = {str(it.get("doi") or "").lower(): it for it in data["items"] if it.get("doi")}
1222
+ by_n = {it.get("n"): it for it in data["items"]}
1223
+ saved, missing = 0, []
1224
+ for raw in items or []:
1225
+ target = by_doi.get(str(raw.get("doi") or "").lower()) or by_n.get(raw.get("n"))
1226
+ text = str(raw.get("abstract_zh") or "").strip()
1227
+ if not target or not text:
1228
+ missing.append(raw.get("doi") or raw.get("n"))
1229
+ continue
1230
+ zh = text[:ABSTRACT_CHARS]
1231
+ conn.execute(
1232
+ "UPDATE decisions SET abstract_zh = ? WHERE session_id = ? AND paper_key = ?",
1233
+ (zh, session, paper_key(target)),
1234
+ )
1235
+ saved += 1
1236
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1237
+ return {"session": session, "saved": saved, "not_found": missing}
1238
+
1239
+
1240
+ def _find_decision_row(
1241
+ conn: sqlite3.Connection,
1242
+ session: str,
1243
+ doi: str | None,
1244
+ title: str | None,
1245
+ ) -> sqlite3.Row | None:
1246
+ key = paper_key({"doi": doi, "title": title})
1247
+ row = conn.execute(
1248
+ "SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
1249
+ (session, key),
1250
+ ).fetchone()
1251
+ if row is not None:
1252
+ return row
1253
+ # Fallback scan in case legacy paper_key differed slightly
1254
+ for candidate in conn.execute(
1255
+ "SELECT * FROM decisions WHERE session_id = ?", (session,)
1256
+ ).fetchall():
1257
+ if paper_key({"doi": candidate["doi"], "title": candidate["title"]}) == key:
1258
+ return candidate
1259
+ return None
1260
+
1261
+
1262
+ def update_item_rating(
1263
+ session: str,
1264
+ doi: str | None,
1265
+ title: str | None,
1266
+ relevance: str | None,
1267
+ *,
1268
+ create_paper: dict[str, Any] | None = None,
1269
+ db_file: Path | str | None = None,
1270
+ ) -> dict[str, Any] | None:
1271
+ """Update or create a user rating for one paper in a session.
1272
+
1273
+ Returns ``None`` if the paper is not in the session and cannot be created.
1274
+ """
1275
+ if relevance is not None and relevance not in _RANK:
1276
+ raise ValueError(f"relevance must be one of {RELEVANCE}, got {relevance!r}")
1277
+ sdir = _screening_dir(db_file)
1278
+ now = _now_iso()
1279
+ with transaction(db_file) as conn:
1280
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1281
+ raise FileNotFoundError(f"no screening session {session!r}")
1282
+ row = _find_decision_row(conn, session, doi, title)
1283
+ if row is None:
1284
+ if relevance is None or not create_paper:
1285
+ return None
1286
+ max_n_row = conn.execute(
1287
+ "SELECT COALESCE(MAX(n), 0) FROM decisions WHERE session_id = ?",
1288
+ (session,),
1289
+ ).fetchone()
1290
+ next_n = int(max_n_row[0] or 0) + 1
1291
+ item = {
1292
+ k: create_paper.get(k)
1293
+ for k in (
1294
+ "doi",
1295
+ "title",
1296
+ "year",
1297
+ "journal",
1298
+ "journal_rank",
1299
+ "authors",
1300
+ "cited_by_count",
1301
+ "abstract",
1302
+ "abstract_missing",
1303
+ "retracted",
1304
+ "concern",
1305
+ )
1306
+ if create_paper.get(k) not in (None, "", [])
1307
+ }
1308
+ if item.get("abstract"):
1309
+ item["abstract"] = str(item["abstract"])[:ABSTRACT_CHARS]
1310
+ if isinstance(item.get("authors"), list):
1311
+ item["authors"] = item["authors"][:3]
1312
+ item["n"] = next_n
1313
+ item["relevance"] = relevance
1314
+ item["user_relevance"] = relevance
1315
+ item["reason"] = ""
1316
+ item["decision"] = None
1317
+ _upsert_decision(conn, session, item)
1318
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1319
+ created = _find_decision_row(conn, session, item.get("doi"), item.get("title"))
1320
+ edited = int(
1321
+ conn.execute(
1322
+ "SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
1323
+ (session,),
1324
+ ).fetchone()[0]
1325
+ )
1326
+ return {"item": _decision_row_to_dict(created) if created else item, "edited": edited}
1327
+
1328
+ pkey = row["paper_key"]
1329
+ if not row["reason"]:
1330
+ if relevance is None:
1331
+ conn.execute(
1332
+ "DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
1333
+ (session, pkey),
1334
+ )
1335
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1336
+ edited = int(
1337
+ conn.execute(
1338
+ "SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
1339
+ (session,),
1340
+ ).fetchone()[0]
1341
+ )
1342
+ return {"item": None, "edited": edited}
1343
+ conn.execute(
1344
+ "UPDATE decisions SET relevance = ?, user_relevance = ? WHERE session_id = ? AND paper_key = ?",
1345
+ (relevance, relevance, session, pkey),
1346
+ )
1347
+ elif relevance is None or relevance == row["relevance"]:
1348
+ conn.execute(
1349
+ "UPDATE decisions SET user_relevance = NULL WHERE session_id = ? AND paper_key = ?",
1350
+ (session, pkey),
1351
+ )
1352
+ else:
1353
+ conn.execute(
1354
+ "UPDATE decisions SET user_relevance = ? WHERE session_id = ? AND paper_key = ?",
1355
+ (relevance, session, pkey),
1356
+ )
1357
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1358
+ updated_row = conn.execute(
1359
+ "SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
1360
+ (session, pkey),
1361
+ ).fetchone()
1362
+ edited = int(
1363
+ conn.execute(
1364
+ "SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
1365
+ (session,),
1366
+ ).fetchone()[0]
1367
+ )
1368
+ return {"item": _decision_row_to_dict(updated_row) if updated_row else None, "edited": edited}
1369
+
1370
+
1371
+ def update_item_decision(
1372
+ session: str,
1373
+ doi: str | None,
1374
+ title: str | None,
1375
+ decision: str | None,
1376
+ reason: str | None = None,
1377
+ *,
1378
+ db_file: Path | str | None = None,
1379
+ ) -> dict[str, Any] | None:
1380
+ """Update decision and reject_reason for one paper in a session."""
1381
+ if decision not in (None, "selected", "rejected"):
1382
+ raise ValueError(f"invalid decision {decision!r}")
1383
+ sdir = _screening_dir(db_file)
1384
+ now = _now_iso()
1385
+ with transaction(db_file) as conn:
1386
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1387
+ raise FileNotFoundError(f"no screening session {session!r}")
1388
+ row = _find_decision_row(conn, session, doi, title)
1389
+ if row is None:
1390
+ return None
1391
+ clean_reason = " ".join((reason or "").split())[:60]
1392
+ reject_reason = clean_reason if decision == "rejected" and clean_reason else None
1393
+ conn.execute(
1394
+ "UPDATE decisions SET decision = ?, reject_reason = ? WHERE session_id = ? AND paper_key = ?",
1395
+ (decision, reject_reason, session, row["paper_key"]),
1396
+ )
1397
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1398
+ updated_row = conn.execute(
1399
+ "SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
1400
+ (session, row["paper_key"]),
1401
+ ).fetchone()
1402
+ return {"item": _decision_row_to_dict(updated_row)} if updated_row else None
1403
+
1404
+
1405
+ def set_item_abstract_zh(
1406
+ session: str,
1407
+ doi: str | None,
1408
+ title: str | None,
1409
+ abstract_zh: str,
1410
+ *,
1411
+ db_file: Path | str | None = None,
1412
+ ) -> bool:
1413
+ """Persist translated Chinese abstract for a single decision row."""
1414
+ sdir = _screening_dir(db_file)
1415
+ now = _now_iso()
1416
+ with transaction(db_file) as conn:
1417
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1418
+ raise FileNotFoundError(f"no screening session {session!r}")
1419
+ row = _find_decision_row(conn, session, doi, title)
1420
+ if row is None:
1421
+ return False
1422
+ conn.execute(
1423
+ "UPDATE decisions SET abstract_zh = ? WHERE session_id = ? AND paper_key = ?",
1424
+ (str(abstract_zh)[:ABSTRACT_CHARS], session, row["paper_key"]),
1425
+ )
1426
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1427
+ return True
1428
+
1429
+
1430
+ def batch_update_items(
1431
+ session: str,
1432
+ ops: list[Any],
1433
+ *,
1434
+ create_papers: dict[str, dict[str, Any]] | None = None,
1435
+ db_file: Path | str | None = None,
1436
+ ) -> dict[str, Any]:
1437
+ """Atomically apply batch rating/decision updates (including unrated batch items) in one transaction."""
1438
+ sdir = _screening_dir(db_file)
1439
+ now = _now_iso()
1440
+ papers_by_key = create_papers or {}
1441
+
1442
+ def _field(op: Any, name: str) -> tuple[bool, Any]:
1443
+ if isinstance(op, dict):
1444
+ return (name in op, op.get(name))
1445
+ return (name in getattr(op, "model_fields_set", set()), getattr(op, name, None))
1446
+
1447
+ with transaction(db_file) as conn:
1448
+ if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
1449
+ raise FileNotFoundError(f"no screening session {session!r}")
1450
+
1451
+ rows = conn.execute(
1452
+ "SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
1453
+ (session,),
1454
+ ).fetchall()
1455
+ by_key: dict[str, dict[str, Any]] = {}
1456
+ max_n = 0
1457
+ for r in rows:
1458
+ it = _decision_row_to_dict(r)
1459
+ by_key[paper_key(it)] = it
1460
+ if int(r["n"] or 0) > max_n:
1461
+ max_n = int(r["n"] or 0)
1462
+
1463
+ created_keys: set[str] = set()
1464
+ for op in ops:
1465
+ _, delete_if_unrated = _field(op, "delete_if_unrated")
1466
+ _, clear_user_rel = _field(op, "clear_user_relevance")
1467
+ has_rel, rel_val = _field(op, "relevance")
1468
+ if delete_if_unrated or clear_user_rel or not has_rel or rel_val is None:
1469
+ continue
1470
+ _, doi = _field(op, "doi")
1471
+ _, title = _field(op, "title")
1472
+ key = paper_key({"doi": doi, "title": title})
1473
+ if key in by_key:
1474
+ continue
1475
+ paper = papers_by_key.get(key)
1476
+ if not paper:
1477
+ continue
1478
+ max_n += 1
1479
+ new_item = {
1480
+ k: paper.get(k)
1481
+ for k in (
1482
+ "doi",
1483
+ "title",
1484
+ "year",
1485
+ "journal",
1486
+ "journal_rank",
1487
+ "authors",
1488
+ "cited_by_count",
1489
+ "abstract",
1490
+ "abstract_missing",
1491
+ "retracted",
1492
+ "concern",
1493
+ )
1494
+ if paper.get(k) not in (None, "", [])
1495
+ }
1496
+ if new_item.get("abstract"):
1497
+ new_item["abstract"] = str(new_item["abstract"])[:ABSTRACT_CHARS]
1498
+ if isinstance(new_item.get("authors"), list):
1499
+ new_item["authors"] = new_item["authors"][:3]
1500
+ new_item["n"] = max_n
1501
+ new_item["relevance"] = _norm_relevance(rel_val)
1502
+ new_item["user_relevance"] = new_item["relevance"]
1503
+ new_item["reason"] = ""
1504
+ new_item["decision"] = None
1505
+ _upsert_decision(conn, session, new_item, pkey=key)
1506
+ by_key[key] = new_item
1507
+ created_keys.add(key)
1508
+
1509
+ updated = 0
1510
+ skipped = 0
1511
+ previous: list[dict[str, Any]] = []
1512
+ for op in ops:
1513
+ _, doi = _field(op, "doi")
1514
+ _, title = _field(op, "title")
1515
+ _, delete_if_unrated = _field(op, "delete_if_unrated")
1516
+ _, clear_user_rel = _field(op, "clear_user_relevance")
1517
+ has_rel, rel_val = _field(op, "relevance")
1518
+ has_dec, dec_val = _field(op, "decision")
1519
+ has_rr, rr_val = _field(op, "reject_reason")
1520
+
1521
+ key = paper_key({"doi": doi, "title": title})
1522
+ item = by_key.get(key)
1523
+ if item is None:
1524
+ skipped += 1
1525
+ continue
1526
+
1527
+ existed = key not in created_keys
1528
+ prev = {
1529
+ "doi": item.get("doi"),
1530
+ "title": item.get("title"),
1531
+ "existed": existed,
1532
+ "relevance": item.get("relevance") if existed else None,
1533
+ "user_relevance": item.get("user_relevance") if existed else None,
1534
+ "decision": item.get("decision") if existed else None,
1535
+ "reject_reason": item.get("reject_reason") if existed else None,
1536
+ }
1537
+ created_keys.discard(key)
1538
+
1539
+ if delete_if_unrated:
1540
+ if not item.get("reason"):
1541
+ conn.execute(
1542
+ "DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
1543
+ (session, key),
1544
+ )
1545
+ by_key.pop(key, None)
1546
+ updated += 1
1547
+ previous.append(prev)
1548
+ continue
1549
+ item.pop("user_relevance", None)
1550
+ else:
1551
+ if clear_user_rel or has_rel:
1552
+ target_rel = None if clear_user_rel else rel_val
1553
+ if not item.get("reason"):
1554
+ if target_rel is None:
1555
+ conn.execute(
1556
+ "DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
1557
+ (session, key),
1558
+ )
1559
+ by_key.pop(key, None)
1560
+ updated += 1
1561
+ previous.append(prev)
1562
+ continue
1563
+ item["relevance"] = item["user_relevance"] = target_rel
1564
+ elif target_rel is None or target_rel == item["relevance"]:
1565
+ item.pop("user_relevance", None)
1566
+ else:
1567
+ item["user_relevance"] = target_rel
1568
+ if has_dec:
1569
+ item["decision"] = dec_val
1570
+ raw_reason = rr_val if has_rr else None
1571
+ reason = " ".join((raw_reason or "").split())[:60]
1572
+ if dec_val == "rejected" and reason:
1573
+ item["reject_reason"] = reason
1574
+ else:
1575
+ item.pop("reject_reason", None)
1576
+
1577
+ _upsert_decision(conn, session, item, pkey=key)
1578
+ updated += 1
1579
+ previous.append(prev)
1580
+
1581
+ conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
1582
+ edited = int(
1583
+ conn.execute(
1584
+ "SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
1585
+ (session,),
1586
+ ).fetchone()[0]
1587
+ )
1588
+ return {"updated": updated, "skipped": skipped, "edited": edited, "previous": previous}
1589
+
1590
+
1591
+ def pending_selected_by_session(*, db_file: Path | str | None = None) -> list[dict[str, Any]]:
1592
+ """Return sessions newest-first with their selected papers that carry a DOI."""
1593
+ sdir = _screening_dir(db_file)
1594
+ with connect(db_file) as conn:
1595
+ _sync_missing_session_jsons(conn, sdir=sdir)
1596
+ rows = conn.execute(
1597
+ """
1598
+ SELECT s.id AS session, s.question, d.doi, d.title, d.year, d.journal
1599
+ FROM sessions s
1600
+ JOIN decisions d ON d.session_id = s.id
1601
+ WHERE d.decision = 'selected' AND d.doi IS NOT NULL AND d.doi != ''
1602
+ ORDER BY s.id DESC, d.n ASC, d.rowid ASC
1603
+ """
1604
+ ).fetchall()
1605
+ grouped: dict[str, dict[str, Any]] = {}
1606
+ order: list[str] = []
1607
+ for r in rows:
1608
+ sid = r["session"]
1609
+ if sid not in grouped:
1610
+ grouped[sid] = {"session": sid, "question": r["question"], "papers": []}
1611
+ order.append(sid)
1612
+ grouped[sid]["papers"].append(
1613
+ {"doi": r["doi"], "title": r["title"], "year": r["year"], "journal": r["journal"]}
1614
+ )
1615
+ return [grouped[sid] for sid in order]
1616
+
1617
+
1618
+ # --------------------------------------------------------------------------- deep search pool & summary
1619
+
1620
+ _BATCH_FIELDS = (
1621
+ "doi",
1622
+ "title",
1623
+ "year",
1624
+ "journal",
1625
+ "journal_rank",
1626
+ "retracted",
1627
+ "concern",
1628
+ "authors",
1629
+ "cited_by_count",
1630
+ "abstract",
1631
+ "abstract_missing",
1632
+ )
1633
+
1634
+
1635
+ def _safe_session_file(session: str, sdir: Path, suffix: str) -> Path | None:
1636
+ if not session or session.startswith(".") or "/" in session or "\\" in session:
1637
+ return None
1638
+ if not _SAFE_SESSION_RE.match(session):
1639
+ return None
1640
+ p = sdir / f"{session}{suffix}"
1641
+ return p if p.is_file() else None
1642
+
1643
+
1644
+ def _ensure_pool_in_conn(
1645
+ conn: sqlite3.Connection,
1646
+ session: str,
1647
+ *,
1648
+ sdir: Path,
1649
+ in_write_tx: bool,
1650
+ ) -> bool:
1651
+ row = conn.execute(
1652
+ "SELECT pool_state_json FROM sessions WHERE id = ?", (session,)
1653
+ ).fetchone()
1654
+ if row is not None and row["pool_state_json"] is not None:
1655
+ return True
1656
+ has_items = conn.execute(
1657
+ "SELECT 1 FROM pool_items WHERE session_id = ? LIMIT 1", (session,)
1658
+ ).fetchone()
1659
+ if has_items is not None:
1660
+ return True
1661
+
1662
+ legacy = _safe_session_file(session, sdir, ".pool.json")
1663
+ if legacy is None:
1664
+ return False
1665
+ pdata = _read_json_safe(legacy)
1666
+ if not isinstance(pdata, dict):
1667
+ return False
1668
+
1669
+ counts = {"sessions": 0, "pool_items": 0}
1670
+ if not in_write_tx:
1671
+ conn.execute("BEGIN IMMEDIATE")
1672
+ try:
1673
+ _import_pool_file(conn, session, pdata, counts)
1674
+ if conn.in_transaction:
1675
+ conn.execute("COMMIT")
1676
+ except BaseException:
1677
+ if conn.in_transaction:
1678
+ conn.execute("ROLLBACK")
1679
+ raise
1680
+ else:
1681
+ _import_pool_file(conn, session, pdata, counts)
1682
+ return True
1683
+
1684
+
1685
+ def _ensure_summary_in_conn(
1686
+ conn: sqlite3.Connection,
1687
+ session: str,
1688
+ *,
1689
+ sdir: Path,
1690
+ in_write_tx: bool,
1691
+ ) -> str | None:
1692
+ row = conn.execute(
1693
+ "SELECT summary_json FROM sessions WHERE id = ?", (session,)
1694
+ ).fetchone()
1695
+ if row is not None and row["summary_json"] is not None:
1696
+ return str(row["summary_json"])
1697
+
1698
+ legacy = _safe_session_file(session, sdir, ".summary.json")
1699
+ if legacy is None:
1700
+ return None
1701
+ summary = _read_json_safe(legacy)
1702
+ if not isinstance(summary, dict):
1703
+ return None
1704
+
1705
+ counts = {"sessions": 0}
1706
+ if not in_write_tx:
1707
+ conn.execute("BEGIN IMMEDIATE")
1708
+ try:
1709
+ _import_summary_file(conn, session, summary, counts)
1710
+ if conn.in_transaction:
1711
+ conn.execute("COMMIT")
1712
+ except BaseException:
1713
+ if conn.in_transaction:
1714
+ conn.execute("ROLLBACK")
1715
+ raise
1716
+ else:
1717
+ _import_summary_file(conn, session, summary, counts)
1718
+ return json.dumps(summary, ensure_ascii=False)
1719
+
1720
+
1721
+ def _pool_row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
1722
+ raw_data = row["data_json"]
1723
+ if raw_data:
1724
+ try:
1725
+ rec = json.loads(raw_data)
1726
+ if isinstance(rec, dict) and rec:
1727
+ return rec
1728
+ except (ValueError, TypeError):
1729
+ pass
1730
+ rec: dict[str, Any] = {}
1731
+ if row["doi"] is not None:
1732
+ rec["doi"] = row["doi"]
1733
+ if row["openalex"] is not None:
1734
+ rec["openalex"] = row["openalex"]
1735
+ rec["title"] = row["title"]
1736
+ if row["year"] is not None:
1737
+ rec["year"] = int(row["year"])
1738
+ if row["journal"]:
1739
+ rec["journal"] = row["journal"]
1740
+ rec["cited_by_count"] = int(row["cited_by_count"] or 0)
1741
+ if row["abstract"]:
1742
+ rec["abstract"] = row["abstract"]
1743
+ if row["score"] is not None:
1744
+ rec["score"] = float(row["score"])
1745
+ return rec
1746
+
1747
+
1748
+ def has_pool(session: str, *, db_file: Path | str | None = None) -> bool:
1749
+ """Return True if a deep-search candidate pool exists in SQLite or legacy JSON."""
1750
+ sdir = _screening_dir(db_file)
1751
+ with connect(db_file) as conn:
1752
+ return _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False)
1753
+
1754
+
1755
+ def load_pool_state(session: str, *, db_file: Path | str | None = None) -> dict[str, Any]:
1756
+ """Load deep-search pool metadata and candidate items from SQLite (auto-importing legacy JSON)."""
1757
+ sdir = _screening_dir(db_file)
1758
+ with connect(db_file) as conn:
1759
+ if not _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False):
1760
+ raise FileNotFoundError(f"no deep search for session {session!r}")
1761
+ srow = conn.execute(
1762
+ "SELECT pool_state_json, question FROM sessions WHERE id = ?", (session,)
1763
+ ).fetchone()
1764
+ meta: dict[str, Any] = {}
1765
+ if srow is not None and srow["pool_state_json"]:
1766
+ parsed = json.loads(srow["pool_state_json"])
1767
+ if isinstance(parsed, dict):
1768
+ meta = parsed
1769
+ meta.setdefault("session", session)
1770
+ if "question" not in meta and srow is not None and srow["question"]:
1771
+ meta["question"] = srow["question"]
1772
+
1773
+ rows = conn.execute(
1774
+ """
1775
+ SELECT paper_key, doi, openalex, title, year, journal,
1776
+ cited_by_count, abstract, score, data_json
1777
+ FROM pool_items
1778
+ WHERE session_id = ?
1779
+ ORDER BY rowid ASC
1780
+ """,
1781
+ (session,),
1782
+ ).fetchall()
1783
+ pool = {str(r["paper_key"]): _pool_row_to_dict(r) for r in rows}
1784
+ return {**meta, "pool": pool}
1785
+
1786
+
1787
+ def save_pool_state(state: dict[str, Any], *, db_file: Path | str | None = None) -> None:
1788
+ """Persist deep-search pool metadata and candidate items in a single transaction."""
1789
+ sid = str(state["session"])
1790
+ now = _now_iso()
1791
+ pool_meta = {k: v for k, v in state.items() if k != "pool"}
1792
+ pool = state.get("pool") or {}
1793
+ pool_meta_json = json.dumps(pool_meta, ensure_ascii=False)
1794
+
1795
+ with transaction(db_file) as conn:
1796
+ conn.execute(
1797
+ """
1798
+ INSERT INTO sessions (id, question, mode, updated, pool_state_json)
1799
+ VALUES (?, ?, 'deep', ?, ?)
1800
+ ON CONFLICT(id) DO UPDATE SET
1801
+ pool_state_json = excluded.pool_state_json,
1802
+ updated = CASE WHEN sessions.updated = '' THEN excluded.updated ELSE sessions.updated END
1803
+ """,
1804
+ (sid, str(state.get("question") or ""), now, pool_meta_json),
1805
+ )
1806
+
1807
+ existing_keys = {
1808
+ r[0]
1809
+ for r in conn.execute(
1810
+ "SELECT paper_key FROM pool_items WHERE session_id = ?", (sid,)
1811
+ ).fetchall()
1812
+ }
1813
+ new_keys = {str(k) for k, v in pool.items() if isinstance(v, dict)}
1814
+ stale_keys = existing_keys - new_keys
1815
+ if stale_keys:
1816
+ conn.executemany(
1817
+ "DELETE FROM pool_items WHERE session_id = ? AND paper_key = ?",
1818
+ [(sid, k) for k in stale_keys],
1819
+ )
1820
+
1821
+ rows = []
1822
+ for pkey, rec in pool.items():
1823
+ if not isinstance(rec, dict):
1824
+ continue
1825
+ rows.append(
1826
+ (
1827
+ sid,
1828
+ str(pkey),
1829
+ str(rec["doi"]) if rec.get("doi") is not None else None,
1830
+ str(rec["openalex"]) if rec.get("openalex") is not None else None,
1831
+ str(rec.get("title") or ""),
1832
+ _opt_int(rec.get("year")),
1833
+ str(rec.get("journal") or ""),
1834
+ _opt_int(rec.get("cited_by_count")) or 0,
1835
+ str(rec.get("abstract") or ""),
1836
+ _opt_float(rec.get("score")),
1837
+ json.dumps(rec, ensure_ascii=False),
1838
+ )
1839
+ )
1840
+ if rows:
1841
+ conn.executemany(
1842
+ """
1843
+ INSERT OR REPLACE INTO pool_items (
1844
+ session_id, paper_key, doi, openalex, title, year, journal,
1845
+ cited_by_count, abstract, score, data_json
1846
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1847
+ """,
1848
+ rows,
1849
+ )
1850
+
1851
+
1852
+ def load_summary(session: str, *, db_file: Path | str | None = None) -> dict[str, Any] | None:
1853
+ """Load deep-search summary dict from SQLite (auto-importing legacy summary JSON)."""
1854
+ sdir = _screening_dir(db_file)
1855
+ with connect(db_file) as conn:
1856
+ raw = _ensure_summary_in_conn(conn, session, sdir=sdir, in_write_tx=False)
1857
+ if raw is None:
1858
+ return None
1859
+ try:
1860
+ parsed = json.loads(raw)
1861
+ return parsed if isinstance(parsed, dict) else None
1862
+ except (ValueError, TypeError):
1863
+ return None
1864
+
1865
+
1866
+ def save_summary(
1867
+ session: str,
1868
+ summary: dict[str, Any],
1869
+ *,
1870
+ db_file: Path | str | None = None,
1871
+ ) -> None:
1872
+ """Persist deep-search summary JSON into sessions.summary_json."""
1873
+ now = _now_iso()
1874
+ summary_json = json.dumps(summary, ensure_ascii=False)
1875
+ with transaction(db_file) as conn:
1876
+ conn.execute(
1877
+ """
1878
+ INSERT INTO sessions (id, question, mode, updated, summary_json)
1879
+ VALUES (?, ?, ?, ?, ?)
1880
+ ON CONFLICT(id) DO UPDATE SET
1881
+ summary_json = excluded.summary_json
1882
+ """,
1883
+ (
1884
+ session,
1885
+ str(summary.get("question") or ""),
1886
+ str(summary.get("mode") or "deep"),
1887
+ str(summary.get("updated") or now),
1888
+ summary_json,
1889
+ ),
1890
+ )
1891
+
1892
+
1893
+ def get_batch_paper(
1894
+ session: str,
1895
+ doi: str | None,
1896
+ title: str | None,
1897
+ *,
1898
+ db_file: Path | str | None = None,
1899
+ ) -> dict[str, Any] | None:
1900
+ """Return one paper from the session's latest batch without loading pool_items."""
1901
+ sdir = _screening_dir(db_file)
1902
+ with connect(db_file) as conn:
1903
+ if not _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False):
1904
+ return None
1905
+ row = conn.execute(
1906
+ "SELECT pool_state_json FROM sessions WHERE id = ?", (session,)
1907
+ ).fetchone()
1908
+ if row is None or not row["pool_state_json"]:
1909
+ return None
1910
+ try:
1911
+ meta = json.loads(row["pool_state_json"])
1912
+ except (ValueError, TypeError):
1913
+ return None
1914
+ if not isinstance(meta, dict):
1915
+ return None
1916
+ target_key = paper_key({"doi": doi, "title": title})
1917
+ for b in meta.get("last_batch") or []:
1918
+ if isinstance(b, dict) and paper_key({"doi": b.get("doi"), "title": b.get("title")}) == target_key:
1919
+ return {f: b.get(f) for f in _BATCH_FIELDS}
1920
+ return None
1921
+
1922
+