litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/db.py
ADDED
|
@@ -0,0 +1,1922 @@
|
|
|
1
|
+
"""SQLite persistence layer for LitLoop (sessions, pool items, decisions, watches).
|
|
2
|
+
|
|
3
|
+
Phase 1 introduces the schema, WAL-mode connections with explicit closing,
|
|
4
|
+
versioned migrations via ``PRAGMA user_version`` + ``schema_migrations``,
|
|
5
|
+
idempotent legacy JSON import, and watch table operations.
|
|
6
|
+
Phase 2 adds screening session & decision operations with automatic on-demand
|
|
7
|
+
import of legacy ``<session>.json`` files.
|
|
8
|
+
Phase 3 adds deep-search candidate pool & summary operations with automatic
|
|
9
|
+
on-demand import of legacy ``<session>.pool.json`` / ``<session>.summary.json`` files.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import re
|
|
16
|
+
import sqlite3
|
|
17
|
+
import time
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from datetime import datetime, timedelta
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Callable, Iterator
|
|
22
|
+
|
|
23
|
+
from . import config, store
|
|
24
|
+
from .log import get_logger
|
|
25
|
+
from .terms.text import paper_key
|
|
26
|
+
|
|
27
|
+
logger = get_logger()
|
|
28
|
+
|
|
29
|
+
DB_SCHEMA = 1
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def db_path() -> Path:
|
|
33
|
+
"""Return the SQLite database path under ``config.DATA_DIR``, creating parents."""
|
|
34
|
+
p = Path(config.DATA_DIR) / "litloop.db"
|
|
35
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
return p
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _resolve_db_file(data_dir: Path | str | None) -> Path | None:
|
|
40
|
+
if data_dir is None:
|
|
41
|
+
return None
|
|
42
|
+
return Path(data_dir) / "litloop.db"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _now_iso() -> str:
|
|
46
|
+
return time.strftime("%Y-%m-%d %H:%M:%S")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _migrate_v1(conn: sqlite3.Connection) -> None:
|
|
50
|
+
statements = [
|
|
51
|
+
"""
|
|
52
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
53
|
+
id TEXT PRIMARY KEY,
|
|
54
|
+
schema_version INTEGER NOT NULL DEFAULT 1,
|
|
55
|
+
created TEXT NOT NULL DEFAULT '',
|
|
56
|
+
updated TEXT NOT NULL DEFAULT '',
|
|
57
|
+
question TEXT NOT NULL DEFAULT '',
|
|
58
|
+
mode TEXT NOT NULL DEFAULT 'quick',
|
|
59
|
+
criteria_json TEXT NOT NULL DEFAULT '[]',
|
|
60
|
+
searches_json TEXT NOT NULL DEFAULT '[]',
|
|
61
|
+
summary_json TEXT,
|
|
62
|
+
pool_state_json TEXT
|
|
63
|
+
)
|
|
64
|
+
""",
|
|
65
|
+
"""
|
|
66
|
+
CREATE TABLE IF NOT EXISTS pool_items (
|
|
67
|
+
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
68
|
+
paper_key TEXT NOT NULL,
|
|
69
|
+
doi TEXT,
|
|
70
|
+
openalex TEXT,
|
|
71
|
+
title TEXT NOT NULL DEFAULT '',
|
|
72
|
+
year INTEGER,
|
|
73
|
+
journal TEXT NOT NULL DEFAULT '',
|
|
74
|
+
cited_by_count INTEGER NOT NULL DEFAULT 0,
|
|
75
|
+
abstract TEXT NOT NULL DEFAULT '',
|
|
76
|
+
score REAL,
|
|
77
|
+
data_json TEXT NOT NULL DEFAULT '{}',
|
|
78
|
+
PRIMARY KEY (session_id, paper_key)
|
|
79
|
+
)
|
|
80
|
+
""",
|
|
81
|
+
"CREATE INDEX IF NOT EXISTS idx_pool_items_session_score ON pool_items(session_id, score DESC)",
|
|
82
|
+
"CREATE INDEX IF NOT EXISTS idx_pool_items_doi ON pool_items(doi)",
|
|
83
|
+
"""
|
|
84
|
+
CREATE TABLE IF NOT EXISTS decisions (
|
|
85
|
+
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
86
|
+
paper_key TEXT NOT NULL,
|
|
87
|
+
n INTEGER,
|
|
88
|
+
doi TEXT,
|
|
89
|
+
title TEXT NOT NULL DEFAULT '',
|
|
90
|
+
year INTEGER,
|
|
91
|
+
journal TEXT NOT NULL DEFAULT '',
|
|
92
|
+
journal_rank TEXT NOT NULL DEFAULT '',
|
|
93
|
+
authors_json TEXT NOT NULL DEFAULT '[]',
|
|
94
|
+
cited_by_count INTEGER NOT NULL DEFAULT 0,
|
|
95
|
+
abstract TEXT NOT NULL DEFAULT '',
|
|
96
|
+
abstract_zh TEXT,
|
|
97
|
+
abstract_missing INTEGER,
|
|
98
|
+
retracted INTEGER,
|
|
99
|
+
concern INTEGER,
|
|
100
|
+
relevance TEXT NOT NULL DEFAULT '',
|
|
101
|
+
reason TEXT NOT NULL DEFAULT '',
|
|
102
|
+
user_relevance TEXT,
|
|
103
|
+
decision TEXT,
|
|
104
|
+
reject_reason TEXT,
|
|
105
|
+
data_json TEXT NOT NULL DEFAULT '{}',
|
|
106
|
+
PRIMARY KEY (session_id, paper_key)
|
|
107
|
+
)
|
|
108
|
+
""",
|
|
109
|
+
"CREATE INDEX IF NOT EXISTS idx_decisions_session_n ON decisions(session_id, n)",
|
|
110
|
+
"CREATE INDEX IF NOT EXISTS idx_decisions_doi ON decisions(doi)",
|
|
111
|
+
"""
|
|
112
|
+
CREATE TABLE IF NOT EXISTS watches (
|
|
113
|
+
id TEXT PRIMARY KEY,
|
|
114
|
+
question TEXT NOT NULL DEFAULT '',
|
|
115
|
+
query TEXT NOT NULL,
|
|
116
|
+
core_terms_json TEXT NOT NULL DEFAULT '[]',
|
|
117
|
+
session TEXT NOT NULL DEFAULT '',
|
|
118
|
+
created TEXT NOT NULL DEFAULT '',
|
|
119
|
+
checked TEXT NOT NULL DEFAULT '',
|
|
120
|
+
last_new INTEGER NOT NULL DEFAULT 0,
|
|
121
|
+
seen_json TEXT NOT NULL DEFAULT '[]'
|
|
122
|
+
)
|
|
123
|
+
""",
|
|
124
|
+
"CREATE INDEX IF NOT EXISTS idx_watches_checked ON watches(checked)",
|
|
125
|
+
]
|
|
126
|
+
for stmt in statements:
|
|
127
|
+
conn.execute(stmt)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
MIGRATIONS: list[tuple[int, str, Callable[[sqlite3.Connection], None]]] = [
|
|
131
|
+
(1, "initial schema", _migrate_v1),
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _apply_pragmas(conn: sqlite3.Connection) -> None:
|
|
136
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
137
|
+
conn.execute("PRAGMA busy_timeout = 30000")
|
|
138
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
139
|
+
conn.execute("PRAGMA synchronous = NORMAL")
|
|
140
|
+
conn.row_factory = sqlite3.Row
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _run_migrations(conn: sqlite3.Connection) -> None:
|
|
144
|
+
ver = int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
145
|
+
if ver > DB_SCHEMA:
|
|
146
|
+
conn.close()
|
|
147
|
+
raise store.SchemaTooNew(
|
|
148
|
+
f"litloop.db was written by a newer LitLoop (schema {ver}, "
|
|
149
|
+
f"this copy knows {DB_SCHEMA}); update LitLoop"
|
|
150
|
+
)
|
|
151
|
+
if ver >= DB_SCHEMA:
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
155
|
+
try:
|
|
156
|
+
ver = int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
157
|
+
if ver > DB_SCHEMA:
|
|
158
|
+
raise store.SchemaTooNew(
|
|
159
|
+
f"litloop.db was written by a newer LitLoop (schema {ver}, "
|
|
160
|
+
f"this copy knows {DB_SCHEMA}); update LitLoop"
|
|
161
|
+
)
|
|
162
|
+
conn.execute(
|
|
163
|
+
"""
|
|
164
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
165
|
+
version INTEGER PRIMARY KEY,
|
|
166
|
+
applied_at TEXT NOT NULL,
|
|
167
|
+
note TEXT NOT NULL
|
|
168
|
+
)
|
|
169
|
+
"""
|
|
170
|
+
)
|
|
171
|
+
for m_ver, m_note, m_fn in sorted(MIGRATIONS, key=lambda item: item[0]):
|
|
172
|
+
if ver < m_ver <= DB_SCHEMA:
|
|
173
|
+
m_fn(conn)
|
|
174
|
+
conn.execute(
|
|
175
|
+
"INSERT OR IGNORE INTO schema_migrations (version, applied_at, note) "
|
|
176
|
+
"VALUES (?, ?, ?)",
|
|
177
|
+
(m_ver, _now_iso(), m_note),
|
|
178
|
+
)
|
|
179
|
+
conn.execute(f"PRAGMA user_version = {int(m_ver)}")
|
|
180
|
+
ver = m_ver
|
|
181
|
+
if conn.in_transaction:
|
|
182
|
+
conn.execute("COMMIT")
|
|
183
|
+
except BaseException:
|
|
184
|
+
if conn.in_transaction:
|
|
185
|
+
try:
|
|
186
|
+
conn.execute("ROLLBACK")
|
|
187
|
+
except sqlite3.Error:
|
|
188
|
+
pass
|
|
189
|
+
conn.close()
|
|
190
|
+
raise
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _open_connection(db_file: Path | str | None = None) -> sqlite3.Connection:
|
|
194
|
+
target = Path(db_file) if db_file is not None else db_path()
|
|
195
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
196
|
+
conn = sqlite3.connect(str(target), timeout=30.0, isolation_level=None)
|
|
197
|
+
try:
|
|
198
|
+
_apply_pragmas(conn)
|
|
199
|
+
_run_migrations(conn)
|
|
200
|
+
except BaseException:
|
|
201
|
+
try:
|
|
202
|
+
conn.close()
|
|
203
|
+
except sqlite3.Error:
|
|
204
|
+
pass
|
|
205
|
+
raise
|
|
206
|
+
return conn
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@contextmanager
|
|
210
|
+
def connect(db_file: Path | str | None = None) -> Iterator[sqlite3.Connection]:
|
|
211
|
+
"""Open a WAL-mode SQLite connection and always close it on exit."""
|
|
212
|
+
conn = _open_connection(db_file)
|
|
213
|
+
try:
|
|
214
|
+
yield conn
|
|
215
|
+
finally:
|
|
216
|
+
conn.close()
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@contextmanager
|
|
220
|
+
def transaction(db_file: Path | str | None = None) -> Iterator[sqlite3.Connection]:
|
|
221
|
+
"""Open a connection wrapped in an IMMEDIATE transaction, closed on exit."""
|
|
222
|
+
with connect(db_file) as conn:
|
|
223
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
224
|
+
try:
|
|
225
|
+
yield conn
|
|
226
|
+
except BaseException:
|
|
227
|
+
if conn.in_transaction:
|
|
228
|
+
conn.execute("ROLLBACK")
|
|
229
|
+
raise
|
|
230
|
+
else:
|
|
231
|
+
if conn.in_transaction:
|
|
232
|
+
conn.execute("COMMIT")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# --------------------------------------------------------------------------- watches
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _watch_row_to_dict(row: sqlite3.Row, *, include_seen: bool) -> dict[str, Any]:
|
|
239
|
+
seen = json.loads(row["seen_json"] or "[]")
|
|
240
|
+
core_terms = json.loads(row["core_terms_json"] or "[]")
|
|
241
|
+
out: dict[str, Any] = {
|
|
242
|
+
"id": row["id"],
|
|
243
|
+
"question": row["question"],
|
|
244
|
+
"query": row["query"],
|
|
245
|
+
"core_terms": core_terms,
|
|
246
|
+
"session": row["session"],
|
|
247
|
+
"created": row["created"],
|
|
248
|
+
"checked": row["checked"],
|
|
249
|
+
"last_new": row["last_new"],
|
|
250
|
+
}
|
|
251
|
+
if include_seen:
|
|
252
|
+
out["seen"] = seen
|
|
253
|
+
else:
|
|
254
|
+
out["seen_count"] = len(seen)
|
|
255
|
+
return out
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def list_watches(db_file: Path | str | None = None) -> list[dict[str, Any]]:
|
|
259
|
+
"""Return watches newest-first without the full ``seen`` list."""
|
|
260
|
+
with connect(db_file) as conn:
|
|
261
|
+
rows = conn.execute("SELECT * FROM watches").fetchall()
|
|
262
|
+
items = [_watch_row_to_dict(r, include_seen=False) for r in rows]
|
|
263
|
+
return sorted(items, key=lambda w: str(w.get("checked") or w.get("created") or ""), reverse=True)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def get_watch(wid: str, db_file: Path | str | None = None) -> dict[str, Any] | None:
|
|
267
|
+
"""Return the full watch dict (including ``seen``), or ``None`` if absent."""
|
|
268
|
+
with connect(db_file) as conn:
|
|
269
|
+
row = conn.execute("SELECT * FROM watches WHERE id = ?", (wid,)).fetchone()
|
|
270
|
+
return _watch_row_to_dict(row, include_seen=True) if row is not None else None
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def add_watch(
|
|
274
|
+
wid: str,
|
|
275
|
+
question: str,
|
|
276
|
+
query: str,
|
|
277
|
+
core_terms: list[str] | None = None,
|
|
278
|
+
seen_keys: list[str] | None = None,
|
|
279
|
+
session: str = "",
|
|
280
|
+
created: str | None = None,
|
|
281
|
+
checked: str = "",
|
|
282
|
+
last_new: int = 0,
|
|
283
|
+
db_file: Path | str | None = None,
|
|
284
|
+
) -> dict[str, Any]:
|
|
285
|
+
"""Insert a watch row and return its summary dict."""
|
|
286
|
+
created_str = created if created is not None else time.strftime("%Y-%m-%d %H:%M")
|
|
287
|
+
core_list = list(core_terms or [])
|
|
288
|
+
seen_list = list(seen_keys or [])
|
|
289
|
+
with transaction(db_file) as conn:
|
|
290
|
+
conn.execute(
|
|
291
|
+
"""
|
|
292
|
+
INSERT INTO watches (
|
|
293
|
+
id, question, query, core_terms_json, session,
|
|
294
|
+
created, checked, last_new, seen_json
|
|
295
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
296
|
+
""",
|
|
297
|
+
(
|
|
298
|
+
wid,
|
|
299
|
+
question,
|
|
300
|
+
query,
|
|
301
|
+
json.dumps(core_list, ensure_ascii=False),
|
|
302
|
+
session,
|
|
303
|
+
created_str,
|
|
304
|
+
checked,
|
|
305
|
+
int(last_new),
|
|
306
|
+
json.dumps(seen_list, ensure_ascii=False),
|
|
307
|
+
),
|
|
308
|
+
)
|
|
309
|
+
return {
|
|
310
|
+
"id": wid,
|
|
311
|
+
"question": question,
|
|
312
|
+
"query": query,
|
|
313
|
+
"core_terms": core_list,
|
|
314
|
+
"session": session,
|
|
315
|
+
"created": created_str,
|
|
316
|
+
"checked": checked,
|
|
317
|
+
"last_new": int(last_new),
|
|
318
|
+
"seen_count": len(seen_list),
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def remove_watch(wid: str, db_file: Path | str | None = None) -> bool:
|
|
323
|
+
"""Delete a watch by id; return ``True`` if a row was removed."""
|
|
324
|
+
with transaction(db_file) as conn:
|
|
325
|
+
cur = conn.execute("DELETE FROM watches WHERE id = ?", (wid,))
|
|
326
|
+
return cur.rowcount > 0
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def update_watch_check(
|
|
330
|
+
wid: str,
|
|
331
|
+
*,
|
|
332
|
+
seen_keys: list[str],
|
|
333
|
+
checked: str,
|
|
334
|
+
last_new: int,
|
|
335
|
+
db_file: Path | str | None = None,
|
|
336
|
+
) -> bool:
|
|
337
|
+
"""Update ``seen_json``, ``checked`` and ``last_new`` after a check run."""
|
|
338
|
+
with transaction(db_file) as conn:
|
|
339
|
+
cur = conn.execute(
|
|
340
|
+
"""
|
|
341
|
+
UPDATE watches
|
|
342
|
+
SET seen_json = ?, checked = ?, last_new = ?
|
|
343
|
+
WHERE id = ?
|
|
344
|
+
""",
|
|
345
|
+
(json.dumps(list(seen_keys), ensure_ascii=False), checked, int(last_new), wid),
|
|
346
|
+
)
|
|
347
|
+
return cur.rowcount > 0
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def due_watches(
|
|
351
|
+
days: float,
|
|
352
|
+
*,
|
|
353
|
+
now: datetime | None = None,
|
|
354
|
+
db_file: Path | str | None = None,
|
|
355
|
+
) -> list[dict[str, Any]]:
|
|
356
|
+
"""Return watches never checked or checked at least ``days`` ago."""
|
|
357
|
+
ref = now or datetime.now()
|
|
358
|
+
threshold = timedelta(days=float(days))
|
|
359
|
+
with connect(db_file) as conn:
|
|
360
|
+
rows = conn.execute("SELECT * FROM watches").fetchall()
|
|
361
|
+
due: list[dict[str, Any]] = []
|
|
362
|
+
for row in rows:
|
|
363
|
+
checked = str(row["checked"] or "")
|
|
364
|
+
if checked:
|
|
365
|
+
try:
|
|
366
|
+
if ref - datetime.fromisoformat(checked) < threshold:
|
|
367
|
+
continue
|
|
368
|
+
except ValueError:
|
|
369
|
+
pass
|
|
370
|
+
due.append(_watch_row_to_dict(row, include_seen=True))
|
|
371
|
+
return due
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# --------------------------------------------------------------------------- legacy JSON import
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _read_json_safe(path: Path) -> Any:
|
|
378
|
+
try:
|
|
379
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
380
|
+
except (OSError, ValueError, TypeError) as exc:
|
|
381
|
+
logger.warning("db: skipping unreadable legacy JSON %s (%s)", path, exc)
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def import_legacy_watches(
|
|
386
|
+
data_dir: Path | str | None = None,
|
|
387
|
+
legacy_path: Path | str | None = None,
|
|
388
|
+
) -> int:
|
|
389
|
+
"""Idempotently import ``watches.json`` into SQLite without deleting the file."""
|
|
390
|
+
base = Path(data_dir) if data_dir is not None else Path(config.DATA_DIR)
|
|
391
|
+
src = Path(legacy_path) if legacy_path is not None else base / "watches.json"
|
|
392
|
+
if not src.exists():
|
|
393
|
+
return 0
|
|
394
|
+
data = _read_json_safe(src)
|
|
395
|
+
if not isinstance(data, dict):
|
|
396
|
+
return 0
|
|
397
|
+
raw_watches = data.get("watches")
|
|
398
|
+
if not isinstance(raw_watches, list):
|
|
399
|
+
return 0
|
|
400
|
+
|
|
401
|
+
db_file = _resolve_db_file(data_dir)
|
|
402
|
+
inserted = 0
|
|
403
|
+
with transaction(db_file) as conn:
|
|
404
|
+
for w in raw_watches:
|
|
405
|
+
if not isinstance(w, dict):
|
|
406
|
+
continue
|
|
407
|
+
wid = str(w.get("id") or "").strip()
|
|
408
|
+
query = str(w.get("query") or "").strip()
|
|
409
|
+
if not wid or not query:
|
|
410
|
+
continue
|
|
411
|
+
core_terms = w.get("core_terms") if isinstance(w.get("core_terms"), list) else []
|
|
412
|
+
seen = w.get("seen") if isinstance(w.get("seen"), list) else []
|
|
413
|
+
cur = conn.execute(
|
|
414
|
+
"""
|
|
415
|
+
INSERT OR IGNORE INTO watches (
|
|
416
|
+
id, question, query, core_terms_json, session,
|
|
417
|
+
created, checked, last_new, seen_json
|
|
418
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
419
|
+
""",
|
|
420
|
+
(
|
|
421
|
+
wid,
|
|
422
|
+
str(w.get("question") or ""),
|
|
423
|
+
query,
|
|
424
|
+
json.dumps(core_terms, ensure_ascii=False),
|
|
425
|
+
str(w.get("session") or ""),
|
|
426
|
+
str(w.get("created") or ""),
|
|
427
|
+
str(w.get("checked") or ""),
|
|
428
|
+
int(w.get("last_new") or 0),
|
|
429
|
+
json.dumps(seen, ensure_ascii=False),
|
|
430
|
+
),
|
|
431
|
+
)
|
|
432
|
+
if cur.rowcount > 0:
|
|
433
|
+
inserted += 1
|
|
434
|
+
return inserted
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
_DECISION_KNOWN_KEYS = frozenset(
|
|
438
|
+
{
|
|
439
|
+
"n",
|
|
440
|
+
"doi",
|
|
441
|
+
"title",
|
|
442
|
+
"year",
|
|
443
|
+
"journal",
|
|
444
|
+
"journal_rank",
|
|
445
|
+
"authors",
|
|
446
|
+
"cited_by_count",
|
|
447
|
+
"abstract",
|
|
448
|
+
"abstract_zh",
|
|
449
|
+
"abstract_missing",
|
|
450
|
+
"retracted",
|
|
451
|
+
"concern",
|
|
452
|
+
"relevance",
|
|
453
|
+
"reason",
|
|
454
|
+
"user_relevance",
|
|
455
|
+
"decision",
|
|
456
|
+
"reject_reason",
|
|
457
|
+
}
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _opt_bool_int(value: Any) -> int | None:
|
|
462
|
+
if value is None:
|
|
463
|
+
return None
|
|
464
|
+
return 1 if bool(value) else 0
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _opt_int(value: Any) -> int | None:
|
|
468
|
+
if value is None or isinstance(value, bool):
|
|
469
|
+
return None
|
|
470
|
+
try:
|
|
471
|
+
return int(value)
|
|
472
|
+
except (TypeError, ValueError):
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _opt_float(value: Any) -> float | None:
|
|
477
|
+
if value is None or isinstance(value, bool):
|
|
478
|
+
return None
|
|
479
|
+
try:
|
|
480
|
+
return float(value)
|
|
481
|
+
except (TypeError, ValueError):
|
|
482
|
+
return None
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _collect_session_ids(sdir: Path) -> list[str]:
|
|
486
|
+
sids: set[str] = set()
|
|
487
|
+
for p in sdir.iterdir():
|
|
488
|
+
if not p.is_file():
|
|
489
|
+
continue
|
|
490
|
+
name = p.name
|
|
491
|
+
if name.startswith(".") or not name.endswith(".json"):
|
|
492
|
+
continue
|
|
493
|
+
if name.endswith(".progress.json"):
|
|
494
|
+
continue
|
|
495
|
+
if name.endswith(".summary.json"):
|
|
496
|
+
sids.add(name[: -len(".summary.json")])
|
|
497
|
+
elif name.endswith(".pool.json"):
|
|
498
|
+
sids.add(name[: -len(".pool.json")])
|
|
499
|
+
else:
|
|
500
|
+
sids.add(name[: -len(".json")])
|
|
501
|
+
return sorted(s for s in sids if s)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _import_session_file(conn: sqlite3.Connection, sid: str, sdata: dict[str, Any], counts: dict[str, int]) -> None:
|
|
505
|
+
criteria = sdata.get("criteria") if isinstance(sdata.get("criteria"), list) else []
|
|
506
|
+
searches = sdata.get("searches") if isinstance(sdata.get("searches"), list) else []
|
|
507
|
+
cur = conn.execute(
|
|
508
|
+
"""
|
|
509
|
+
INSERT OR IGNORE INTO sessions (
|
|
510
|
+
id, schema_version, created, updated, question, mode,
|
|
511
|
+
criteria_json, searches_json
|
|
512
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
513
|
+
""",
|
|
514
|
+
(
|
|
515
|
+
sid,
|
|
516
|
+
_opt_int(sdata.get("schema")) or 1,
|
|
517
|
+
str(sdata.get("created") or ""),
|
|
518
|
+
str(sdata.get("updated") or ""),
|
|
519
|
+
str(sdata.get("question") or ""),
|
|
520
|
+
str(sdata.get("mode") or "quick"),
|
|
521
|
+
json.dumps(criteria, ensure_ascii=False),
|
|
522
|
+
json.dumps(searches, ensure_ascii=False),
|
|
523
|
+
),
|
|
524
|
+
)
|
|
525
|
+
if cur.rowcount > 0:
|
|
526
|
+
counts["sessions"] += 1
|
|
527
|
+
|
|
528
|
+
items = sdata.get("items")
|
|
529
|
+
if not isinstance(items, list):
|
|
530
|
+
return
|
|
531
|
+
for item in items:
|
|
532
|
+
if not isinstance(item, dict):
|
|
533
|
+
continue
|
|
534
|
+
pkey = paper_key(item)
|
|
535
|
+
authors = item.get("authors") if isinstance(item.get("authors"), list) else []
|
|
536
|
+
extra = {k: v for k, v in item.items() if k not in _DECISION_KNOWN_KEYS}
|
|
537
|
+
dcur = conn.execute(
|
|
538
|
+
"""
|
|
539
|
+
INSERT OR IGNORE INTO decisions (
|
|
540
|
+
session_id, paper_key, n, doi, title, year, journal,
|
|
541
|
+
journal_rank, authors_json, cited_by_count, abstract,
|
|
542
|
+
abstract_zh, abstract_missing, retracted, concern,
|
|
543
|
+
relevance, reason, user_relevance, decision, reject_reason,
|
|
544
|
+
data_json
|
|
545
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
546
|
+
""",
|
|
547
|
+
(
|
|
548
|
+
sid,
|
|
549
|
+
pkey,
|
|
550
|
+
_opt_int(item.get("n")),
|
|
551
|
+
str(item["doi"]) if item.get("doi") is not None else None,
|
|
552
|
+
str(item.get("title") or ""),
|
|
553
|
+
_opt_int(item.get("year")),
|
|
554
|
+
str(item.get("journal") or ""),
|
|
555
|
+
str(item.get("journal_rank") or ""),
|
|
556
|
+
json.dumps(authors, ensure_ascii=False),
|
|
557
|
+
_opt_int(item.get("cited_by_count")) or 0,
|
|
558
|
+
str(item.get("abstract") or ""),
|
|
559
|
+
str(item["abstract_zh"]) if item.get("abstract_zh") is not None else None,
|
|
560
|
+
_opt_bool_int(item.get("abstract_missing")),
|
|
561
|
+
_opt_bool_int(item.get("retracted")),
|
|
562
|
+
_opt_bool_int(item.get("concern")),
|
|
563
|
+
str(item.get("relevance") or ""),
|
|
564
|
+
str(item.get("reason") or ""),
|
|
565
|
+
str(item["user_relevance"]) if item.get("user_relevance") is not None else None,
|
|
566
|
+
str(item["decision"]) if item.get("decision") is not None else None,
|
|
567
|
+
str(item["reject_reason"]) if item.get("reject_reason") is not None else None,
|
|
568
|
+
json.dumps(extra, ensure_ascii=False),
|
|
569
|
+
),
|
|
570
|
+
)
|
|
571
|
+
if dcur.rowcount > 0:
|
|
572
|
+
counts["decisions"] += 1
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _import_summary_file(conn: sqlite3.Connection, sid: str, summary: dict[str, Any], counts: dict[str, int]) -> None:
|
|
576
|
+
cur = conn.execute(
|
|
577
|
+
"INSERT OR IGNORE INTO sessions (id, question, mode, updated) VALUES (?, ?, ?, ?)",
|
|
578
|
+
(
|
|
579
|
+
sid,
|
|
580
|
+
str(summary.get("question") or ""),
|
|
581
|
+
str(summary.get("mode") or "deep"),
|
|
582
|
+
str(summary.get("updated") or ""),
|
|
583
|
+
),
|
|
584
|
+
)
|
|
585
|
+
if cur.rowcount > 0:
|
|
586
|
+
counts["sessions"] += 1
|
|
587
|
+
conn.execute(
|
|
588
|
+
"UPDATE sessions SET summary_json = ? WHERE id = ? AND summary_json IS NULL",
|
|
589
|
+
(json.dumps(summary, ensure_ascii=False), sid),
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _import_pool_file(conn: sqlite3.Connection, sid: str, pdata: dict[str, Any], counts: dict[str, int]) -> None:
|
|
594
|
+
cur = conn.execute(
|
|
595
|
+
"INSERT OR IGNORE INTO sessions (id, question, mode) VALUES (?, ?, ?)",
|
|
596
|
+
(sid, str(pdata.get("question") or ""), "deep"),
|
|
597
|
+
)
|
|
598
|
+
if cur.rowcount > 0:
|
|
599
|
+
counts["sessions"] += 1
|
|
600
|
+
|
|
601
|
+
pool_meta = {k: v for k, v in pdata.items() if k != "pool"}
|
|
602
|
+
conn.execute(
|
|
603
|
+
"UPDATE sessions SET pool_state_json = ? WHERE id = ? AND pool_state_json IS NULL",
|
|
604
|
+
(json.dumps(pool_meta, ensure_ascii=False), sid),
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
pool = pdata.get("pool")
|
|
608
|
+
if not isinstance(pool, dict):
|
|
609
|
+
return
|
|
610
|
+
for pkey, rec in pool.items():
|
|
611
|
+
if not isinstance(rec, dict):
|
|
612
|
+
continue
|
|
613
|
+
pcur = conn.execute(
|
|
614
|
+
"""
|
|
615
|
+
INSERT OR IGNORE INTO pool_items (
|
|
616
|
+
session_id, paper_key, doi, openalex, title, year, journal,
|
|
617
|
+
cited_by_count, abstract, score, data_json
|
|
618
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
619
|
+
""",
|
|
620
|
+
(
|
|
621
|
+
sid,
|
|
622
|
+
str(pkey),
|
|
623
|
+
str(rec["doi"]) if rec.get("doi") is not None else None,
|
|
624
|
+
str(rec["openalex"]) if rec.get("openalex") is not None else None,
|
|
625
|
+
str(rec.get("title") or ""),
|
|
626
|
+
_opt_int(rec.get("year")),
|
|
627
|
+
str(rec.get("journal") or ""),
|
|
628
|
+
_opt_int(rec.get("cited_by_count")) or 0,
|
|
629
|
+
str(rec.get("abstract") or ""),
|
|
630
|
+
_opt_float(rec.get("score")),
|
|
631
|
+
json.dumps(rec, ensure_ascii=False),
|
|
632
|
+
),
|
|
633
|
+
)
|
|
634
|
+
if pcur.rowcount > 0:
|
|
635
|
+
counts["pool_items"] += 1
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def import_legacy_data(data_dir: Path | str | None = None) -> dict[str, int]:
|
|
639
|
+
"""Idempotently import legacy JSON files (watches + screening sessions/summaries/pools).
|
|
640
|
+
|
|
641
|
+
Never deletes any source JSON file and skips malformed JSON without crashing.
|
|
642
|
+
"""
|
|
643
|
+
base = Path(data_dir) if data_dir is not None else Path(config.DATA_DIR)
|
|
644
|
+
counts: dict[str, int] = {
|
|
645
|
+
"sessions": 0,
|
|
646
|
+
"decisions": 0,
|
|
647
|
+
"pool_items": 0,
|
|
648
|
+
"watches": import_legacy_watches(data_dir=data_dir),
|
|
649
|
+
}
|
|
650
|
+
sdir = base / "screening"
|
|
651
|
+
if not sdir.is_dir():
|
|
652
|
+
return counts
|
|
653
|
+
|
|
654
|
+
db_file = _resolve_db_file(data_dir)
|
|
655
|
+
sids = _collect_session_ids(sdir)
|
|
656
|
+
with transaction(db_file) as conn:
|
|
657
|
+
for sid in sids:
|
|
658
|
+
main_file = sdir / f"{sid}.json"
|
|
659
|
+
if main_file.is_file():
|
|
660
|
+
sdata = _read_json_safe(main_file)
|
|
661
|
+
if isinstance(sdata, dict):
|
|
662
|
+
_import_session_file(conn, sid, sdata, counts)
|
|
663
|
+
|
|
664
|
+
summary_file = sdir / f"{sid}.summary.json"
|
|
665
|
+
if summary_file.is_file():
|
|
666
|
+
summary = _read_json_safe(summary_file)
|
|
667
|
+
if isinstance(summary, dict):
|
|
668
|
+
_import_summary_file(conn, sid, summary, counts)
|
|
669
|
+
|
|
670
|
+
pool_file = sdir / f"{sid}.pool.json"
|
|
671
|
+
if pool_file.is_file():
|
|
672
|
+
pdata = _read_json_safe(pool_file)
|
|
673
|
+
if isinstance(pdata, dict):
|
|
674
|
+
_import_pool_file(conn, sid, pdata, counts)
|
|
675
|
+
|
|
676
|
+
return counts
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
# --------------------------------------------------------------------------- screening sessions & decisions
|
|
680
|
+
|
|
681
|
+
RELEVANCE = ("高", "中", "低", "不相关")
|
|
682
|
+
ABSTRACT_CHARS = 1200
|
|
683
|
+
_RANK = {r: i for i, r in enumerate(RELEVANCE)}
|
|
684
|
+
_EN_RELEVANCE = {"high": "高", "medium": "中", "low": "低", "irrelevant": "不相关", "none": "不相关"}
|
|
685
|
+
_SAFE_SESSION_RE = re.compile(r"^[\w\-. \u4e00-\u9fff]+$")
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _norm_relevance(value: Any) -> str:
|
|
689
|
+
v = str(value or "").strip()
|
|
690
|
+
v = _EN_RELEVANCE.get(v.lower(), v)
|
|
691
|
+
if v not in _RANK:
|
|
692
|
+
raise ValueError(f"relevance must be one of {RELEVANCE}, got {value!r}")
|
|
693
|
+
return v
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _slug(text: str) -> str:
|
|
697
|
+
s = re.sub(r"[^\w\u4e00-\u9fff]+", "-", text.strip().lower()).strip("-")
|
|
698
|
+
return s[:40] or "search"
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _screening_dir(db_file: Path | str | None = None) -> Path:
|
|
702
|
+
if db_file is not None:
|
|
703
|
+
return Path(db_file).parent / "screening"
|
|
704
|
+
from .search.screening import screening_dir
|
|
705
|
+
return screening_dir()
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _decision_brief(it: dict[str, Any]) -> dict[str, Any]:
|
|
709
|
+
return {
|
|
710
|
+
"n": it["n"],
|
|
711
|
+
"relevance": it["relevance"],
|
|
712
|
+
"title": str(it.get("title") or "")[:90],
|
|
713
|
+
"doi": it.get("doi"),
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def _decision_row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
|
|
718
|
+
item: dict[str, Any] = {}
|
|
719
|
+
if row["n"] is not None:
|
|
720
|
+
item["n"] = int(row["n"])
|
|
721
|
+
if row["doi"]:
|
|
722
|
+
item["doi"] = row["doi"]
|
|
723
|
+
if row["title"]:
|
|
724
|
+
item["title"] = row["title"]
|
|
725
|
+
if row["year"] is not None:
|
|
726
|
+
item["year"] = int(row["year"])
|
|
727
|
+
if row["journal"]:
|
|
728
|
+
item["journal"] = row["journal"]
|
|
729
|
+
if row["journal_rank"]:
|
|
730
|
+
item["journal_rank"] = row["journal_rank"]
|
|
731
|
+
authors = json.loads(row["authors_json"] or "[]")
|
|
732
|
+
if authors:
|
|
733
|
+
item["authors"] = authors
|
|
734
|
+
if row["cited_by_count"] is not None:
|
|
735
|
+
item["cited_by_count"] = int(row["cited_by_count"])
|
|
736
|
+
if row["abstract"]:
|
|
737
|
+
item["abstract"] = row["abstract"]
|
|
738
|
+
if row["abstract_zh"]:
|
|
739
|
+
item["abstract_zh"] = row["abstract_zh"]
|
|
740
|
+
if row["abstract_missing"] is not None:
|
|
741
|
+
item["abstract_missing"] = bool(row["abstract_missing"])
|
|
742
|
+
if row["retracted"] is not None:
|
|
743
|
+
item["retracted"] = bool(row["retracted"])
|
|
744
|
+
if row["concern"] is not None:
|
|
745
|
+
item["concern"] = bool(row["concern"])
|
|
746
|
+
item["relevance"] = row["relevance"]
|
|
747
|
+
item["reason"] = row["reason"]
|
|
748
|
+
if row["user_relevance"] is not None:
|
|
749
|
+
item["user_relevance"] = row["user_relevance"]
|
|
750
|
+
item["decision"] = row["decision"]
|
|
751
|
+
if row["reject_reason"] is not None:
|
|
752
|
+
item["reject_reason"] = row["reject_reason"]
|
|
753
|
+
extra = json.loads(row["data_json"] or "{}")
|
|
754
|
+
if isinstance(extra, dict) and extra:
|
|
755
|
+
for k, v in extra.items():
|
|
756
|
+
if k not in item:
|
|
757
|
+
item[k] = v
|
|
758
|
+
return item
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def _load_session_record(conn: sqlite3.Connection, session: str) -> dict[str, Any]:
|
|
762
|
+
srow = conn.execute("SELECT * FROM sessions WHERE id = ?", (session,)).fetchone()
|
|
763
|
+
if srow is None:
|
|
764
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
765
|
+
ver = int(srow["schema_version"] or 1)
|
|
766
|
+
if ver > store.SCHEMA:
|
|
767
|
+
raise store.SchemaTooNew(
|
|
768
|
+
f"{session} was written by a newer LitLoop (schema {ver}, this copy knows {store.SCHEMA}); update LitLoop"
|
|
769
|
+
)
|
|
770
|
+
drows = conn.execute(
|
|
771
|
+
"SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
|
|
772
|
+
(session,),
|
|
773
|
+
).fetchall()
|
|
774
|
+
return {
|
|
775
|
+
"schema": ver,
|
|
776
|
+
"id": srow["id"],
|
|
777
|
+
"created": srow["created"],
|
|
778
|
+
"updated": srow["updated"],
|
|
779
|
+
"question": srow["question"],
|
|
780
|
+
"mode": srow["mode"],
|
|
781
|
+
"criteria": json.loads(srow["criteria_json"] or "[]"),
|
|
782
|
+
"searches": json.loads(srow["searches_json"] or "[]"),
|
|
783
|
+
"items": [_decision_row_to_dict(r) for r in drows],
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def _legacy_session_file(session: str, sdir: Path) -> Path | None:
|
|
788
|
+
if not session or session.startswith(".") or "/" in session or "\\" in session:
|
|
789
|
+
return None
|
|
790
|
+
if not _SAFE_SESSION_RE.match(session):
|
|
791
|
+
return None
|
|
792
|
+
p = sdir / f"{session}.json"
|
|
793
|
+
return p if p.is_file() else None
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _read_legacy_session(path: Path) -> dict[str, Any] | None:
|
|
797
|
+
sdata = _read_json_safe(path)
|
|
798
|
+
if not isinstance(sdata, dict):
|
|
799
|
+
return None
|
|
800
|
+
ver = _opt_int(sdata.get("schema")) or 1
|
|
801
|
+
if ver > store.SCHEMA:
|
|
802
|
+
raise store.SchemaTooNew(
|
|
803
|
+
f"{path.name} was written by a newer LitLoop (schema {ver}, this copy knows {store.SCHEMA}); update LitLoop"
|
|
804
|
+
)
|
|
805
|
+
return sdata
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _ensure_session_in_conn(
|
|
809
|
+
conn: sqlite3.Connection,
|
|
810
|
+
session: str,
|
|
811
|
+
*,
|
|
812
|
+
sdir: Path,
|
|
813
|
+
in_write_tx: bool,
|
|
814
|
+
) -> bool:
|
|
815
|
+
row = conn.execute("SELECT 1 FROM sessions WHERE id = ?", (session,)).fetchone()
|
|
816
|
+
if row is not None:
|
|
817
|
+
return True
|
|
818
|
+
legacy = _legacy_session_file(session, sdir)
|
|
819
|
+
if legacy is None:
|
|
820
|
+
return False
|
|
821
|
+
sdata = _read_legacy_session(legacy)
|
|
822
|
+
if sdata is None:
|
|
823
|
+
return False
|
|
824
|
+
counts = {"sessions": 0, "decisions": 0}
|
|
825
|
+
if not in_write_tx:
|
|
826
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
827
|
+
try:
|
|
828
|
+
_import_session_file(conn, session, sdata, counts)
|
|
829
|
+
if conn.in_transaction:
|
|
830
|
+
conn.execute("COMMIT")
|
|
831
|
+
except BaseException:
|
|
832
|
+
if conn.in_transaction:
|
|
833
|
+
conn.execute("ROLLBACK")
|
|
834
|
+
raise
|
|
835
|
+
else:
|
|
836
|
+
_import_session_file(conn, session, sdata, counts)
|
|
837
|
+
return True
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _sync_missing_session_jsons(conn: sqlite3.Connection, *, sdir: Path) -> None:
|
|
841
|
+
if not sdir.is_dir():
|
|
842
|
+
return
|
|
843
|
+
disk_files = [
|
|
844
|
+
p for p in sdir.glob("*.json")
|
|
845
|
+
if p.name.count(".") == 1 and not p.name.startswith(".")
|
|
846
|
+
]
|
|
847
|
+
if not disk_files:
|
|
848
|
+
return
|
|
849
|
+
existing = {r[0] for r in conn.execute("SELECT id FROM sessions").fetchall()}
|
|
850
|
+
missing = [(p.stem, p) for p in disk_files if p.stem not in existing]
|
|
851
|
+
if not missing:
|
|
852
|
+
return
|
|
853
|
+
counts = {"sessions": 0, "decisions": 0}
|
|
854
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
855
|
+
try:
|
|
856
|
+
for sid, path in missing:
|
|
857
|
+
sdata = _read_legacy_session(path)
|
|
858
|
+
if sdata is not None:
|
|
859
|
+
_import_session_file(conn, sid, sdata, counts)
|
|
860
|
+
if conn.in_transaction:
|
|
861
|
+
conn.execute("COMMIT")
|
|
862
|
+
except BaseException:
|
|
863
|
+
if conn.in_transaction:
|
|
864
|
+
conn.execute("ROLLBACK")
|
|
865
|
+
raise
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def _upsert_decision(
|
|
869
|
+
conn: sqlite3.Connection,
|
|
870
|
+
sid: str,
|
|
871
|
+
item: dict[str, Any],
|
|
872
|
+
pkey: str | None = None,
|
|
873
|
+
) -> None:
|
|
874
|
+
key = pkey or paper_key(item)
|
|
875
|
+
authors = item.get("authors") if isinstance(item.get("authors"), list) else []
|
|
876
|
+
extra = {k: v for k, v in item.items() if k not in _DECISION_KNOWN_KEYS}
|
|
877
|
+
conn.execute(
|
|
878
|
+
"""
|
|
879
|
+
INSERT INTO decisions (
|
|
880
|
+
session_id, paper_key, n, doi, title, year, journal,
|
|
881
|
+
journal_rank, authors_json, cited_by_count, abstract,
|
|
882
|
+
abstract_zh, abstract_missing, retracted, concern,
|
|
883
|
+
relevance, reason, user_relevance, decision, reject_reason,
|
|
884
|
+
data_json
|
|
885
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
886
|
+
ON CONFLICT(session_id, paper_key) DO UPDATE SET
|
|
887
|
+
n = excluded.n,
|
|
888
|
+
doi = excluded.doi,
|
|
889
|
+
title = excluded.title,
|
|
890
|
+
year = excluded.year,
|
|
891
|
+
journal = excluded.journal,
|
|
892
|
+
journal_rank = excluded.journal_rank,
|
|
893
|
+
authors_json = excluded.authors_json,
|
|
894
|
+
cited_by_count = excluded.cited_by_count,
|
|
895
|
+
abstract = excluded.abstract,
|
|
896
|
+
abstract_zh = excluded.abstract_zh,
|
|
897
|
+
abstract_missing = excluded.abstract_missing,
|
|
898
|
+
retracted = excluded.retracted,
|
|
899
|
+
concern = excluded.concern,
|
|
900
|
+
relevance = excluded.relevance,
|
|
901
|
+
reason = excluded.reason,
|
|
902
|
+
user_relevance = excluded.user_relevance,
|
|
903
|
+
decision = excluded.decision,
|
|
904
|
+
reject_reason = excluded.reject_reason,
|
|
905
|
+
data_json = excluded.data_json
|
|
906
|
+
""",
|
|
907
|
+
(
|
|
908
|
+
sid,
|
|
909
|
+
key,
|
|
910
|
+
_opt_int(item.get("n")),
|
|
911
|
+
str(item["doi"]) if item.get("doi") else None,
|
|
912
|
+
str(item.get("title") or ""),
|
|
913
|
+
_opt_int(item.get("year")),
|
|
914
|
+
str(item.get("journal") or ""),
|
|
915
|
+
str(item.get("journal_rank") or ""),
|
|
916
|
+
json.dumps(authors, ensure_ascii=False),
|
|
917
|
+
_opt_int(item.get("cited_by_count")) or 0,
|
|
918
|
+
str(item.get("abstract") or ""),
|
|
919
|
+
str(item["abstract_zh"]) if item.get("abstract_zh") else None,
|
|
920
|
+
_opt_bool_int(item.get("abstract_missing")),
|
|
921
|
+
_opt_bool_int(item.get("retracted")),
|
|
922
|
+
_opt_bool_int(item.get("concern")),
|
|
923
|
+
str(item.get("relevance") or ""),
|
|
924
|
+
str(item.get("reason") or ""),
|
|
925
|
+
str(item["user_relevance"]) if item.get("user_relevance") is not None else None,
|
|
926
|
+
str(item["decision"]) if item.get("decision") is not None else None,
|
|
927
|
+
str(item["reject_reason"]) if item.get("reject_reason") is not None else None,
|
|
928
|
+
json.dumps(extra, ensure_ascii=False),
|
|
929
|
+
),
|
|
930
|
+
)
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def session_exists(session: str, *, db_file: Path | str | None = None) -> bool:
|
|
934
|
+
"""Return True if SESSION exists in SQLite or can be imported from legacy JSON."""
|
|
935
|
+
sdir = _screening_dir(db_file)
|
|
936
|
+
with connect(db_file) as conn:
|
|
937
|
+
return _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=False)
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
def load_screening(session: str, *, db_file: Path | str | None = None) -> dict[str, Any]:
|
|
941
|
+
"""Load one screening session record (auto-importing legacy JSON if present)."""
|
|
942
|
+
sdir = _screening_dir(db_file)
|
|
943
|
+
with connect(db_file) as conn:
|
|
944
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=False):
|
|
945
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
946
|
+
return _load_session_record(conn, session)
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def save_screening(
|
|
950
|
+
question: str,
|
|
951
|
+
items: list[dict[str, Any]],
|
|
952
|
+
criteria: list[str] | str | None = None,
|
|
953
|
+
query: str = "",
|
|
954
|
+
mode: str = "quick",
|
|
955
|
+
session: str | None = None,
|
|
956
|
+
search_params: dict[str, Any] | None = None,
|
|
957
|
+
*,
|
|
958
|
+
db_file: Path | str | None = None,
|
|
959
|
+
) -> dict[str, Any]:
|
|
960
|
+
"""Create or merge a screening session in SQLite and return the full record + summary."""
|
|
961
|
+
sid = session or time.strftime("%Y%m%d-%H%M%S") + "-" + _slug(question or query)
|
|
962
|
+
sdir = _screening_dir(db_file)
|
|
963
|
+
now = _now_iso()
|
|
964
|
+
with transaction(db_file) as conn:
|
|
965
|
+
if session:
|
|
966
|
+
if not _ensure_session_in_conn(conn, sid, sdir=sdir, in_write_tx=True):
|
|
967
|
+
raise FileNotFoundError(f"no screening session {sid!r}")
|
|
968
|
+
data = _load_session_record(conn, sid)
|
|
969
|
+
else:
|
|
970
|
+
data = {
|
|
971
|
+
"schema": store.SCHEMA,
|
|
972
|
+
"id": sid,
|
|
973
|
+
"created": now,
|
|
974
|
+
"updated": now,
|
|
975
|
+
"question": question,
|
|
976
|
+
"mode": mode,
|
|
977
|
+
"criteria": [],
|
|
978
|
+
"searches": [],
|
|
979
|
+
"items": [],
|
|
980
|
+
}
|
|
981
|
+
if criteria:
|
|
982
|
+
data["criteria"] = [criteria] if isinstance(criteria, str) else list(criteria)
|
|
983
|
+
|
|
984
|
+
by_key = {paper_key(it): it for it in data["items"]}
|
|
985
|
+
added = 0
|
|
986
|
+
for raw in items:
|
|
987
|
+
item = {
|
|
988
|
+
k: raw.get(k)
|
|
989
|
+
for k in (
|
|
990
|
+
"doi",
|
|
991
|
+
"title",
|
|
992
|
+
"year",
|
|
993
|
+
"journal",
|
|
994
|
+
"journal_rank",
|
|
995
|
+
"authors",
|
|
996
|
+
"cited_by_count",
|
|
997
|
+
"abstract",
|
|
998
|
+
"abstract_missing",
|
|
999
|
+
"retracted",
|
|
1000
|
+
"concern",
|
|
1001
|
+
)
|
|
1002
|
+
if raw.get(k) not in (None, "", [])
|
|
1003
|
+
}
|
|
1004
|
+
if item.get("abstract"):
|
|
1005
|
+
item["abstract"] = str(item["abstract"])[:ABSTRACT_CHARS]
|
|
1006
|
+
if isinstance(item.get("authors"), list):
|
|
1007
|
+
item["authors"] = item["authors"][:3]
|
|
1008
|
+
if raw.get("abstract_zh"):
|
|
1009
|
+
item["abstract_zh"] = str(raw["abstract_zh"]).strip()[:ABSTRACT_CHARS]
|
|
1010
|
+
item["relevance"] = _norm_relevance(raw.get("relevance"))
|
|
1011
|
+
item["reason"] = str(raw.get("reason") or "").strip()
|
|
1012
|
+
|
|
1013
|
+
key = paper_key(item)
|
|
1014
|
+
old = by_key.get(key)
|
|
1015
|
+
if old:
|
|
1016
|
+
old.update(item)
|
|
1017
|
+
else:
|
|
1018
|
+
item["decision"] = None
|
|
1019
|
+
data["items"].append(item)
|
|
1020
|
+
by_key[key] = item
|
|
1021
|
+
added += 1
|
|
1022
|
+
|
|
1023
|
+
data["items"].sort(key=lambda it: (_RANK[it["relevance"]], -(it.get("cited_by_count") or 0)))
|
|
1024
|
+
for n, it in enumerate(data["items"], 1):
|
|
1025
|
+
it["n"] = n
|
|
1026
|
+
|
|
1027
|
+
if query:
|
|
1028
|
+
data["searches"].append(
|
|
1029
|
+
{
|
|
1030
|
+
"query": query,
|
|
1031
|
+
"time": now,
|
|
1032
|
+
"params": search_params or {},
|
|
1033
|
+
"results": len(items),
|
|
1034
|
+
"new": added,
|
|
1035
|
+
}
|
|
1036
|
+
)
|
|
1037
|
+
data["updated"] = now
|
|
1038
|
+
|
|
1039
|
+
conn.execute(
|
|
1040
|
+
"""
|
|
1041
|
+
INSERT INTO sessions (
|
|
1042
|
+
id, schema_version, created, updated, question, mode,
|
|
1043
|
+
criteria_json, searches_json
|
|
1044
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1045
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1046
|
+
schema_version = excluded.schema_version,
|
|
1047
|
+
updated = excluded.updated,
|
|
1048
|
+
question = excluded.question,
|
|
1049
|
+
mode = excluded.mode,
|
|
1050
|
+
criteria_json = excluded.criteria_json,
|
|
1051
|
+
searches_json = excluded.searches_json
|
|
1052
|
+
""",
|
|
1053
|
+
(
|
|
1054
|
+
data["id"],
|
|
1055
|
+
int(data.get("schema") or store.SCHEMA),
|
|
1056
|
+
data["created"],
|
|
1057
|
+
data["updated"],
|
|
1058
|
+
data["question"],
|
|
1059
|
+
data["mode"],
|
|
1060
|
+
json.dumps(data["criteria"], ensure_ascii=False),
|
|
1061
|
+
json.dumps(data["searches"], ensure_ascii=False),
|
|
1062
|
+
),
|
|
1063
|
+
)
|
|
1064
|
+
for it in data["items"]:
|
|
1065
|
+
_upsert_decision(conn, data["id"], it)
|
|
1066
|
+
|
|
1067
|
+
return {
|
|
1068
|
+
"session": data["id"],
|
|
1069
|
+
"added": added,
|
|
1070
|
+
"total": len(data["items"]),
|
|
1071
|
+
"by_relevance": {r: sum(1 for it in data["items"] if it["relevance"] == r) for r in RELEVANCE},
|
|
1072
|
+
"numbering": [_decision_brief(it) for it in data["items"]],
|
|
1073
|
+
"record": data,
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def list_sessions(limit: int = 20, *, db_file: Path | str | None = None) -> list[dict[str, Any]]:
|
|
1078
|
+
"""Return screening sessions newest-first with item and selected counts."""
|
|
1079
|
+
sdir = _screening_dir(db_file)
|
|
1080
|
+
with connect(db_file) as conn:
|
|
1081
|
+
_sync_missing_session_jsons(conn, sdir=sdir)
|
|
1082
|
+
rows = conn.execute(
|
|
1083
|
+
"""
|
|
1084
|
+
SELECT s.id, s.question, s.updated,
|
|
1085
|
+
COUNT(d.paper_key) AS items,
|
|
1086
|
+
COALESCE(SUM(CASE WHEN d.decision = 'selected' THEN 1 ELSE 0 END), 0) AS selected
|
|
1087
|
+
FROM sessions s
|
|
1088
|
+
LEFT JOIN decisions d ON d.session_id = s.id
|
|
1089
|
+
GROUP BY s.id
|
|
1090
|
+
ORDER BY s.id DESC
|
|
1091
|
+
LIMIT ?
|
|
1092
|
+
""",
|
|
1093
|
+
(int(limit),),
|
|
1094
|
+
).fetchall()
|
|
1095
|
+
return [
|
|
1096
|
+
{
|
|
1097
|
+
"session": r["id"],
|
|
1098
|
+
"question": r["question"],
|
|
1099
|
+
"updated": r["updated"],
|
|
1100
|
+
"items": int(r["items"]),
|
|
1101
|
+
"selected": int(r["selected"]),
|
|
1102
|
+
}
|
|
1103
|
+
for r in rows
|
|
1104
|
+
]
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def history(limit: int = 500, *, db_file: Path | str | None = None) -> list[dict[str, Any]]:
|
|
1108
|
+
"""Return screening session history with relevance counts and selected picks, newest-first."""
|
|
1109
|
+
sdir = _screening_dir(db_file)
|
|
1110
|
+
with connect(db_file) as conn:
|
|
1111
|
+
_sync_missing_session_jsons(conn, sdir=sdir)
|
|
1112
|
+
srows = conn.execute(
|
|
1113
|
+
"SELECT * FROM sessions ORDER BY id DESC LIMIT ?",
|
|
1114
|
+
(int(limit),),
|
|
1115
|
+
).fetchall()
|
|
1116
|
+
out: list[dict[str, Any]] = []
|
|
1117
|
+
for srow in srows:
|
|
1118
|
+
sid = srow["id"]
|
|
1119
|
+
drows = conn.execute(
|
|
1120
|
+
"SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
|
|
1121
|
+
(sid,),
|
|
1122
|
+
).fetchall()
|
|
1123
|
+
items = [_decision_row_to_dict(r) for r in drows]
|
|
1124
|
+
out.append(
|
|
1125
|
+
{
|
|
1126
|
+
"session": sid,
|
|
1127
|
+
"question": srow["question"],
|
|
1128
|
+
"mode": srow["mode"],
|
|
1129
|
+
"created": srow["created"],
|
|
1130
|
+
"updated": srow["updated"],
|
|
1131
|
+
"items": len(items),
|
|
1132
|
+
"by_relevance": {
|
|
1133
|
+
r: sum(1 for it in items if (it.get("user_relevance") or it["relevance"]) == r)
|
|
1134
|
+
for r in RELEVANCE
|
|
1135
|
+
},
|
|
1136
|
+
"selected": [
|
|
1137
|
+
{k: it.get(k) for k in ("doi", "title", "year", "journal", "journal_rank")}
|
|
1138
|
+
for it in items
|
|
1139
|
+
if it.get("decision") == "selected"
|
|
1140
|
+
],
|
|
1141
|
+
}
|
|
1142
|
+
)
|
|
1143
|
+
return out
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def decide_screening(
|
|
1147
|
+
session: str,
|
|
1148
|
+
selected: list[int | str],
|
|
1149
|
+
rejected: list[int | str] | None = None,
|
|
1150
|
+
*,
|
|
1151
|
+
db_file: Path | str | None = None,
|
|
1152
|
+
) -> dict[str, Any]:
|
|
1153
|
+
"""Record selected/rejected picks by item number or DOI, replacing prior decisions."""
|
|
1154
|
+
sdir = _screening_dir(db_file)
|
|
1155
|
+
now = _now_iso()
|
|
1156
|
+
with transaction(db_file) as conn:
|
|
1157
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1158
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1159
|
+
data = _load_session_record(conn, session)
|
|
1160
|
+
index: dict[str, dict[str, Any]] = {}
|
|
1161
|
+
for it in data["items"]:
|
|
1162
|
+
index[str(it["n"])] = it
|
|
1163
|
+
if it.get("doi"):
|
|
1164
|
+
index[str(it["doi"]).lower()] = it
|
|
1165
|
+
refs = [(ref, "selected") for ref in selected or []] + [
|
|
1166
|
+
(ref, "rejected") for ref in rejected or []
|
|
1167
|
+
]
|
|
1168
|
+
unknown = [ref for ref, _ in refs if str(ref).strip().lower() not in index]
|
|
1169
|
+
if unknown:
|
|
1170
|
+
return {
|
|
1171
|
+
"session": session,
|
|
1172
|
+
"error": "unknown item numbers/DOIs; nothing recorded",
|
|
1173
|
+
"unknown": unknown,
|
|
1174
|
+
"numbering": [_decision_brief(it) for it in data["items"]],
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
for it in data["items"]:
|
|
1178
|
+
it["decision"] = None
|
|
1179
|
+
it.pop("reject_reason", None)
|
|
1180
|
+
for ref, decision in refs:
|
|
1181
|
+
index[str(ref).strip().lower()]["decision"] = decision
|
|
1182
|
+
|
|
1183
|
+
conn.execute(
|
|
1184
|
+
"UPDATE decisions SET decision = NULL, reject_reason = NULL WHERE session_id = ?",
|
|
1185
|
+
(session,),
|
|
1186
|
+
)
|
|
1187
|
+
for ref, decision in refs:
|
|
1188
|
+
it = index[str(ref).strip().lower()]
|
|
1189
|
+
conn.execute(
|
|
1190
|
+
"UPDATE decisions SET decision = ? WHERE session_id = ? AND paper_key = ?",
|
|
1191
|
+
(decision, session, paper_key(it)),
|
|
1192
|
+
)
|
|
1193
|
+
data["updated"] = now
|
|
1194
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1195
|
+
|
|
1196
|
+
chosen = [it for it in data["items"] if it.get("decision") == "selected"]
|
|
1197
|
+
return {
|
|
1198
|
+
"session": session,
|
|
1199
|
+
"selected": len(chosen),
|
|
1200
|
+
"identifiers": [it["doi"] for it in chosen if it.get("doi")],
|
|
1201
|
+
"chosen": [_decision_brief(it) for it in chosen],
|
|
1202
|
+
"rejected": [_decision_brief(it) for it in data["items"] if it.get("decision") == "rejected"],
|
|
1203
|
+
"unknown": unknown,
|
|
1204
|
+
"record": data,
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def translate_screening(
|
|
1209
|
+
session: str,
|
|
1210
|
+
items: list[dict[str, Any]],
|
|
1211
|
+
*,
|
|
1212
|
+
db_file: Path | str | None = None,
|
|
1213
|
+
) -> dict[str, Any]:
|
|
1214
|
+
"""Store Chinese abstracts in SQLite without altering ratings or decisions."""
|
|
1215
|
+
sdir = _screening_dir(db_file)
|
|
1216
|
+
now = _now_iso()
|
|
1217
|
+
with transaction(db_file) as conn:
|
|
1218
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1219
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1220
|
+
data = _load_session_record(conn, session)
|
|
1221
|
+
by_doi = {str(it.get("doi") or "").lower(): it for it in data["items"] if it.get("doi")}
|
|
1222
|
+
by_n = {it.get("n"): it for it in data["items"]}
|
|
1223
|
+
saved, missing = 0, []
|
|
1224
|
+
for raw in items or []:
|
|
1225
|
+
target = by_doi.get(str(raw.get("doi") or "").lower()) or by_n.get(raw.get("n"))
|
|
1226
|
+
text = str(raw.get("abstract_zh") or "").strip()
|
|
1227
|
+
if not target or not text:
|
|
1228
|
+
missing.append(raw.get("doi") or raw.get("n"))
|
|
1229
|
+
continue
|
|
1230
|
+
zh = text[:ABSTRACT_CHARS]
|
|
1231
|
+
conn.execute(
|
|
1232
|
+
"UPDATE decisions SET abstract_zh = ? WHERE session_id = ? AND paper_key = ?",
|
|
1233
|
+
(zh, session, paper_key(target)),
|
|
1234
|
+
)
|
|
1235
|
+
saved += 1
|
|
1236
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1237
|
+
return {"session": session, "saved": saved, "not_found": missing}
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def _find_decision_row(
|
|
1241
|
+
conn: sqlite3.Connection,
|
|
1242
|
+
session: str,
|
|
1243
|
+
doi: str | None,
|
|
1244
|
+
title: str | None,
|
|
1245
|
+
) -> sqlite3.Row | None:
|
|
1246
|
+
key = paper_key({"doi": doi, "title": title})
|
|
1247
|
+
row = conn.execute(
|
|
1248
|
+
"SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1249
|
+
(session, key),
|
|
1250
|
+
).fetchone()
|
|
1251
|
+
if row is not None:
|
|
1252
|
+
return row
|
|
1253
|
+
# Fallback scan in case legacy paper_key differed slightly
|
|
1254
|
+
for candidate in conn.execute(
|
|
1255
|
+
"SELECT * FROM decisions WHERE session_id = ?", (session,)
|
|
1256
|
+
).fetchall():
|
|
1257
|
+
if paper_key({"doi": candidate["doi"], "title": candidate["title"]}) == key:
|
|
1258
|
+
return candidate
|
|
1259
|
+
return None
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def update_item_rating(
|
|
1263
|
+
session: str,
|
|
1264
|
+
doi: str | None,
|
|
1265
|
+
title: str | None,
|
|
1266
|
+
relevance: str | None,
|
|
1267
|
+
*,
|
|
1268
|
+
create_paper: dict[str, Any] | None = None,
|
|
1269
|
+
db_file: Path | str | None = None,
|
|
1270
|
+
) -> dict[str, Any] | None:
|
|
1271
|
+
"""Update or create a user rating for one paper in a session.
|
|
1272
|
+
|
|
1273
|
+
Returns ``None`` if the paper is not in the session and cannot be created.
|
|
1274
|
+
"""
|
|
1275
|
+
if relevance is not None and relevance not in _RANK:
|
|
1276
|
+
raise ValueError(f"relevance must be one of {RELEVANCE}, got {relevance!r}")
|
|
1277
|
+
sdir = _screening_dir(db_file)
|
|
1278
|
+
now = _now_iso()
|
|
1279
|
+
with transaction(db_file) as conn:
|
|
1280
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1281
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1282
|
+
row = _find_decision_row(conn, session, doi, title)
|
|
1283
|
+
if row is None:
|
|
1284
|
+
if relevance is None or not create_paper:
|
|
1285
|
+
return None
|
|
1286
|
+
max_n_row = conn.execute(
|
|
1287
|
+
"SELECT COALESCE(MAX(n), 0) FROM decisions WHERE session_id = ?",
|
|
1288
|
+
(session,),
|
|
1289
|
+
).fetchone()
|
|
1290
|
+
next_n = int(max_n_row[0] or 0) + 1
|
|
1291
|
+
item = {
|
|
1292
|
+
k: create_paper.get(k)
|
|
1293
|
+
for k in (
|
|
1294
|
+
"doi",
|
|
1295
|
+
"title",
|
|
1296
|
+
"year",
|
|
1297
|
+
"journal",
|
|
1298
|
+
"journal_rank",
|
|
1299
|
+
"authors",
|
|
1300
|
+
"cited_by_count",
|
|
1301
|
+
"abstract",
|
|
1302
|
+
"abstract_missing",
|
|
1303
|
+
"retracted",
|
|
1304
|
+
"concern",
|
|
1305
|
+
)
|
|
1306
|
+
if create_paper.get(k) not in (None, "", [])
|
|
1307
|
+
}
|
|
1308
|
+
if item.get("abstract"):
|
|
1309
|
+
item["abstract"] = str(item["abstract"])[:ABSTRACT_CHARS]
|
|
1310
|
+
if isinstance(item.get("authors"), list):
|
|
1311
|
+
item["authors"] = item["authors"][:3]
|
|
1312
|
+
item["n"] = next_n
|
|
1313
|
+
item["relevance"] = relevance
|
|
1314
|
+
item["user_relevance"] = relevance
|
|
1315
|
+
item["reason"] = ""
|
|
1316
|
+
item["decision"] = None
|
|
1317
|
+
_upsert_decision(conn, session, item)
|
|
1318
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1319
|
+
created = _find_decision_row(conn, session, item.get("doi"), item.get("title"))
|
|
1320
|
+
edited = int(
|
|
1321
|
+
conn.execute(
|
|
1322
|
+
"SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
|
|
1323
|
+
(session,),
|
|
1324
|
+
).fetchone()[0]
|
|
1325
|
+
)
|
|
1326
|
+
return {"item": _decision_row_to_dict(created) if created else item, "edited": edited}
|
|
1327
|
+
|
|
1328
|
+
pkey = row["paper_key"]
|
|
1329
|
+
if not row["reason"]:
|
|
1330
|
+
if relevance is None:
|
|
1331
|
+
conn.execute(
|
|
1332
|
+
"DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1333
|
+
(session, pkey),
|
|
1334
|
+
)
|
|
1335
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1336
|
+
edited = int(
|
|
1337
|
+
conn.execute(
|
|
1338
|
+
"SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
|
|
1339
|
+
(session,),
|
|
1340
|
+
).fetchone()[0]
|
|
1341
|
+
)
|
|
1342
|
+
return {"item": None, "edited": edited}
|
|
1343
|
+
conn.execute(
|
|
1344
|
+
"UPDATE decisions SET relevance = ?, user_relevance = ? WHERE session_id = ? AND paper_key = ?",
|
|
1345
|
+
(relevance, relevance, session, pkey),
|
|
1346
|
+
)
|
|
1347
|
+
elif relevance is None or relevance == row["relevance"]:
|
|
1348
|
+
conn.execute(
|
|
1349
|
+
"UPDATE decisions SET user_relevance = NULL WHERE session_id = ? AND paper_key = ?",
|
|
1350
|
+
(session, pkey),
|
|
1351
|
+
)
|
|
1352
|
+
else:
|
|
1353
|
+
conn.execute(
|
|
1354
|
+
"UPDATE decisions SET user_relevance = ? WHERE session_id = ? AND paper_key = ?",
|
|
1355
|
+
(relevance, session, pkey),
|
|
1356
|
+
)
|
|
1357
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1358
|
+
updated_row = conn.execute(
|
|
1359
|
+
"SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1360
|
+
(session, pkey),
|
|
1361
|
+
).fetchone()
|
|
1362
|
+
edited = int(
|
|
1363
|
+
conn.execute(
|
|
1364
|
+
"SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
|
|
1365
|
+
(session,),
|
|
1366
|
+
).fetchone()[0]
|
|
1367
|
+
)
|
|
1368
|
+
return {"item": _decision_row_to_dict(updated_row) if updated_row else None, "edited": edited}
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def update_item_decision(
|
|
1372
|
+
session: str,
|
|
1373
|
+
doi: str | None,
|
|
1374
|
+
title: str | None,
|
|
1375
|
+
decision: str | None,
|
|
1376
|
+
reason: str | None = None,
|
|
1377
|
+
*,
|
|
1378
|
+
db_file: Path | str | None = None,
|
|
1379
|
+
) -> dict[str, Any] | None:
|
|
1380
|
+
"""Update decision and reject_reason for one paper in a session."""
|
|
1381
|
+
if decision not in (None, "selected", "rejected"):
|
|
1382
|
+
raise ValueError(f"invalid decision {decision!r}")
|
|
1383
|
+
sdir = _screening_dir(db_file)
|
|
1384
|
+
now = _now_iso()
|
|
1385
|
+
with transaction(db_file) as conn:
|
|
1386
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1387
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1388
|
+
row = _find_decision_row(conn, session, doi, title)
|
|
1389
|
+
if row is None:
|
|
1390
|
+
return None
|
|
1391
|
+
clean_reason = " ".join((reason or "").split())[:60]
|
|
1392
|
+
reject_reason = clean_reason if decision == "rejected" and clean_reason else None
|
|
1393
|
+
conn.execute(
|
|
1394
|
+
"UPDATE decisions SET decision = ?, reject_reason = ? WHERE session_id = ? AND paper_key = ?",
|
|
1395
|
+
(decision, reject_reason, session, row["paper_key"]),
|
|
1396
|
+
)
|
|
1397
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1398
|
+
updated_row = conn.execute(
|
|
1399
|
+
"SELECT * FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1400
|
+
(session, row["paper_key"]),
|
|
1401
|
+
).fetchone()
|
|
1402
|
+
return {"item": _decision_row_to_dict(updated_row)} if updated_row else None
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
def set_item_abstract_zh(
|
|
1406
|
+
session: str,
|
|
1407
|
+
doi: str | None,
|
|
1408
|
+
title: str | None,
|
|
1409
|
+
abstract_zh: str,
|
|
1410
|
+
*,
|
|
1411
|
+
db_file: Path | str | None = None,
|
|
1412
|
+
) -> bool:
|
|
1413
|
+
"""Persist translated Chinese abstract for a single decision row."""
|
|
1414
|
+
sdir = _screening_dir(db_file)
|
|
1415
|
+
now = _now_iso()
|
|
1416
|
+
with transaction(db_file) as conn:
|
|
1417
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1418
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1419
|
+
row = _find_decision_row(conn, session, doi, title)
|
|
1420
|
+
if row is None:
|
|
1421
|
+
return False
|
|
1422
|
+
conn.execute(
|
|
1423
|
+
"UPDATE decisions SET abstract_zh = ? WHERE session_id = ? AND paper_key = ?",
|
|
1424
|
+
(str(abstract_zh)[:ABSTRACT_CHARS], session, row["paper_key"]),
|
|
1425
|
+
)
|
|
1426
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1427
|
+
return True
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def batch_update_items(
|
|
1431
|
+
session: str,
|
|
1432
|
+
ops: list[Any],
|
|
1433
|
+
*,
|
|
1434
|
+
create_papers: dict[str, dict[str, Any]] | None = None,
|
|
1435
|
+
db_file: Path | str | None = None,
|
|
1436
|
+
) -> dict[str, Any]:
|
|
1437
|
+
"""Atomically apply batch rating/decision updates (including unrated batch items) in one transaction."""
|
|
1438
|
+
sdir = _screening_dir(db_file)
|
|
1439
|
+
now = _now_iso()
|
|
1440
|
+
papers_by_key = create_papers or {}
|
|
1441
|
+
|
|
1442
|
+
def _field(op: Any, name: str) -> tuple[bool, Any]:
|
|
1443
|
+
if isinstance(op, dict):
|
|
1444
|
+
return (name in op, op.get(name))
|
|
1445
|
+
return (name in getattr(op, "model_fields_set", set()), getattr(op, name, None))
|
|
1446
|
+
|
|
1447
|
+
with transaction(db_file) as conn:
|
|
1448
|
+
if not _ensure_session_in_conn(conn, session, sdir=sdir, in_write_tx=True):
|
|
1449
|
+
raise FileNotFoundError(f"no screening session {session!r}")
|
|
1450
|
+
|
|
1451
|
+
rows = conn.execute(
|
|
1452
|
+
"SELECT * FROM decisions WHERE session_id = ? ORDER BY n ASC, rowid ASC",
|
|
1453
|
+
(session,),
|
|
1454
|
+
).fetchall()
|
|
1455
|
+
by_key: dict[str, dict[str, Any]] = {}
|
|
1456
|
+
max_n = 0
|
|
1457
|
+
for r in rows:
|
|
1458
|
+
it = _decision_row_to_dict(r)
|
|
1459
|
+
by_key[paper_key(it)] = it
|
|
1460
|
+
if int(r["n"] or 0) > max_n:
|
|
1461
|
+
max_n = int(r["n"] or 0)
|
|
1462
|
+
|
|
1463
|
+
created_keys: set[str] = set()
|
|
1464
|
+
for op in ops:
|
|
1465
|
+
_, delete_if_unrated = _field(op, "delete_if_unrated")
|
|
1466
|
+
_, clear_user_rel = _field(op, "clear_user_relevance")
|
|
1467
|
+
has_rel, rel_val = _field(op, "relevance")
|
|
1468
|
+
if delete_if_unrated or clear_user_rel or not has_rel or rel_val is None:
|
|
1469
|
+
continue
|
|
1470
|
+
_, doi = _field(op, "doi")
|
|
1471
|
+
_, title = _field(op, "title")
|
|
1472
|
+
key = paper_key({"doi": doi, "title": title})
|
|
1473
|
+
if key in by_key:
|
|
1474
|
+
continue
|
|
1475
|
+
paper = papers_by_key.get(key)
|
|
1476
|
+
if not paper:
|
|
1477
|
+
continue
|
|
1478
|
+
max_n += 1
|
|
1479
|
+
new_item = {
|
|
1480
|
+
k: paper.get(k)
|
|
1481
|
+
for k in (
|
|
1482
|
+
"doi",
|
|
1483
|
+
"title",
|
|
1484
|
+
"year",
|
|
1485
|
+
"journal",
|
|
1486
|
+
"journal_rank",
|
|
1487
|
+
"authors",
|
|
1488
|
+
"cited_by_count",
|
|
1489
|
+
"abstract",
|
|
1490
|
+
"abstract_missing",
|
|
1491
|
+
"retracted",
|
|
1492
|
+
"concern",
|
|
1493
|
+
)
|
|
1494
|
+
if paper.get(k) not in (None, "", [])
|
|
1495
|
+
}
|
|
1496
|
+
if new_item.get("abstract"):
|
|
1497
|
+
new_item["abstract"] = str(new_item["abstract"])[:ABSTRACT_CHARS]
|
|
1498
|
+
if isinstance(new_item.get("authors"), list):
|
|
1499
|
+
new_item["authors"] = new_item["authors"][:3]
|
|
1500
|
+
new_item["n"] = max_n
|
|
1501
|
+
new_item["relevance"] = _norm_relevance(rel_val)
|
|
1502
|
+
new_item["user_relevance"] = new_item["relevance"]
|
|
1503
|
+
new_item["reason"] = ""
|
|
1504
|
+
new_item["decision"] = None
|
|
1505
|
+
_upsert_decision(conn, session, new_item, pkey=key)
|
|
1506
|
+
by_key[key] = new_item
|
|
1507
|
+
created_keys.add(key)
|
|
1508
|
+
|
|
1509
|
+
updated = 0
|
|
1510
|
+
skipped = 0
|
|
1511
|
+
previous: list[dict[str, Any]] = []
|
|
1512
|
+
for op in ops:
|
|
1513
|
+
_, doi = _field(op, "doi")
|
|
1514
|
+
_, title = _field(op, "title")
|
|
1515
|
+
_, delete_if_unrated = _field(op, "delete_if_unrated")
|
|
1516
|
+
_, clear_user_rel = _field(op, "clear_user_relevance")
|
|
1517
|
+
has_rel, rel_val = _field(op, "relevance")
|
|
1518
|
+
has_dec, dec_val = _field(op, "decision")
|
|
1519
|
+
has_rr, rr_val = _field(op, "reject_reason")
|
|
1520
|
+
|
|
1521
|
+
key = paper_key({"doi": doi, "title": title})
|
|
1522
|
+
item = by_key.get(key)
|
|
1523
|
+
if item is None:
|
|
1524
|
+
skipped += 1
|
|
1525
|
+
continue
|
|
1526
|
+
|
|
1527
|
+
existed = key not in created_keys
|
|
1528
|
+
prev = {
|
|
1529
|
+
"doi": item.get("doi"),
|
|
1530
|
+
"title": item.get("title"),
|
|
1531
|
+
"existed": existed,
|
|
1532
|
+
"relevance": item.get("relevance") if existed else None,
|
|
1533
|
+
"user_relevance": item.get("user_relevance") if existed else None,
|
|
1534
|
+
"decision": item.get("decision") if existed else None,
|
|
1535
|
+
"reject_reason": item.get("reject_reason") if existed else None,
|
|
1536
|
+
}
|
|
1537
|
+
created_keys.discard(key)
|
|
1538
|
+
|
|
1539
|
+
if delete_if_unrated:
|
|
1540
|
+
if not item.get("reason"):
|
|
1541
|
+
conn.execute(
|
|
1542
|
+
"DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1543
|
+
(session, key),
|
|
1544
|
+
)
|
|
1545
|
+
by_key.pop(key, None)
|
|
1546
|
+
updated += 1
|
|
1547
|
+
previous.append(prev)
|
|
1548
|
+
continue
|
|
1549
|
+
item.pop("user_relevance", None)
|
|
1550
|
+
else:
|
|
1551
|
+
if clear_user_rel or has_rel:
|
|
1552
|
+
target_rel = None if clear_user_rel else rel_val
|
|
1553
|
+
if not item.get("reason"):
|
|
1554
|
+
if target_rel is None:
|
|
1555
|
+
conn.execute(
|
|
1556
|
+
"DELETE FROM decisions WHERE session_id = ? AND paper_key = ?",
|
|
1557
|
+
(session, key),
|
|
1558
|
+
)
|
|
1559
|
+
by_key.pop(key, None)
|
|
1560
|
+
updated += 1
|
|
1561
|
+
previous.append(prev)
|
|
1562
|
+
continue
|
|
1563
|
+
item["relevance"] = item["user_relevance"] = target_rel
|
|
1564
|
+
elif target_rel is None or target_rel == item["relevance"]:
|
|
1565
|
+
item.pop("user_relevance", None)
|
|
1566
|
+
else:
|
|
1567
|
+
item["user_relevance"] = target_rel
|
|
1568
|
+
if has_dec:
|
|
1569
|
+
item["decision"] = dec_val
|
|
1570
|
+
raw_reason = rr_val if has_rr else None
|
|
1571
|
+
reason = " ".join((raw_reason or "").split())[:60]
|
|
1572
|
+
if dec_val == "rejected" and reason:
|
|
1573
|
+
item["reject_reason"] = reason
|
|
1574
|
+
else:
|
|
1575
|
+
item.pop("reject_reason", None)
|
|
1576
|
+
|
|
1577
|
+
_upsert_decision(conn, session, item, pkey=key)
|
|
1578
|
+
updated += 1
|
|
1579
|
+
previous.append(prev)
|
|
1580
|
+
|
|
1581
|
+
conn.execute("UPDATE sessions SET updated = ? WHERE id = ?", (now, session))
|
|
1582
|
+
edited = int(
|
|
1583
|
+
conn.execute(
|
|
1584
|
+
"SELECT COUNT(*) FROM decisions WHERE session_id = ? AND user_relevance IS NOT NULL",
|
|
1585
|
+
(session,),
|
|
1586
|
+
).fetchone()[0]
|
|
1587
|
+
)
|
|
1588
|
+
return {"updated": updated, "skipped": skipped, "edited": edited, "previous": previous}
|
|
1589
|
+
|
|
1590
|
+
|
|
1591
|
+
def pending_selected_by_session(*, db_file: Path | str | None = None) -> list[dict[str, Any]]:
|
|
1592
|
+
"""Return sessions newest-first with their selected papers that carry a DOI."""
|
|
1593
|
+
sdir = _screening_dir(db_file)
|
|
1594
|
+
with connect(db_file) as conn:
|
|
1595
|
+
_sync_missing_session_jsons(conn, sdir=sdir)
|
|
1596
|
+
rows = conn.execute(
|
|
1597
|
+
"""
|
|
1598
|
+
SELECT s.id AS session, s.question, d.doi, d.title, d.year, d.journal
|
|
1599
|
+
FROM sessions s
|
|
1600
|
+
JOIN decisions d ON d.session_id = s.id
|
|
1601
|
+
WHERE d.decision = 'selected' AND d.doi IS NOT NULL AND d.doi != ''
|
|
1602
|
+
ORDER BY s.id DESC, d.n ASC, d.rowid ASC
|
|
1603
|
+
"""
|
|
1604
|
+
).fetchall()
|
|
1605
|
+
grouped: dict[str, dict[str, Any]] = {}
|
|
1606
|
+
order: list[str] = []
|
|
1607
|
+
for r in rows:
|
|
1608
|
+
sid = r["session"]
|
|
1609
|
+
if sid not in grouped:
|
|
1610
|
+
grouped[sid] = {"session": sid, "question": r["question"], "papers": []}
|
|
1611
|
+
order.append(sid)
|
|
1612
|
+
grouped[sid]["papers"].append(
|
|
1613
|
+
{"doi": r["doi"], "title": r["title"], "year": r["year"], "journal": r["journal"]}
|
|
1614
|
+
)
|
|
1615
|
+
return [grouped[sid] for sid in order]
|
|
1616
|
+
|
|
1617
|
+
|
|
1618
|
+
# --------------------------------------------------------------------------- deep search pool & summary
|
|
1619
|
+
|
|
1620
|
+
_BATCH_FIELDS = (
|
|
1621
|
+
"doi",
|
|
1622
|
+
"title",
|
|
1623
|
+
"year",
|
|
1624
|
+
"journal",
|
|
1625
|
+
"journal_rank",
|
|
1626
|
+
"retracted",
|
|
1627
|
+
"concern",
|
|
1628
|
+
"authors",
|
|
1629
|
+
"cited_by_count",
|
|
1630
|
+
"abstract",
|
|
1631
|
+
"abstract_missing",
|
|
1632
|
+
)
|
|
1633
|
+
|
|
1634
|
+
|
|
1635
|
+
def _safe_session_file(session: str, sdir: Path, suffix: str) -> Path | None:
|
|
1636
|
+
if not session or session.startswith(".") or "/" in session or "\\" in session:
|
|
1637
|
+
return None
|
|
1638
|
+
if not _SAFE_SESSION_RE.match(session):
|
|
1639
|
+
return None
|
|
1640
|
+
p = sdir / f"{session}{suffix}"
|
|
1641
|
+
return p if p.is_file() else None
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
def _ensure_pool_in_conn(
|
|
1645
|
+
conn: sqlite3.Connection,
|
|
1646
|
+
session: str,
|
|
1647
|
+
*,
|
|
1648
|
+
sdir: Path,
|
|
1649
|
+
in_write_tx: bool,
|
|
1650
|
+
) -> bool:
|
|
1651
|
+
row = conn.execute(
|
|
1652
|
+
"SELECT pool_state_json FROM sessions WHERE id = ?", (session,)
|
|
1653
|
+
).fetchone()
|
|
1654
|
+
if row is not None and row["pool_state_json"] is not None:
|
|
1655
|
+
return True
|
|
1656
|
+
has_items = conn.execute(
|
|
1657
|
+
"SELECT 1 FROM pool_items WHERE session_id = ? LIMIT 1", (session,)
|
|
1658
|
+
).fetchone()
|
|
1659
|
+
if has_items is not None:
|
|
1660
|
+
return True
|
|
1661
|
+
|
|
1662
|
+
legacy = _safe_session_file(session, sdir, ".pool.json")
|
|
1663
|
+
if legacy is None:
|
|
1664
|
+
return False
|
|
1665
|
+
pdata = _read_json_safe(legacy)
|
|
1666
|
+
if not isinstance(pdata, dict):
|
|
1667
|
+
return False
|
|
1668
|
+
|
|
1669
|
+
counts = {"sessions": 0, "pool_items": 0}
|
|
1670
|
+
if not in_write_tx:
|
|
1671
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
1672
|
+
try:
|
|
1673
|
+
_import_pool_file(conn, session, pdata, counts)
|
|
1674
|
+
if conn.in_transaction:
|
|
1675
|
+
conn.execute("COMMIT")
|
|
1676
|
+
except BaseException:
|
|
1677
|
+
if conn.in_transaction:
|
|
1678
|
+
conn.execute("ROLLBACK")
|
|
1679
|
+
raise
|
|
1680
|
+
else:
|
|
1681
|
+
_import_pool_file(conn, session, pdata, counts)
|
|
1682
|
+
return True
|
|
1683
|
+
|
|
1684
|
+
|
|
1685
|
+
def _ensure_summary_in_conn(
|
|
1686
|
+
conn: sqlite3.Connection,
|
|
1687
|
+
session: str,
|
|
1688
|
+
*,
|
|
1689
|
+
sdir: Path,
|
|
1690
|
+
in_write_tx: bool,
|
|
1691
|
+
) -> str | None:
|
|
1692
|
+
row = conn.execute(
|
|
1693
|
+
"SELECT summary_json FROM sessions WHERE id = ?", (session,)
|
|
1694
|
+
).fetchone()
|
|
1695
|
+
if row is not None and row["summary_json"] is not None:
|
|
1696
|
+
return str(row["summary_json"])
|
|
1697
|
+
|
|
1698
|
+
legacy = _safe_session_file(session, sdir, ".summary.json")
|
|
1699
|
+
if legacy is None:
|
|
1700
|
+
return None
|
|
1701
|
+
summary = _read_json_safe(legacy)
|
|
1702
|
+
if not isinstance(summary, dict):
|
|
1703
|
+
return None
|
|
1704
|
+
|
|
1705
|
+
counts = {"sessions": 0}
|
|
1706
|
+
if not in_write_tx:
|
|
1707
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
1708
|
+
try:
|
|
1709
|
+
_import_summary_file(conn, session, summary, counts)
|
|
1710
|
+
if conn.in_transaction:
|
|
1711
|
+
conn.execute("COMMIT")
|
|
1712
|
+
except BaseException:
|
|
1713
|
+
if conn.in_transaction:
|
|
1714
|
+
conn.execute("ROLLBACK")
|
|
1715
|
+
raise
|
|
1716
|
+
else:
|
|
1717
|
+
_import_summary_file(conn, session, summary, counts)
|
|
1718
|
+
return json.dumps(summary, ensure_ascii=False)
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
def _pool_row_to_dict(row: sqlite3.Row) -> dict[str, Any]:
|
|
1722
|
+
raw_data = row["data_json"]
|
|
1723
|
+
if raw_data:
|
|
1724
|
+
try:
|
|
1725
|
+
rec = json.loads(raw_data)
|
|
1726
|
+
if isinstance(rec, dict) and rec:
|
|
1727
|
+
return rec
|
|
1728
|
+
except (ValueError, TypeError):
|
|
1729
|
+
pass
|
|
1730
|
+
rec: dict[str, Any] = {}
|
|
1731
|
+
if row["doi"] is not None:
|
|
1732
|
+
rec["doi"] = row["doi"]
|
|
1733
|
+
if row["openalex"] is not None:
|
|
1734
|
+
rec["openalex"] = row["openalex"]
|
|
1735
|
+
rec["title"] = row["title"]
|
|
1736
|
+
if row["year"] is not None:
|
|
1737
|
+
rec["year"] = int(row["year"])
|
|
1738
|
+
if row["journal"]:
|
|
1739
|
+
rec["journal"] = row["journal"]
|
|
1740
|
+
rec["cited_by_count"] = int(row["cited_by_count"] or 0)
|
|
1741
|
+
if row["abstract"]:
|
|
1742
|
+
rec["abstract"] = row["abstract"]
|
|
1743
|
+
if row["score"] is not None:
|
|
1744
|
+
rec["score"] = float(row["score"])
|
|
1745
|
+
return rec
|
|
1746
|
+
|
|
1747
|
+
|
|
1748
|
+
def has_pool(session: str, *, db_file: Path | str | None = None) -> bool:
|
|
1749
|
+
"""Return True if a deep-search candidate pool exists in SQLite or legacy JSON."""
|
|
1750
|
+
sdir = _screening_dir(db_file)
|
|
1751
|
+
with connect(db_file) as conn:
|
|
1752
|
+
return _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False)
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
def load_pool_state(session: str, *, db_file: Path | str | None = None) -> dict[str, Any]:
|
|
1756
|
+
"""Load deep-search pool metadata and candidate items from SQLite (auto-importing legacy JSON)."""
|
|
1757
|
+
sdir = _screening_dir(db_file)
|
|
1758
|
+
with connect(db_file) as conn:
|
|
1759
|
+
if not _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False):
|
|
1760
|
+
raise FileNotFoundError(f"no deep search for session {session!r}")
|
|
1761
|
+
srow = conn.execute(
|
|
1762
|
+
"SELECT pool_state_json, question FROM sessions WHERE id = ?", (session,)
|
|
1763
|
+
).fetchone()
|
|
1764
|
+
meta: dict[str, Any] = {}
|
|
1765
|
+
if srow is not None and srow["pool_state_json"]:
|
|
1766
|
+
parsed = json.loads(srow["pool_state_json"])
|
|
1767
|
+
if isinstance(parsed, dict):
|
|
1768
|
+
meta = parsed
|
|
1769
|
+
meta.setdefault("session", session)
|
|
1770
|
+
if "question" not in meta and srow is not None and srow["question"]:
|
|
1771
|
+
meta["question"] = srow["question"]
|
|
1772
|
+
|
|
1773
|
+
rows = conn.execute(
|
|
1774
|
+
"""
|
|
1775
|
+
SELECT paper_key, doi, openalex, title, year, journal,
|
|
1776
|
+
cited_by_count, abstract, score, data_json
|
|
1777
|
+
FROM pool_items
|
|
1778
|
+
WHERE session_id = ?
|
|
1779
|
+
ORDER BY rowid ASC
|
|
1780
|
+
""",
|
|
1781
|
+
(session,),
|
|
1782
|
+
).fetchall()
|
|
1783
|
+
pool = {str(r["paper_key"]): _pool_row_to_dict(r) for r in rows}
|
|
1784
|
+
return {**meta, "pool": pool}
|
|
1785
|
+
|
|
1786
|
+
|
|
1787
|
+
def save_pool_state(state: dict[str, Any], *, db_file: Path | str | None = None) -> None:
|
|
1788
|
+
"""Persist deep-search pool metadata and candidate items in a single transaction."""
|
|
1789
|
+
sid = str(state["session"])
|
|
1790
|
+
now = _now_iso()
|
|
1791
|
+
pool_meta = {k: v for k, v in state.items() if k != "pool"}
|
|
1792
|
+
pool = state.get("pool") or {}
|
|
1793
|
+
pool_meta_json = json.dumps(pool_meta, ensure_ascii=False)
|
|
1794
|
+
|
|
1795
|
+
with transaction(db_file) as conn:
|
|
1796
|
+
conn.execute(
|
|
1797
|
+
"""
|
|
1798
|
+
INSERT INTO sessions (id, question, mode, updated, pool_state_json)
|
|
1799
|
+
VALUES (?, ?, 'deep', ?, ?)
|
|
1800
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1801
|
+
pool_state_json = excluded.pool_state_json,
|
|
1802
|
+
updated = CASE WHEN sessions.updated = '' THEN excluded.updated ELSE sessions.updated END
|
|
1803
|
+
""",
|
|
1804
|
+
(sid, str(state.get("question") or ""), now, pool_meta_json),
|
|
1805
|
+
)
|
|
1806
|
+
|
|
1807
|
+
existing_keys = {
|
|
1808
|
+
r[0]
|
|
1809
|
+
for r in conn.execute(
|
|
1810
|
+
"SELECT paper_key FROM pool_items WHERE session_id = ?", (sid,)
|
|
1811
|
+
).fetchall()
|
|
1812
|
+
}
|
|
1813
|
+
new_keys = {str(k) for k, v in pool.items() if isinstance(v, dict)}
|
|
1814
|
+
stale_keys = existing_keys - new_keys
|
|
1815
|
+
if stale_keys:
|
|
1816
|
+
conn.executemany(
|
|
1817
|
+
"DELETE FROM pool_items WHERE session_id = ? AND paper_key = ?",
|
|
1818
|
+
[(sid, k) for k in stale_keys],
|
|
1819
|
+
)
|
|
1820
|
+
|
|
1821
|
+
rows = []
|
|
1822
|
+
for pkey, rec in pool.items():
|
|
1823
|
+
if not isinstance(rec, dict):
|
|
1824
|
+
continue
|
|
1825
|
+
rows.append(
|
|
1826
|
+
(
|
|
1827
|
+
sid,
|
|
1828
|
+
str(pkey),
|
|
1829
|
+
str(rec["doi"]) if rec.get("doi") is not None else None,
|
|
1830
|
+
str(rec["openalex"]) if rec.get("openalex") is not None else None,
|
|
1831
|
+
str(rec.get("title") or ""),
|
|
1832
|
+
_opt_int(rec.get("year")),
|
|
1833
|
+
str(rec.get("journal") or ""),
|
|
1834
|
+
_opt_int(rec.get("cited_by_count")) or 0,
|
|
1835
|
+
str(rec.get("abstract") or ""),
|
|
1836
|
+
_opt_float(rec.get("score")),
|
|
1837
|
+
json.dumps(rec, ensure_ascii=False),
|
|
1838
|
+
)
|
|
1839
|
+
)
|
|
1840
|
+
if rows:
|
|
1841
|
+
conn.executemany(
|
|
1842
|
+
"""
|
|
1843
|
+
INSERT OR REPLACE INTO pool_items (
|
|
1844
|
+
session_id, paper_key, doi, openalex, title, year, journal,
|
|
1845
|
+
cited_by_count, abstract, score, data_json
|
|
1846
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1847
|
+
""",
|
|
1848
|
+
rows,
|
|
1849
|
+
)
|
|
1850
|
+
|
|
1851
|
+
|
|
1852
|
+
def load_summary(session: str, *, db_file: Path | str | None = None) -> dict[str, Any] | None:
|
|
1853
|
+
"""Load deep-search summary dict from SQLite (auto-importing legacy summary JSON)."""
|
|
1854
|
+
sdir = _screening_dir(db_file)
|
|
1855
|
+
with connect(db_file) as conn:
|
|
1856
|
+
raw = _ensure_summary_in_conn(conn, session, sdir=sdir, in_write_tx=False)
|
|
1857
|
+
if raw is None:
|
|
1858
|
+
return None
|
|
1859
|
+
try:
|
|
1860
|
+
parsed = json.loads(raw)
|
|
1861
|
+
return parsed if isinstance(parsed, dict) else None
|
|
1862
|
+
except (ValueError, TypeError):
|
|
1863
|
+
return None
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
def save_summary(
|
|
1867
|
+
session: str,
|
|
1868
|
+
summary: dict[str, Any],
|
|
1869
|
+
*,
|
|
1870
|
+
db_file: Path | str | None = None,
|
|
1871
|
+
) -> None:
|
|
1872
|
+
"""Persist deep-search summary JSON into sessions.summary_json."""
|
|
1873
|
+
now = _now_iso()
|
|
1874
|
+
summary_json = json.dumps(summary, ensure_ascii=False)
|
|
1875
|
+
with transaction(db_file) as conn:
|
|
1876
|
+
conn.execute(
|
|
1877
|
+
"""
|
|
1878
|
+
INSERT INTO sessions (id, question, mode, updated, summary_json)
|
|
1879
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1880
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1881
|
+
summary_json = excluded.summary_json
|
|
1882
|
+
""",
|
|
1883
|
+
(
|
|
1884
|
+
session,
|
|
1885
|
+
str(summary.get("question") or ""),
|
|
1886
|
+
str(summary.get("mode") or "deep"),
|
|
1887
|
+
str(summary.get("updated") or now),
|
|
1888
|
+
summary_json,
|
|
1889
|
+
),
|
|
1890
|
+
)
|
|
1891
|
+
|
|
1892
|
+
|
|
1893
|
+
def get_batch_paper(
|
|
1894
|
+
session: str,
|
|
1895
|
+
doi: str | None,
|
|
1896
|
+
title: str | None,
|
|
1897
|
+
*,
|
|
1898
|
+
db_file: Path | str | None = None,
|
|
1899
|
+
) -> dict[str, Any] | None:
|
|
1900
|
+
"""Return one paper from the session's latest batch without loading pool_items."""
|
|
1901
|
+
sdir = _screening_dir(db_file)
|
|
1902
|
+
with connect(db_file) as conn:
|
|
1903
|
+
if not _ensure_pool_in_conn(conn, session, sdir=sdir, in_write_tx=False):
|
|
1904
|
+
return None
|
|
1905
|
+
row = conn.execute(
|
|
1906
|
+
"SELECT pool_state_json FROM sessions WHERE id = ?", (session,)
|
|
1907
|
+
).fetchone()
|
|
1908
|
+
if row is None or not row["pool_state_json"]:
|
|
1909
|
+
return None
|
|
1910
|
+
try:
|
|
1911
|
+
meta = json.loads(row["pool_state_json"])
|
|
1912
|
+
except (ValueError, TypeError):
|
|
1913
|
+
return None
|
|
1914
|
+
if not isinstance(meta, dict):
|
|
1915
|
+
return None
|
|
1916
|
+
target_key = paper_key({"doi": doi, "title": title})
|
|
1917
|
+
for b in meta.get("last_batch") or []:
|
|
1918
|
+
if isinstance(b, dict) and paper_key({"doi": b.get("doi"), "title": b.get("title")}) == target_key:
|
|
1919
|
+
return {f: b.get(f) for f in _BATCH_FIELDS}
|
|
1920
|
+
return None
|
|
1921
|
+
|
|
1922
|
+
|