papersift 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- papersift/__init__.py +8 -0
- papersift/cli.py +365 -0
- papersift/core.py +216 -0
- papersift/download.py +66 -0
- papersift/gui/__init__.py +1 -0
- papersift/gui/app.py +328 -0
- papersift/gui/ollama.py +36 -0
- papersift/keywords.py +40 -0
- papersift/models.py +35 -0
- papersift/providers/__init__.py +22 -0
- papersift/providers/_http.py +64 -0
- papersift/providers/base.py +50 -0
- papersift/providers/fallback.py +29 -0
- papersift/providers/gemini.py +78 -0
- papersift/providers/mock.py +35 -0
- papersift/providers/ollama.py +50 -0
- papersift/providers/openai_compat.py +70 -0
- papersift/providers/registry.py +59 -0
- papersift/ranking/__init__.py +6 -0
- papersift/ranking/abstract.py +21 -0
- papersift/ranking/fulltext.py +127 -0
- papersift/ranking/rubric.py +122 -0
- papersift/sources/__init__.py +19 -0
- papersift/sources/arxiv.py +78 -0
- papersift/store.py +90 -0
- papersift/utils.py +38 -0
- papersift-0.1.0.dist-info/METADATA +249 -0
- papersift-0.1.0.dist-info/RECORD +32 -0
- papersift-0.1.0.dist-info/WHEEL +5 -0
- papersift-0.1.0.dist-info/entry_points.txt +2 -0
- papersift-0.1.0.dist-info/licenses/LICENSE +21 -0
- papersift-0.1.0.dist-info/top_level.txt +1 -0
papersift/__init__.py
ADDED
papersift/cli.py
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
|
|
2
|
+
"""Command-line interface for PaperSift."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import importlib.util
|
|
8
|
+
import logging
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from papersift import __version__
|
|
14
|
+
from papersift.core import Scout
|
|
15
|
+
from papersift.ranking.fulltext import FulltextError
|
|
16
|
+
from papersift.providers import (
|
|
17
|
+
FallbackProvider,
|
|
18
|
+
LLMProvider,
|
|
19
|
+
ProviderError,
|
|
20
|
+
available_providers,
|
|
21
|
+
create_provider,
|
|
22
|
+
)
|
|
23
|
+
from papersift.sources import SourceError
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
27
|
+
formatter = argparse.ArgumentDefaultsHelpFormatter
|
|
28
|
+
|
|
29
|
+
parser = argparse.ArgumentParser(
|
|
30
|
+
prog="papersift",
|
|
31
|
+
description=(
|
|
32
|
+
"PaperSift - an LLM-powered research assistant for "
|
|
33
|
+
"discovering, evaluating, ranking, and organizing academic papers."
|
|
34
|
+
),
|
|
35
|
+
formatter_class=formatter,
|
|
36
|
+
epilog=(
|
|
37
|
+
"Examples:\n"
|
|
38
|
+
" papersift providers\n"
|
|
39
|
+
' papersift run "Autonomous AI agents" --model llama3\n'
|
|
40
|
+
' papersift run "LLM memory mechanisms" --provider gemini '
|
|
41
|
+
'--model gemini-2.0-flash\n'
|
|
42
|
+
" papersift gui\n\n"
|
|
43
|
+
"For command-specific options, use: papersift COMMAND --help"
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"--version",
|
|
49
|
+
action="version",
|
|
50
|
+
version=f"papersift {__version__}",
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
sub = parser.add_subparsers(
|
|
54
|
+
dest="command",
|
|
55
|
+
title="commands",
|
|
56
|
+
metavar="COMMAND",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# --------------------------------------------------
|
|
60
|
+
# Run command
|
|
61
|
+
# --------------------------------------------------
|
|
62
|
+
run = sub.add_parser(
|
|
63
|
+
"run",
|
|
64
|
+
help="search arXiv and rank relevant research papers",
|
|
65
|
+
description=(
|
|
66
|
+
"Search arXiv using keywords generated from your research "
|
|
67
|
+
"project, evaluate paper relevance with an LLM, and save "
|
|
68
|
+
"the results."
|
|
69
|
+
),
|
|
70
|
+
formatter_class=formatter,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
run.add_argument(
|
|
74
|
+
"project_title",
|
|
75
|
+
metavar="PROJECT_TITLE",
|
|
76
|
+
help="short description of your research project",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
run.add_argument(
|
|
80
|
+
"--provider",
|
|
81
|
+
default="ollama",
|
|
82
|
+
help="LLM provider to use (see 'papersift providers')",
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
run.add_argument(
|
|
86
|
+
"--model",
|
|
87
|
+
required=True,
|
|
88
|
+
metavar="MODEL",
|
|
89
|
+
help="model name for the selected provider",
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
run.add_argument(
|
|
93
|
+
"--base-url",
|
|
94
|
+
metavar="URL",
|
|
95
|
+
help="custom API base URL (required for openai_compat)",
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
run.add_argument(
|
|
99
|
+
"--fallback",
|
|
100
|
+
action="append",
|
|
101
|
+
default=[],
|
|
102
|
+
metavar="PROVIDER=MODEL",
|
|
103
|
+
help=(
|
|
104
|
+
"backup provider if the primary provider fails; "
|
|
105
|
+
"can be specified multiple times"
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
run.add_argument(
|
|
110
|
+
"--threshold",
|
|
111
|
+
type=int,
|
|
112
|
+
default=80,
|
|
113
|
+
metavar="SCORE",
|
|
114
|
+
help="minimum relevance score (0-100) to keep a paper",
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
run.add_argument(
|
|
118
|
+
"--since-days",
|
|
119
|
+
type=int,
|
|
120
|
+
default=365,
|
|
121
|
+
metavar="DAYS",
|
|
122
|
+
help="only consider papers published within the last N days",
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
run.add_argument(
|
|
126
|
+
"--max-results",
|
|
127
|
+
type=int,
|
|
128
|
+
default=100,
|
|
129
|
+
metavar="N",
|
|
130
|
+
help="maximum number of arXiv results to consider",
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
run.add_argument(
|
|
134
|
+
"--keywords",
|
|
135
|
+
type=int,
|
|
136
|
+
default=6,
|
|
137
|
+
metavar="N",
|
|
138
|
+
help="number of search keywords to generate",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
run.add_argument(
|
|
142
|
+
"--output",
|
|
143
|
+
default="papersift_output",
|
|
144
|
+
metavar="DIR",
|
|
145
|
+
help="directory where research results will be saved",
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
run.add_argument(
|
|
149
|
+
"--fulltext",
|
|
150
|
+
action="store_true",
|
|
151
|
+
help="also evaluate the full text of selected papers",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
run.add_argument(
|
|
155
|
+
"--triage-threshold",
|
|
156
|
+
type=int,
|
|
157
|
+
default=60,
|
|
158
|
+
metavar="SCORE",
|
|
159
|
+
help="minimum abstract score required for full-text evaluation",
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
run.add_argument(
|
|
163
|
+
"--top-k",
|
|
164
|
+
type=int,
|
|
165
|
+
default=10,
|
|
166
|
+
metavar="N",
|
|
167
|
+
help="number of papers to evaluate using their full text",
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
run.add_argument(
|
|
171
|
+
"--fulltext-weight",
|
|
172
|
+
type=float,
|
|
173
|
+
default=0.6,
|
|
174
|
+
metavar="WEIGHT",
|
|
175
|
+
help="weight assigned to the full-text score (0-1)",
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
run.add_argument(
|
|
179
|
+
"--no-cache",
|
|
180
|
+
action="store_true",
|
|
181
|
+
help="ignore previously cached paper scores",
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
run.add_argument(
|
|
185
|
+
"-v",
|
|
186
|
+
"--verbose",
|
|
187
|
+
action="store_true",
|
|
188
|
+
help="display detailed execution progress",
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# --------------------------------------------------
|
|
192
|
+
# Providers command
|
|
193
|
+
# --------------------------------------------------
|
|
194
|
+
sub.add_parser(
|
|
195
|
+
"providers",
|
|
196
|
+
help="list available LLM providers",
|
|
197
|
+
description="Display all LLM providers supported by PaperSift.",
|
|
198
|
+
formatter_class=formatter,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# --------------------------------------------------
|
|
202
|
+
# GUI command
|
|
203
|
+
# --------------------------------------------------
|
|
204
|
+
sub.add_parser(
|
|
205
|
+
"gui",
|
|
206
|
+
help="launch the graphical user interface",
|
|
207
|
+
description=(
|
|
208
|
+
"Launch the PaperSift graphical interface using Streamlit. "
|
|
209
|
+
"Requires the optional GUI dependencies."
|
|
210
|
+
),
|
|
211
|
+
formatter_class=formatter,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
return parser
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def build_provider(args: argparse.Namespace) -> LLMProvider:
|
|
218
|
+
main = create_provider(
|
|
219
|
+
args.provider,
|
|
220
|
+
args.model,
|
|
221
|
+
base_url=args.base_url,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if not args.fallback:
|
|
225
|
+
return main
|
|
226
|
+
|
|
227
|
+
chain = [main]
|
|
228
|
+
|
|
229
|
+
for spec in args.fallback:
|
|
230
|
+
name, sep, model = spec.partition("=")
|
|
231
|
+
|
|
232
|
+
if not sep or not name or not model:
|
|
233
|
+
raise ProviderError(
|
|
234
|
+
f"Invalid --fallback '{spec}'. "
|
|
235
|
+
"Use PROVIDER=MODEL, e.g. gemini=gemini-2.0-flash"
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
chain.append(create_provider(name, model))
|
|
239
|
+
|
|
240
|
+
return FallbackProvider(chain)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _run(args: argparse.Namespace) -> int:
|
|
244
|
+
logging.basicConfig(
|
|
245
|
+
level=logging.INFO if args.verbose else logging.WARNING,
|
|
246
|
+
format="%(levelname)s: %(message)s",
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
provider = build_provider(args)
|
|
250
|
+
|
|
251
|
+
scout = Scout(
|
|
252
|
+
args.project_title,
|
|
253
|
+
provider,
|
|
254
|
+
relevance_threshold=args.threshold,
|
|
255
|
+
since_days=args.since_days,
|
|
256
|
+
max_results=args.max_results,
|
|
257
|
+
n_keywords=args.keywords,
|
|
258
|
+
output_dir=args.output,
|
|
259
|
+
use_cache=not args.no_cache,
|
|
260
|
+
fulltext=args.fulltext,
|
|
261
|
+
triage_threshold=args.triage_threshold,
|
|
262
|
+
top_k_fulltext=args.top_k,
|
|
263
|
+
fulltext_weight=args.fulltext_weight,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
results = scout.run()
|
|
267
|
+
|
|
268
|
+
if not results:
|
|
269
|
+
print(
|
|
270
|
+
f"No paper reached the threshold of {args.threshold}. "
|
|
271
|
+
"Try lowering --threshold."
|
|
272
|
+
)
|
|
273
|
+
return 0
|
|
274
|
+
|
|
275
|
+
print(f"\n{len(results)} relevant paper(s) saved in {args.output}/\n")
|
|
276
|
+
|
|
277
|
+
for result in results:
|
|
278
|
+
print(
|
|
279
|
+
f" {result.score:>3} "
|
|
280
|
+
f"{result.paper.year} "
|
|
281
|
+
f"{result.paper.title}"
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
return 0
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _launch_gui() -> int:
|
|
288
|
+
"""Launch the Streamlit GUI."""
|
|
289
|
+
|
|
290
|
+
if importlib.util.find_spec("streamlit") is None:
|
|
291
|
+
print(
|
|
292
|
+
"Error: Streamlit is not installed.\n"
|
|
293
|
+
'Install the GUI dependencies with: pip install "papersift[gui]"',
|
|
294
|
+
file=sys.stderr,
|
|
295
|
+
)
|
|
296
|
+
return 1
|
|
297
|
+
|
|
298
|
+
app_path = Path(__file__).resolve().parent / "gui" / "app.py"
|
|
299
|
+
|
|
300
|
+
if not app_path.is_file():
|
|
301
|
+
print(
|
|
302
|
+
f"Error: PaperSift GUI application was not found: {app_path}",
|
|
303
|
+
file=sys.stderr,
|
|
304
|
+
)
|
|
305
|
+
return 1
|
|
306
|
+
|
|
307
|
+
print("Launching PaperSift GUI...")
|
|
308
|
+
|
|
309
|
+
try:
|
|
310
|
+
return subprocess.call(
|
|
311
|
+
[
|
|
312
|
+
sys.executable,
|
|
313
|
+
"-m",
|
|
314
|
+
"streamlit",
|
|
315
|
+
"run",
|
|
316
|
+
str(app_path),
|
|
317
|
+
]
|
|
318
|
+
)
|
|
319
|
+
except OSError as exc:
|
|
320
|
+
print(f"Error launching GUI: {exc}", file=sys.stderr)
|
|
321
|
+
return 1
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def main(argv: list[str] | None = None) -> int:
|
|
325
|
+
try:
|
|
326
|
+
from dotenv import load_dotenv
|
|
327
|
+
|
|
328
|
+
load_dotenv()
|
|
329
|
+
except ImportError: # pragma: no cover
|
|
330
|
+
pass
|
|
331
|
+
|
|
332
|
+
parser = build_parser()
|
|
333
|
+
args = parser.parse_args(argv)
|
|
334
|
+
|
|
335
|
+
if args.command == "providers":
|
|
336
|
+
print("\n".join(available_providers()))
|
|
337
|
+
return 0
|
|
338
|
+
|
|
339
|
+
if args.command == "gui":
|
|
340
|
+
return _launch_gui()
|
|
341
|
+
|
|
342
|
+
if args.command == "run":
|
|
343
|
+
try:
|
|
344
|
+
return _run(args)
|
|
345
|
+
except (
|
|
346
|
+
ProviderError,
|
|
347
|
+
SourceError,
|
|
348
|
+
FulltextError,
|
|
349
|
+
ValueError,
|
|
350
|
+
) as exc:
|
|
351
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
352
|
+
return 1
|
|
353
|
+
except KeyboardInterrupt:
|
|
354
|
+
print(
|
|
355
|
+
"\nInterrupted. Scores already computed are cached.",
|
|
356
|
+
file=sys.stderr,
|
|
357
|
+
)
|
|
358
|
+
return 130
|
|
359
|
+
|
|
360
|
+
parser.print_help()
|
|
361
|
+
return 0
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
if __name__ == "__main__":
|
|
365
|
+
sys.exit(main())
|
papersift/core.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Scout: the pipeline that ties providers, sources, ranking and storage together."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import shutil
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from papersift.download import DownloadError, download_pdf, pdf_filename, safe_id, write_metadata
|
|
11
|
+
from papersift.keywords import generate_keywords
|
|
12
|
+
from papersift.models import Paper
|
|
13
|
+
from papersift.providers import LLMProvider, ProviderError
|
|
14
|
+
from papersift.ranking import Assessment, AssessmentError, assess_abstract
|
|
15
|
+
from papersift.ranking.fulltext import (
|
|
16
|
+
FulltextError,
|
|
17
|
+
assess_fulltext,
|
|
18
|
+
combine_scores,
|
|
19
|
+
extract_text,
|
|
20
|
+
require_pymupdf,
|
|
21
|
+
select_sections,
|
|
22
|
+
)
|
|
23
|
+
from papersift.sources import PaperSource
|
|
24
|
+
from papersift.sources.arxiv import ArxivSource
|
|
25
|
+
from papersift.store import Store
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger("papersift")
|
|
28
|
+
|
|
29
|
+
MAX_CONSECUTIVE_PROVIDER_ERRORS = 5
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class ScoutResult:
|
|
34
|
+
paper: Paper
|
|
35
|
+
score: int
|
|
36
|
+
abstract_assessment: Assessment
|
|
37
|
+
fulltext_assessment: Assessment | None = None
|
|
38
|
+
pdf_path: Path | None = None
|
|
39
|
+
metadata_path: Path | None = None
|
|
40
|
+
from_cache: bool = False
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Scout:
|
|
44
|
+
"""Find, score and download the papers most relevant to a project.
|
|
45
|
+
|
|
46
|
+
Typical use::
|
|
47
|
+
|
|
48
|
+
scout = Scout("my project", provider)
|
|
49
|
+
results = scout.run()
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
project_title: str,
|
|
55
|
+
provider: LLMProvider,
|
|
56
|
+
*,
|
|
57
|
+
relevance_threshold: int = 80,
|
|
58
|
+
since_days: int = 365,
|
|
59
|
+
max_results: int = 100,
|
|
60
|
+
n_keywords: int = 6,
|
|
61
|
+
output_dir: str | Path = "papersift_output",
|
|
62
|
+
source: PaperSource | None = None,
|
|
63
|
+
use_cache: bool = True,
|
|
64
|
+
fulltext: bool = False,
|
|
65
|
+
triage_threshold: int = 60,
|
|
66
|
+
top_k_fulltext: int = 10,
|
|
67
|
+
fulltext_weight: float = 0.6,
|
|
68
|
+
max_fulltext_chars: int = 12000,
|
|
69
|
+
):
|
|
70
|
+
self.project_title = project_title
|
|
71
|
+
self.provider = provider
|
|
72
|
+
self.relevance_threshold = relevance_threshold
|
|
73
|
+
self.since_days = since_days
|
|
74
|
+
self.max_results = max_results
|
|
75
|
+
self.n_keywords = n_keywords
|
|
76
|
+
self.output_dir = Path(output_dir)
|
|
77
|
+
self.source = source or ArxivSource()
|
|
78
|
+
self.use_cache = use_cache
|
|
79
|
+
self.fulltext = fulltext
|
|
80
|
+
self.triage_threshold = triage_threshold
|
|
81
|
+
self.top_k_fulltext = top_k_fulltext
|
|
82
|
+
self.fulltext_weight = fulltext_weight
|
|
83
|
+
self.max_fulltext_chars = max_fulltext_chars
|
|
84
|
+
if fulltext:
|
|
85
|
+
require_pymupdf() # fail early, before spending any LLM calls
|
|
86
|
+
self.pdfs_dir = self.output_dir / "pdfs"
|
|
87
|
+
self.metadata_dir = self.output_dir / "metadata"
|
|
88
|
+
self.cache_dir = self.output_dir / ".cache"
|
|
89
|
+
|
|
90
|
+
# ------------------------------------------------------------------ run
|
|
91
|
+
def run(self) -> list[ScoutResult]:
|
|
92
|
+
with Store(self.cache_dir / "papersift.sqlite3", self.project_title) as store:
|
|
93
|
+
keywords = generate_keywords(self.provider, self.project_title, self.n_keywords)
|
|
94
|
+
logger.info("Search keywords: %s", ", ".join(keywords))
|
|
95
|
+
papers = self.source.search(keywords, since_days=self.since_days, max_results=self.max_results)
|
|
96
|
+
logger.info("Found %d candidate papers", len(papers))
|
|
97
|
+
|
|
98
|
+
scored = self._stage_one(papers, store)
|
|
99
|
+
selected = self._select(scored, store)
|
|
100
|
+
|
|
101
|
+
results = [self._finalize(paper, score, assessments, store) for paper, score, assessments in selected]
|
|
102
|
+
results.sort(key=lambda r: r.score, reverse=True)
|
|
103
|
+
logger.info("Kept %d papers (threshold %d)", len(results), self.relevance_threshold)
|
|
104
|
+
return results
|
|
105
|
+
|
|
106
|
+
# -------------------------------------------------------------- stage 1
|
|
107
|
+
def _stage_one(self, papers: list[Paper], store: Store) -> list[tuple[Paper, Assessment, bool]]:
|
|
108
|
+
scored: list[tuple[Paper, Assessment, bool]] = []
|
|
109
|
+
consecutive_errors = 0
|
|
110
|
+
for index, paper in enumerate(papers, start=1):
|
|
111
|
+
cached = store.get(paper.arxiv_id) if self.use_cache else None
|
|
112
|
+
if cached and cached["abstract_assessment"]:
|
|
113
|
+
scored.append((paper, Assessment.from_dict(cached["abstract_assessment"]), True))
|
|
114
|
+
continue
|
|
115
|
+
try:
|
|
116
|
+
assessment = assess_abstract(self.provider, self.project_title, paper)
|
|
117
|
+
consecutive_errors = 0
|
|
118
|
+
except AssessmentError as exc:
|
|
119
|
+
logger.warning("Skipping '%s': %s", paper.title, exc)
|
|
120
|
+
continue
|
|
121
|
+
except ProviderError as exc:
|
|
122
|
+
consecutive_errors += 1
|
|
123
|
+
logger.warning("Provider error on '%s': %s", paper.title, exc)
|
|
124
|
+
if consecutive_errors >= MAX_CONSECUTIVE_PROVIDER_ERRORS:
|
|
125
|
+
raise ProviderError(
|
|
126
|
+
f"Stopping: {consecutive_errors} provider errors in a row. Last error: {exc}"
|
|
127
|
+
) from exc
|
|
128
|
+
continue
|
|
129
|
+
store.upsert(
|
|
130
|
+
paper.arxiv_id,
|
|
131
|
+
title=paper.title,
|
|
132
|
+
published=paper.published.isoformat(),
|
|
133
|
+
abstract_assessment=assessment.to_dict(),
|
|
134
|
+
)
|
|
135
|
+
logger.info("[%d/%d] %d %s", index, len(papers), assessment.score, paper.title)
|
|
136
|
+
scored.append((paper, assessment, False))
|
|
137
|
+
return scored
|
|
138
|
+
|
|
139
|
+
# ------------------------------------------------------------ selection
|
|
140
|
+
def _select(self, scored, store: Store) -> list[tuple[Paper, int, dict]]:
|
|
141
|
+
"""Decide which papers to keep. Returns (paper, final_score, assessments)."""
|
|
142
|
+
if not self.fulltext:
|
|
143
|
+
return [
|
|
144
|
+
(paper, assessment.score, {"abstract": assessment.to_dict()})
|
|
145
|
+
for paper, assessment, _ in scored
|
|
146
|
+
if assessment.score >= self.relevance_threshold
|
|
147
|
+
]
|
|
148
|
+
# Stage 2: only the best abstract scores are read in full.
|
|
149
|
+
candidates = sorted(
|
|
150
|
+
(item for item in scored if item[1].score >= self.triage_threshold),
|
|
151
|
+
key=lambda item: item[1].score,
|
|
152
|
+
reverse=True,
|
|
153
|
+
)[: self.top_k_fulltext]
|
|
154
|
+
logger.info("Reading the full text of %d paper(s)", len(candidates))
|
|
155
|
+
selected = []
|
|
156
|
+
for paper, abstract, _ in candidates:
|
|
157
|
+
assessments = {"abstract": abstract.to_dict()}
|
|
158
|
+
final = abstract.score
|
|
159
|
+
fulltext = self._stage_two(paper, store)
|
|
160
|
+
if fulltext is not None:
|
|
161
|
+
assessments["fulltext"] = fulltext.to_dict()
|
|
162
|
+
final = combine_scores(abstract.score, fulltext.score, self.fulltext_weight)
|
|
163
|
+
logger.info("Full text %d, combined %d: %s", fulltext.score, final, paper.title)
|
|
164
|
+
if final >= self.relevance_threshold:
|
|
165
|
+
selected.append((paper, final, assessments))
|
|
166
|
+
return selected
|
|
167
|
+
|
|
168
|
+
# -------------------------------------------------------------- stage 2
|
|
169
|
+
def _stage_two(self, paper: Paper, store: Store) -> Assessment | None:
|
|
170
|
+
"""Score the paper from its full text; None if that is not possible."""
|
|
171
|
+
cached = store.get(paper.arxiv_id) if self.use_cache else None
|
|
172
|
+
if cached and cached["fulltext_assessment"]:
|
|
173
|
+
return Assessment.from_dict(cached["fulltext_assessment"])
|
|
174
|
+
pdf = self.cache_dir / "pdfs" / f"{safe_id(paper.arxiv_id)}.pdf"
|
|
175
|
+
try:
|
|
176
|
+
if not pdf.exists():
|
|
177
|
+
download_pdf(paper, pdf)
|
|
178
|
+
excerpts = select_sections(extract_text(pdf), self.max_fulltext_chars)
|
|
179
|
+
assessment = assess_fulltext(self.provider, self.project_title, paper, excerpts)
|
|
180
|
+
except (DownloadError, FulltextError, AssessmentError, ProviderError) as exc:
|
|
181
|
+
logger.warning("Full-text scoring failed for '%s' (%s); using the abstract score", paper.title, exc)
|
|
182
|
+
return None
|
|
183
|
+
store.upsert(paper.arxiv_id, fulltext_assessment=assessment.to_dict())
|
|
184
|
+
return assessment
|
|
185
|
+
|
|
186
|
+
# ------------------------------------------------------------- finalize
|
|
187
|
+
def _finalize(self, paper: Paper, score: int, assessments: dict, store: Store) -> ScoutResult:
|
|
188
|
+
pdf_path = self._place_pdf(paper, score)
|
|
189
|
+
metadata_path = write_metadata(paper, assessments, pdf_path, self.metadata_dir, score)
|
|
190
|
+
store.upsert(paper.arxiv_id, final_score=score, pdf_path=str(pdf_path) if pdf_path else None)
|
|
191
|
+
logger.info("Kept (%d): %s", score, paper.title)
|
|
192
|
+
return ScoutResult(
|
|
193
|
+
paper=paper,
|
|
194
|
+
score=score,
|
|
195
|
+
abstract_assessment=Assessment.from_dict(assessments["abstract"]),
|
|
196
|
+
fulltext_assessment=(
|
|
197
|
+
Assessment.from_dict(assessments["fulltext"]) if "fulltext" in assessments else None
|
|
198
|
+
),
|
|
199
|
+
pdf_path=pdf_path,
|
|
200
|
+
metadata_path=metadata_path,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def _place_pdf(self, paper: Paper, score: int) -> Path | None:
|
|
204
|
+
"""Put the PDF at its final path, reusing a cached download when there is one."""
|
|
205
|
+
final = self.pdfs_dir / pdf_filename(paper, score)
|
|
206
|
+
cached = self.cache_dir / "pdfs" / f"{safe_id(paper.arxiv_id)}.pdf"
|
|
207
|
+
try:
|
|
208
|
+
if cached.exists():
|
|
209
|
+
final.parent.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
shutil.move(str(cached), str(final))
|
|
211
|
+
elif not final.exists():
|
|
212
|
+
download_pdf(paper, final)
|
|
213
|
+
except (DownloadError, OSError) as exc:
|
|
214
|
+
logger.warning("PDF not saved for '%s': %s", paper.title, exc)
|
|
215
|
+
return None
|
|
216
|
+
return final
|
papersift/download.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Download PDFs and write the metadata JSON next to them."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from papersift.models import Paper
|
|
12
|
+
from papersift.utils import sanitize_filename
|
|
13
|
+
|
|
14
|
+
USER_AGENT = "papersift (https://github.com/RAHAMNIabdelkaderseifelislem/papersift)"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DownloadError(Exception):
|
|
18
|
+
"""A PDF could not be downloaded."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def safe_id(arxiv_id: str) -> str:
|
|
22
|
+
"""Old-style arXiv ids contain a slash (hep-th/9901001); make them file-safe."""
|
|
23
|
+
return arxiv_id.replace("/", "_")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def pdf_filename(paper: Paper, score: int) -> str:
|
|
27
|
+
return f"{score}-{paper.year}-{sanitize_filename(paper.title)}.pdf"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def download_pdf(paper: Paper, dest: Path, session=None, timeout: float = 60.0) -> Path:
|
|
31
|
+
"""Download ``paper``'s PDF to ``dest`` (written atomically)."""
|
|
32
|
+
dest = Path(dest)
|
|
33
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
session = session or requests.Session()
|
|
35
|
+
tmp = dest.with_suffix(dest.suffix + ".part")
|
|
36
|
+
try:
|
|
37
|
+
response = session.get(
|
|
38
|
+
paper.pdf_url, timeout=timeout, headers={"User-Agent": USER_AGENT}, stream=True
|
|
39
|
+
)
|
|
40
|
+
if response.status_code != 200:
|
|
41
|
+
raise DownloadError(f"HTTP {response.status_code} for {paper.pdf_url}")
|
|
42
|
+
with open(tmp, "wb") as handle:
|
|
43
|
+
for chunk in response.iter_content(chunk_size=65536):
|
|
44
|
+
handle.write(chunk)
|
|
45
|
+
os.replace(tmp, dest)
|
|
46
|
+
except requests.RequestException as exc:
|
|
47
|
+
raise DownloadError(f"Could not download {paper.pdf_url}: {exc}") from exc
|
|
48
|
+
finally:
|
|
49
|
+
if tmp.exists():
|
|
50
|
+
tmp.unlink()
|
|
51
|
+
return dest
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def write_metadata(paper: Paper, assessments: dict, pdf_path: Path | None, metadata_dir: Path, score: int) -> Path:
|
|
55
|
+
"""Write ``<score>-<year>-<title>_metadata.json`` and return its path."""
|
|
56
|
+
metadata_dir = Path(metadata_dir)
|
|
57
|
+
metadata_dir.mkdir(parents=True, exist_ok=True)
|
|
58
|
+
data = {
|
|
59
|
+
**paper.to_dict(),
|
|
60
|
+
"final_score": score,
|
|
61
|
+
"local_pdf_path": str(pdf_path) if pdf_path else None,
|
|
62
|
+
"assessments": assessments,
|
|
63
|
+
}
|
|
64
|
+
path = metadata_dir / f"{score}-{paper.year}-{sanitize_filename(paper.title)}_metadata.json"
|
|
65
|
+
path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
66
|
+
return path
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional GUI (install with: pip install "papersift[gui]")"""
|