papersift 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
papersift/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """papersift: find and rank research papers with LLMs."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from papersift.core import Scout, ScoutResult # noqa: E402
6
+ from papersift.models import Paper # noqa: E402
7
+
8
+ __all__ = ["Paper", "Scout", "ScoutResult", "__version__"]
papersift/cli.py ADDED
@@ -0,0 +1,365 @@
1
+
2
+ """Command-line interface for PaperSift."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import importlib.util
8
+ import logging
9
+ import subprocess
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ from papersift import __version__
14
+ from papersift.core import Scout
15
+ from papersift.ranking.fulltext import FulltextError
16
+ from papersift.providers import (
17
+ FallbackProvider,
18
+ LLMProvider,
19
+ ProviderError,
20
+ available_providers,
21
+ create_provider,
22
+ )
23
+ from papersift.sources import SourceError
24
+
25
+
26
+ def build_parser() -> argparse.ArgumentParser:
27
+ formatter = argparse.ArgumentDefaultsHelpFormatter
28
+
29
+ parser = argparse.ArgumentParser(
30
+ prog="papersift",
31
+ description=(
32
+ "PaperSift - an LLM-powered research assistant for "
33
+ "discovering, evaluating, ranking, and organizing academic papers."
34
+ ),
35
+ formatter_class=formatter,
36
+ epilog=(
37
+ "Examples:\n"
38
+ " papersift providers\n"
39
+ ' papersift run "Autonomous AI agents" --model llama3\n'
40
+ ' papersift run "LLM memory mechanisms" --provider gemini '
41
+ '--model gemini-2.0-flash\n'
42
+ " papersift gui\n\n"
43
+ "For command-specific options, use: papersift COMMAND --help"
44
+ ),
45
+ )
46
+
47
+ parser.add_argument(
48
+ "--version",
49
+ action="version",
50
+ version=f"papersift {__version__}",
51
+ )
52
+
53
+ sub = parser.add_subparsers(
54
+ dest="command",
55
+ title="commands",
56
+ metavar="COMMAND",
57
+ )
58
+
59
+ # --------------------------------------------------
60
+ # Run command
61
+ # --------------------------------------------------
62
+ run = sub.add_parser(
63
+ "run",
64
+ help="search arXiv and rank relevant research papers",
65
+ description=(
66
+ "Search arXiv using keywords generated from your research "
67
+ "project, evaluate paper relevance with an LLM, and save "
68
+ "the results."
69
+ ),
70
+ formatter_class=formatter,
71
+ )
72
+
73
+ run.add_argument(
74
+ "project_title",
75
+ metavar="PROJECT_TITLE",
76
+ help="short description of your research project",
77
+ )
78
+
79
+ run.add_argument(
80
+ "--provider",
81
+ default="ollama",
82
+ help="LLM provider to use (see 'papersift providers')",
83
+ )
84
+
85
+ run.add_argument(
86
+ "--model",
87
+ required=True,
88
+ metavar="MODEL",
89
+ help="model name for the selected provider",
90
+ )
91
+
92
+ run.add_argument(
93
+ "--base-url",
94
+ metavar="URL",
95
+ help="custom API base URL (required for openai_compat)",
96
+ )
97
+
98
+ run.add_argument(
99
+ "--fallback",
100
+ action="append",
101
+ default=[],
102
+ metavar="PROVIDER=MODEL",
103
+ help=(
104
+ "backup provider if the primary provider fails; "
105
+ "can be specified multiple times"
106
+ ),
107
+ )
108
+
109
+ run.add_argument(
110
+ "--threshold",
111
+ type=int,
112
+ default=80,
113
+ metavar="SCORE",
114
+ help="minimum relevance score (0-100) to keep a paper",
115
+ )
116
+
117
+ run.add_argument(
118
+ "--since-days",
119
+ type=int,
120
+ default=365,
121
+ metavar="DAYS",
122
+ help="only consider papers published within the last N days",
123
+ )
124
+
125
+ run.add_argument(
126
+ "--max-results",
127
+ type=int,
128
+ default=100,
129
+ metavar="N",
130
+ help="maximum number of arXiv results to consider",
131
+ )
132
+
133
+ run.add_argument(
134
+ "--keywords",
135
+ type=int,
136
+ default=6,
137
+ metavar="N",
138
+ help="number of search keywords to generate",
139
+ )
140
+
141
+ run.add_argument(
142
+ "--output",
143
+ default="papersift_output",
144
+ metavar="DIR",
145
+ help="directory where research results will be saved",
146
+ )
147
+
148
+ run.add_argument(
149
+ "--fulltext",
150
+ action="store_true",
151
+ help="also evaluate the full text of selected papers",
152
+ )
153
+
154
+ run.add_argument(
155
+ "--triage-threshold",
156
+ type=int,
157
+ default=60,
158
+ metavar="SCORE",
159
+ help="minimum abstract score required for full-text evaluation",
160
+ )
161
+
162
+ run.add_argument(
163
+ "--top-k",
164
+ type=int,
165
+ default=10,
166
+ metavar="N",
167
+ help="number of papers to evaluate using their full text",
168
+ )
169
+
170
+ run.add_argument(
171
+ "--fulltext-weight",
172
+ type=float,
173
+ default=0.6,
174
+ metavar="WEIGHT",
175
+ help="weight assigned to the full-text score (0-1)",
176
+ )
177
+
178
+ run.add_argument(
179
+ "--no-cache",
180
+ action="store_true",
181
+ help="ignore previously cached paper scores",
182
+ )
183
+
184
+ run.add_argument(
185
+ "-v",
186
+ "--verbose",
187
+ action="store_true",
188
+ help="display detailed execution progress",
189
+ )
190
+
191
+ # --------------------------------------------------
192
+ # Providers command
193
+ # --------------------------------------------------
194
+ sub.add_parser(
195
+ "providers",
196
+ help="list available LLM providers",
197
+ description="Display all LLM providers supported by PaperSift.",
198
+ formatter_class=formatter,
199
+ )
200
+
201
+ # --------------------------------------------------
202
+ # GUI command
203
+ # --------------------------------------------------
204
+ sub.add_parser(
205
+ "gui",
206
+ help="launch the graphical user interface",
207
+ description=(
208
+ "Launch the PaperSift graphical interface using Streamlit. "
209
+ "Requires the optional GUI dependencies."
210
+ ),
211
+ formatter_class=formatter,
212
+ )
213
+
214
+ return parser
215
+
216
+
217
+ def build_provider(args: argparse.Namespace) -> LLMProvider:
218
+ main = create_provider(
219
+ args.provider,
220
+ args.model,
221
+ base_url=args.base_url,
222
+ )
223
+
224
+ if not args.fallback:
225
+ return main
226
+
227
+ chain = [main]
228
+
229
+ for spec in args.fallback:
230
+ name, sep, model = spec.partition("=")
231
+
232
+ if not sep or not name or not model:
233
+ raise ProviderError(
234
+ f"Invalid --fallback '{spec}'. "
235
+ "Use PROVIDER=MODEL, e.g. gemini=gemini-2.0-flash"
236
+ )
237
+
238
+ chain.append(create_provider(name, model))
239
+
240
+ return FallbackProvider(chain)
241
+
242
+
243
+ def _run(args: argparse.Namespace) -> int:
244
+ logging.basicConfig(
245
+ level=logging.INFO if args.verbose else logging.WARNING,
246
+ format="%(levelname)s: %(message)s",
247
+ )
248
+
249
+ provider = build_provider(args)
250
+
251
+ scout = Scout(
252
+ args.project_title,
253
+ provider,
254
+ relevance_threshold=args.threshold,
255
+ since_days=args.since_days,
256
+ max_results=args.max_results,
257
+ n_keywords=args.keywords,
258
+ output_dir=args.output,
259
+ use_cache=not args.no_cache,
260
+ fulltext=args.fulltext,
261
+ triage_threshold=args.triage_threshold,
262
+ top_k_fulltext=args.top_k,
263
+ fulltext_weight=args.fulltext_weight,
264
+ )
265
+
266
+ results = scout.run()
267
+
268
+ if not results:
269
+ print(
270
+ f"No paper reached the threshold of {args.threshold}. "
271
+ "Try lowering --threshold."
272
+ )
273
+ return 0
274
+
275
+ print(f"\n{len(results)} relevant paper(s) saved in {args.output}/\n")
276
+
277
+ for result in results:
278
+ print(
279
+ f" {result.score:>3} "
280
+ f"{result.paper.year} "
281
+ f"{result.paper.title}"
282
+ )
283
+
284
+ return 0
285
+
286
+
287
+ def _launch_gui() -> int:
288
+ """Launch the Streamlit GUI."""
289
+
290
+ if importlib.util.find_spec("streamlit") is None:
291
+ print(
292
+ "Error: Streamlit is not installed.\n"
293
+ 'Install the GUI dependencies with: pip install "papersift[gui]"',
294
+ file=sys.stderr,
295
+ )
296
+ return 1
297
+
298
+ app_path = Path(__file__).resolve().parent / "gui" / "app.py"
299
+
300
+ if not app_path.is_file():
301
+ print(
302
+ f"Error: PaperSift GUI application was not found: {app_path}",
303
+ file=sys.stderr,
304
+ )
305
+ return 1
306
+
307
+ print("Launching PaperSift GUI...")
308
+
309
+ try:
310
+ return subprocess.call(
311
+ [
312
+ sys.executable,
313
+ "-m",
314
+ "streamlit",
315
+ "run",
316
+ str(app_path),
317
+ ]
318
+ )
319
+ except OSError as exc:
320
+ print(f"Error launching GUI: {exc}", file=sys.stderr)
321
+ return 1
322
+
323
+
324
+ def main(argv: list[str] | None = None) -> int:
325
+ try:
326
+ from dotenv import load_dotenv
327
+
328
+ load_dotenv()
329
+ except ImportError: # pragma: no cover
330
+ pass
331
+
332
+ parser = build_parser()
333
+ args = parser.parse_args(argv)
334
+
335
+ if args.command == "providers":
336
+ print("\n".join(available_providers()))
337
+ return 0
338
+
339
+ if args.command == "gui":
340
+ return _launch_gui()
341
+
342
+ if args.command == "run":
343
+ try:
344
+ return _run(args)
345
+ except (
346
+ ProviderError,
347
+ SourceError,
348
+ FulltextError,
349
+ ValueError,
350
+ ) as exc:
351
+ print(f"Error: {exc}", file=sys.stderr)
352
+ return 1
353
+ except KeyboardInterrupt:
354
+ print(
355
+ "\nInterrupted. Scores already computed are cached.",
356
+ file=sys.stderr,
357
+ )
358
+ return 130
359
+
360
+ parser.print_help()
361
+ return 0
362
+
363
+
364
+ if __name__ == "__main__":
365
+ sys.exit(main())
papersift/core.py ADDED
@@ -0,0 +1,216 @@
1
+ """Scout: the pipeline that ties providers, sources, ranking and storage together."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import shutil
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+
10
+ from papersift.download import DownloadError, download_pdf, pdf_filename, safe_id, write_metadata
11
+ from papersift.keywords import generate_keywords
12
+ from papersift.models import Paper
13
+ from papersift.providers import LLMProvider, ProviderError
14
+ from papersift.ranking import Assessment, AssessmentError, assess_abstract
15
+ from papersift.ranking.fulltext import (
16
+ FulltextError,
17
+ assess_fulltext,
18
+ combine_scores,
19
+ extract_text,
20
+ require_pymupdf,
21
+ select_sections,
22
+ )
23
+ from papersift.sources import PaperSource
24
+ from papersift.sources.arxiv import ArxivSource
25
+ from papersift.store import Store
26
+
27
+ logger = logging.getLogger("papersift")
28
+
29
+ MAX_CONSECUTIVE_PROVIDER_ERRORS = 5
30
+
31
+
32
+ @dataclass
33
+ class ScoutResult:
34
+ paper: Paper
35
+ score: int
36
+ abstract_assessment: Assessment
37
+ fulltext_assessment: Assessment | None = None
38
+ pdf_path: Path | None = None
39
+ metadata_path: Path | None = None
40
+ from_cache: bool = False
41
+
42
+
43
+ class Scout:
44
+ """Find, score and download the papers most relevant to a project.
45
+
46
+ Typical use::
47
+
48
+ scout = Scout("my project", provider)
49
+ results = scout.run()
50
+ """
51
+
52
+ def __init__(
53
+ self,
54
+ project_title: str,
55
+ provider: LLMProvider,
56
+ *,
57
+ relevance_threshold: int = 80,
58
+ since_days: int = 365,
59
+ max_results: int = 100,
60
+ n_keywords: int = 6,
61
+ output_dir: str | Path = "papersift_output",
62
+ source: PaperSource | None = None,
63
+ use_cache: bool = True,
64
+ fulltext: bool = False,
65
+ triage_threshold: int = 60,
66
+ top_k_fulltext: int = 10,
67
+ fulltext_weight: float = 0.6,
68
+ max_fulltext_chars: int = 12000,
69
+ ):
70
+ self.project_title = project_title
71
+ self.provider = provider
72
+ self.relevance_threshold = relevance_threshold
73
+ self.since_days = since_days
74
+ self.max_results = max_results
75
+ self.n_keywords = n_keywords
76
+ self.output_dir = Path(output_dir)
77
+ self.source = source or ArxivSource()
78
+ self.use_cache = use_cache
79
+ self.fulltext = fulltext
80
+ self.triage_threshold = triage_threshold
81
+ self.top_k_fulltext = top_k_fulltext
82
+ self.fulltext_weight = fulltext_weight
83
+ self.max_fulltext_chars = max_fulltext_chars
84
+ if fulltext:
85
+ require_pymupdf() # fail early, before spending any LLM calls
86
+ self.pdfs_dir = self.output_dir / "pdfs"
87
+ self.metadata_dir = self.output_dir / "metadata"
88
+ self.cache_dir = self.output_dir / ".cache"
89
+
90
+ # ------------------------------------------------------------------ run
91
+ def run(self) -> list[ScoutResult]:
92
+ with Store(self.cache_dir / "papersift.sqlite3", self.project_title) as store:
93
+ keywords = generate_keywords(self.provider, self.project_title, self.n_keywords)
94
+ logger.info("Search keywords: %s", ", ".join(keywords))
95
+ papers = self.source.search(keywords, since_days=self.since_days, max_results=self.max_results)
96
+ logger.info("Found %d candidate papers", len(papers))
97
+
98
+ scored = self._stage_one(papers, store)
99
+ selected = self._select(scored, store)
100
+
101
+ results = [self._finalize(paper, score, assessments, store) for paper, score, assessments in selected]
102
+ results.sort(key=lambda r: r.score, reverse=True)
103
+ logger.info("Kept %d papers (threshold %d)", len(results), self.relevance_threshold)
104
+ return results
105
+
106
+ # -------------------------------------------------------------- stage 1
107
+ def _stage_one(self, papers: list[Paper], store: Store) -> list[tuple[Paper, Assessment, bool]]:
108
+ scored: list[tuple[Paper, Assessment, bool]] = []
109
+ consecutive_errors = 0
110
+ for index, paper in enumerate(papers, start=1):
111
+ cached = store.get(paper.arxiv_id) if self.use_cache else None
112
+ if cached and cached["abstract_assessment"]:
113
+ scored.append((paper, Assessment.from_dict(cached["abstract_assessment"]), True))
114
+ continue
115
+ try:
116
+ assessment = assess_abstract(self.provider, self.project_title, paper)
117
+ consecutive_errors = 0
118
+ except AssessmentError as exc:
119
+ logger.warning("Skipping '%s': %s", paper.title, exc)
120
+ continue
121
+ except ProviderError as exc:
122
+ consecutive_errors += 1
123
+ logger.warning("Provider error on '%s': %s", paper.title, exc)
124
+ if consecutive_errors >= MAX_CONSECUTIVE_PROVIDER_ERRORS:
125
+ raise ProviderError(
126
+ f"Stopping: {consecutive_errors} provider errors in a row. Last error: {exc}"
127
+ ) from exc
128
+ continue
129
+ store.upsert(
130
+ paper.arxiv_id,
131
+ title=paper.title,
132
+ published=paper.published.isoformat(),
133
+ abstract_assessment=assessment.to_dict(),
134
+ )
135
+ logger.info("[%d/%d] %d %s", index, len(papers), assessment.score, paper.title)
136
+ scored.append((paper, assessment, False))
137
+ return scored
138
+
139
+ # ------------------------------------------------------------ selection
140
+ def _select(self, scored, store: Store) -> list[tuple[Paper, int, dict]]:
141
+ """Decide which papers to keep. Returns (paper, final_score, assessments)."""
142
+ if not self.fulltext:
143
+ return [
144
+ (paper, assessment.score, {"abstract": assessment.to_dict()})
145
+ for paper, assessment, _ in scored
146
+ if assessment.score >= self.relevance_threshold
147
+ ]
148
+ # Stage 2: only the best abstract scores are read in full.
149
+ candidates = sorted(
150
+ (item for item in scored if item[1].score >= self.triage_threshold),
151
+ key=lambda item: item[1].score,
152
+ reverse=True,
153
+ )[: self.top_k_fulltext]
154
+ logger.info("Reading the full text of %d paper(s)", len(candidates))
155
+ selected = []
156
+ for paper, abstract, _ in candidates:
157
+ assessments = {"abstract": abstract.to_dict()}
158
+ final = abstract.score
159
+ fulltext = self._stage_two(paper, store)
160
+ if fulltext is not None:
161
+ assessments["fulltext"] = fulltext.to_dict()
162
+ final = combine_scores(abstract.score, fulltext.score, self.fulltext_weight)
163
+ logger.info("Full text %d, combined %d: %s", fulltext.score, final, paper.title)
164
+ if final >= self.relevance_threshold:
165
+ selected.append((paper, final, assessments))
166
+ return selected
167
+
168
+ # -------------------------------------------------------------- stage 2
169
+ def _stage_two(self, paper: Paper, store: Store) -> Assessment | None:
170
+ """Score the paper from its full text; None if that is not possible."""
171
+ cached = store.get(paper.arxiv_id) if self.use_cache else None
172
+ if cached and cached["fulltext_assessment"]:
173
+ return Assessment.from_dict(cached["fulltext_assessment"])
174
+ pdf = self.cache_dir / "pdfs" / f"{safe_id(paper.arxiv_id)}.pdf"
175
+ try:
176
+ if not pdf.exists():
177
+ download_pdf(paper, pdf)
178
+ excerpts = select_sections(extract_text(pdf), self.max_fulltext_chars)
179
+ assessment = assess_fulltext(self.provider, self.project_title, paper, excerpts)
180
+ except (DownloadError, FulltextError, AssessmentError, ProviderError) as exc:
181
+ logger.warning("Full-text scoring failed for '%s' (%s); using the abstract score", paper.title, exc)
182
+ return None
183
+ store.upsert(paper.arxiv_id, fulltext_assessment=assessment.to_dict())
184
+ return assessment
185
+
186
+ # ------------------------------------------------------------- finalize
187
+ def _finalize(self, paper: Paper, score: int, assessments: dict, store: Store) -> ScoutResult:
188
+ pdf_path = self._place_pdf(paper, score)
189
+ metadata_path = write_metadata(paper, assessments, pdf_path, self.metadata_dir, score)
190
+ store.upsert(paper.arxiv_id, final_score=score, pdf_path=str(pdf_path) if pdf_path else None)
191
+ logger.info("Kept (%d): %s", score, paper.title)
192
+ return ScoutResult(
193
+ paper=paper,
194
+ score=score,
195
+ abstract_assessment=Assessment.from_dict(assessments["abstract"]),
196
+ fulltext_assessment=(
197
+ Assessment.from_dict(assessments["fulltext"]) if "fulltext" in assessments else None
198
+ ),
199
+ pdf_path=pdf_path,
200
+ metadata_path=metadata_path,
201
+ )
202
+
203
+ def _place_pdf(self, paper: Paper, score: int) -> Path | None:
204
+ """Put the PDF at its final path, reusing a cached download when there is one."""
205
+ final = self.pdfs_dir / pdf_filename(paper, score)
206
+ cached = self.cache_dir / "pdfs" / f"{safe_id(paper.arxiv_id)}.pdf"
207
+ try:
208
+ if cached.exists():
209
+ final.parent.mkdir(parents=True, exist_ok=True)
210
+ shutil.move(str(cached), str(final))
211
+ elif not final.exists():
212
+ download_pdf(paper, final)
213
+ except (DownloadError, OSError) as exc:
214
+ logger.warning("PDF not saved for '%s': %s", paper.title, exc)
215
+ return None
216
+ return final
papersift/download.py ADDED
@@ -0,0 +1,66 @@
1
+ """Download PDFs and write the metadata JSON next to them."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ from pathlib import Path
8
+
9
+ import requests
10
+
11
+ from papersift.models import Paper
12
+ from papersift.utils import sanitize_filename
13
+
14
+ USER_AGENT = "papersift (https://github.com/RAHAMNIabdelkaderseifelislem/papersift)"
15
+
16
+
17
+ class DownloadError(Exception):
18
+ """A PDF could not be downloaded."""
19
+
20
+
21
+ def safe_id(arxiv_id: str) -> str:
22
+ """Old-style arXiv ids contain a slash (hep-th/9901001); make them file-safe."""
23
+ return arxiv_id.replace("/", "_")
24
+
25
+
26
+ def pdf_filename(paper: Paper, score: int) -> str:
27
+ return f"{score}-{paper.year}-{sanitize_filename(paper.title)}.pdf"
28
+
29
+
30
+ def download_pdf(paper: Paper, dest: Path, session=None, timeout: float = 60.0) -> Path:
31
+ """Download ``paper``'s PDF to ``dest`` (written atomically)."""
32
+ dest = Path(dest)
33
+ dest.parent.mkdir(parents=True, exist_ok=True)
34
+ session = session or requests.Session()
35
+ tmp = dest.with_suffix(dest.suffix + ".part")
36
+ try:
37
+ response = session.get(
38
+ paper.pdf_url, timeout=timeout, headers={"User-Agent": USER_AGENT}, stream=True
39
+ )
40
+ if response.status_code != 200:
41
+ raise DownloadError(f"HTTP {response.status_code} for {paper.pdf_url}")
42
+ with open(tmp, "wb") as handle:
43
+ for chunk in response.iter_content(chunk_size=65536):
44
+ handle.write(chunk)
45
+ os.replace(tmp, dest)
46
+ except requests.RequestException as exc:
47
+ raise DownloadError(f"Could not download {paper.pdf_url}: {exc}") from exc
48
+ finally:
49
+ if tmp.exists():
50
+ tmp.unlink()
51
+ return dest
52
+
53
+
54
+ def write_metadata(paper: Paper, assessments: dict, pdf_path: Path | None, metadata_dir: Path, score: int) -> Path:
55
+ """Write ``<score>-<year>-<title>_metadata.json`` and return its path."""
56
+ metadata_dir = Path(metadata_dir)
57
+ metadata_dir.mkdir(parents=True, exist_ok=True)
58
+ data = {
59
+ **paper.to_dict(),
60
+ "final_score": score,
61
+ "local_pdf_path": str(pdf_path) if pdf_path else None,
62
+ "assessments": assessments,
63
+ }
64
+ path = metadata_dir / f"{score}-{paper.year}-{sanitize_filename(paper.title)}_metadata.json"
65
+ path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
66
+ return path
@@ -0,0 +1 @@
1
+ """Optional GUI (install with: pip install "papersift[gui]")"""