papersift 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
papersift/gui/app.py ADDED
@@ -0,0 +1,328 @@
1
+ """Streamlit interface for the PaperSift research pipeline.
2
+
3
+ Run with: streamlit run papersift/gui/app.py
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from pathlib import Path
9
+
10
+ import pandas as pd
11
+ import streamlit as st
12
+
13
+ from papersift.core import Scout, ScoutResult
14
+ from papersift.gui.ollama import OllamaDiscoveryError, list_ollama_models
15
+ from papersift.providers import ProviderError, available_providers, create_provider
16
+ from papersift.ranking import AssessmentError
17
+ from papersift.sources import SourceError
18
+ from papersift.ranking.fulltext import FulltextError
19
+
20
+
21
+ PROVIDER_LABELS = {
22
+ "ollama": "Ollama (local)",
23
+ "gemini": "Google Gemini",
24
+ "groq": "Groq",
25
+ "openrouter": "OpenRouter",
26
+ "together": "Together AI",
27
+ "cerebras": "Cerebras",
28
+ "mistral": "Mistral",
29
+ "openai": "OpenAI",
30
+ "openai_compat": "Custom OpenAI-compatible endpoint",
31
+ }
32
+
33
+
34
+ def _provider_label(provider_name: str) -> str:
35
+ return PROVIDER_LABELS.get(provider_name, provider_name.replace("_", " ").title())
36
+
37
+
38
+ @st.cache_data(ttl=30, show_spinner=False)
39
+ def _cached_ollama_models(host: str) -> list[str]:
40
+ return list_ollama_models(host)
41
+
42
+
43
+ def _result_frame(results: list[ScoutResult]) -> pd.DataFrame:
44
+ rows = []
45
+ for result in results:
46
+ paper = result.paper
47
+ assessment = result.abstract_assessment
48
+ rows.append(
49
+ {
50
+ "Title": paper.title,
51
+ "Year": paper.year,
52
+ "Score": result.score,
53
+ "Recommendation": assessment.recommendation,
54
+ "Reasoning": assessment.rubric.reasoning,
55
+ "Authors": ", ".join(paper.authors),
56
+ "arXiv": paper.url,
57
+ "PDF": paper.pdf_url,
58
+ "_result": result,
59
+ }
60
+ )
61
+ return pd.DataFrame(rows)
62
+
63
+
64
+ def _show_result_details(result: ScoutResult) -> None:
65
+ paper = result.paper
66
+ assessment = result.abstract_assessment
67
+ rubric = assessment.rubric
68
+
69
+ st.markdown(f"#### {paper.title}")
70
+ st.caption(f"{paper.year} · {', '.join(paper.authors) or 'Authors unavailable'} · arXiv:{paper.arxiv_id}")
71
+ st.write(paper.abstract)
72
+
73
+ score_cols = st.columns(4)
74
+ score_cols[0].metric("Final score", f"{result.score}/100")
75
+ score_cols[1].metric("Topic match", f"{rubric.topic_match}/3")
76
+ score_cols[2].metric("Method match", f"{rubric.method_match}/3")
77
+ score_cols[3].metric("Novelty", f"{rubric.novelty}/3")
78
+
79
+ st.markdown(f"**Recommendation:** {assessment.recommendation}")
80
+ if rubric.reasoning:
81
+ st.write(rubric.reasoning)
82
+
83
+ left, right = st.columns(2)
84
+ with left:
85
+ st.markdown("**Key strengths**")
86
+ if rubric.key_strengths:
87
+ for item in rubric.key_strengths:
88
+ st.markdown(f"- {item}")
89
+ else:
90
+ st.caption("No strengths were returned.")
91
+ with right:
92
+ st.markdown("**Potential limitations**")
93
+ if rubric.potential_limitations:
94
+ for item in rubric.potential_limitations:
95
+ st.markdown(f"- {item}")
96
+ else:
97
+ st.caption("No limitations were returned.")
98
+
99
+ if result.fulltext_assessment:
100
+ fulltext = result.fulltext_assessment
101
+ with st.expander("Full-text assessment"):
102
+ st.write(f"Score: **{fulltext.score}/100** · {fulltext.recommendation}")
103
+ st.write(fulltext.rubric.reasoning)
104
+
105
+ links = st.columns(3)
106
+ links[0].link_button("Open arXiv page", paper.url, use_container_width=True)
107
+ if result.pdf_path and Path(result.pdf_path).exists():
108
+ with open(result.pdf_path, "rb") as pdf_file:
109
+ links[1].download_button(
110
+ "Download saved PDF",
111
+ data=pdf_file,
112
+ file_name=Path(result.pdf_path).name,
113
+ mime="application/pdf",
114
+ use_container_width=True,
115
+ )
116
+ else:
117
+ links[1].link_button("Open PDF", paper.pdf_url, use_container_width=True)
118
+ if result.metadata_path and Path(result.metadata_path).exists():
119
+ with open(result.metadata_path, "rb") as metadata_file:
120
+ links[2].download_button(
121
+ "Download metadata",
122
+ data=metadata_file,
123
+ file_name=Path(result.metadata_path).name,
124
+ mime="application/json",
125
+ use_container_width=True,
126
+ )
127
+
128
+
129
+ def main() -> None:
130
+ st.set_page_config(page_title="PaperSift", page_icon="📚", layout="wide")
131
+ st.title("PaperSift")
132
+ st.caption("Find the papers that matter to your research, automatically.")
133
+
134
+ settings_tab, results_tab = st.tabs(["Settings", "Research results"])
135
+
136
+ with settings_tab:
137
+ st.header("Research agent configuration")
138
+ st.write("Configure a research topic, an AI provider, and the search parameters.")
139
+
140
+ project_title = st.text_input(
141
+ "Research project",
142
+ placeholder="e.g. Self-improving generative AI agents",
143
+ )
144
+
145
+ provider_names = available_providers()
146
+ provider_name = st.selectbox(
147
+ "Provider",
148
+ options=provider_names,
149
+ format_func=_provider_label,
150
+ )
151
+ api_key = ""
152
+ base_url = ""
153
+ if provider_name == "ollama":
154
+ base_url = st.text_input(
155
+ "Ollama host",
156
+ value="http://localhost:11434",
157
+ help="The address of the Ollama server.",
158
+ )
159
+ st.caption("Ollama runs locally and does not require an API key.")
160
+
161
+ try:
162
+ installed_models = _cached_ollama_models(base_url.strip() or "http://localhost:11434")
163
+ except OllamaDiscoveryError as exc:
164
+ installed_models = []
165
+ st.warning(f"Could not discover Ollama models: {exc}")
166
+
167
+ if installed_models:
168
+ model_options = [*installed_models, "Enter a model name manually"]
169
+ selected_model = st.selectbox("Installed Ollama model", model_options)
170
+ if selected_model == "Enter a model name manually":
171
+ model_name = st.text_input("Model name", placeholder="e.g. llama3.1:8b")
172
+ else:
173
+ model_name = selected_model
174
+ else:
175
+ st.info("No installed models were detected. Start Ollama and pull a model, or enter its name manually.")
176
+ model_name = st.text_input("Model name", placeholder="e.g. llama3.1:8b")
177
+
178
+ if st.button("Refresh installed models", key="refresh_ollama_models"):
179
+ _cached_ollama_models.clear()
180
+ st.rerun()
181
+ elif provider_name == "openai_compat":
182
+ model_name = st.text_input(
183
+ "Model name",
184
+ placeholder="e.g. provider-model-name",
185
+ )
186
+ base_url = st.text_input(
187
+ "Base URL",
188
+ placeholder="https://your-provider.example/v1",
189
+ )
190
+ api_key = st.text_input("API key", type="password")
191
+ else:
192
+ model_name = st.text_input(
193
+ "Model name",
194
+ placeholder="e.g. gemini-2.5-flash, gpt-4.1-mini",
195
+ )
196
+ api_key = st.text_input("API key", type="password")
197
+
198
+ st.subheader("Search parameters")
199
+ relevance_threshold = st.slider(
200
+ "Minimum relevance score",
201
+ min_value=0,
202
+ max_value=100,
203
+ value=80,
204
+ help="Only papers meeting this final score are included in the results.",
205
+ )
206
+ days_to_search = st.number_input(
207
+ "Search window (days)", min_value=1, max_value=3650, value=365, step=1
208
+ )
209
+ max_results = st.number_input(
210
+ "Maximum candidate papers", min_value=1, max_value=500, value=100, step=10
211
+ )
212
+ fulltext = st.checkbox(
213
+ "Enable full-text assessment",
214
+ value=False,
215
+ help="Downloads and evaluates the full text of the highest-scoring candidates. Requires PyMuPDF.",
216
+ )
217
+ output_dir = st.text_input("Output directory", value="papersift_output")
218
+
219
+ submitted = st.button("Run research agent", type="primary", use_container_width=True)
220
+
221
+ if submitted:
222
+ if not project_title.strip():
223
+ st.error("Enter a research project title.")
224
+ elif not model_name.strip():
225
+ st.error("Enter a model name.")
226
+ elif provider_name != "ollama" and not api_key.strip():
227
+ st.error("Enter the API key for the selected provider.")
228
+ elif provider_name == "openai_compat" and not base_url.strip():
229
+ st.error("Enter the base URL for the custom OpenAI-compatible provider.")
230
+ elif provider_name == "ollama" and not base_url.strip():
231
+ st.error("Enter the Ollama host URL.")
232
+ else:
233
+ try:
234
+ provider = create_provider(
235
+ provider_name,
236
+ model_name.strip(),
237
+ api_key=api_key.strip() or None,
238
+ base_url=base_url.strip() or None,
239
+ )
240
+ scout = Scout(
241
+ project_title=project_title.strip(),
242
+ provider=provider,
243
+ relevance_threshold=int(relevance_threshold),
244
+ since_days=int(days_to_search),
245
+ max_results=int(max_results),
246
+ output_dir=output_dir.strip() or "papersift_output",
247
+ fulltext=fulltext,
248
+ )
249
+ with st.spinner("PaperSift is generating search keywords, searching arXiv, and assessing papers..."):
250
+ results = scout.run()
251
+ st.session_state["papersift_results"] = results
252
+ st.session_state["papersift_project_title"] = project_title.strip()
253
+ st.session_state["papersift_run_config"] = {
254
+ "provider": provider_name,
255
+ "model": model_name.strip(),
256
+ "threshold": int(relevance_threshold),
257
+ "days": int(days_to_search),
258
+ "max_results": int(max_results),
259
+ "fulltext": fulltext,
260
+ "output_dir": output_dir.strip() or "papersift_output",
261
+ }
262
+ st.success(f"Research completed. {len(results)} relevant paper(s) found.")
263
+ st.info("API credentials are used to create the provider and are not included in the saved run configuration.")
264
+ except (ProviderError, SourceError, AssessmentError, FulltextError, ValueError) as exc:
265
+ st.error(f"Research could not be completed: {exc}")
266
+ except Exception as exc: # noqa: BLE001 - keep unexpected runtime failures visible in the UI
267
+ st.exception(exc)
268
+
269
+ with results_tab:
270
+ st.header("Research results")
271
+ results = st.session_state.get("papersift_results", [])
272
+ project_title = st.session_state.get("papersift_project_title")
273
+
274
+ if not results:
275
+ st.info("No research results yet. Configure your project and run the research agent from the Settings tab.")
276
+ else:
277
+ st.caption(f"Project: {project_title}")
278
+ df = _result_frame(results)
279
+
280
+ metric_cols = st.columns(3)
281
+ metric_cols[0].metric("Papers retained", len(results))
282
+ metric_cols[1].metric("Average score", f"{df['Score'].mean():.1f}/100")
283
+ metric_cols[2].metric("Highest score", f"{df['Score'].max()}/100")
284
+
285
+ st.subheader("Filter and sort")
286
+ filter_cols = st.columns([2, 2, 1])
287
+ min_score = filter_cols[0].slider(
288
+ "Minimum score", 0, 100, 0, key="results_min_score"
289
+ )
290
+ sort_by = filter_cols[1].selectbox(
291
+ "Sort by", ["Score", "Year", "Title"], key="results_sort_by"
292
+ )
293
+ descending = filter_cols[2].checkbox(
294
+ "Descending", value=True, key="results_descending"
295
+ )
296
+
297
+ filtered = df[df["Score"] >= min_score].sort_values(
298
+ by=sort_by, ascending=not descending
299
+ )
300
+ st.caption(f"Showing {len(filtered)} of {len(df)} retained papers.")
301
+ display_df = filtered.drop(columns=["_result"])
302
+ st.dataframe(
303
+ display_df,
304
+ hide_index=True,
305
+ use_container_width=True,
306
+ column_config={
307
+ "Title": st.column_config.TextColumn("Paper title", width="large"),
308
+ "Score": st.column_config.ProgressColumn(
309
+ "Relevance score", min_value=0, max_value=100, format="%d/100"
310
+ ),
311
+ "arXiv": st.column_config.LinkColumn("arXiv", display_text="Open paper"),
312
+ "PDF": st.column_config.LinkColumn("PDF / saved path"),
313
+ "Reasoning": st.column_config.TextColumn("Assessment reasoning", width="large"),
314
+ },
315
+ )
316
+
317
+ st.subheader("Paper assessments")
318
+ for _, row in filtered.iterrows():
319
+ result = row["_result"]
320
+ with st.expander(f"{result.score}/100 · {result.paper.title}"):
321
+ _show_result_details(result)
322
+
323
+ st.divider()
324
+ st.caption("PaperSift · Research assistant")
325
+
326
+
327
+ if __name__ == "__main__":
328
+ main()
@@ -0,0 +1,36 @@
1
+ """Helpers for discovering locally installed Ollama models."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import requests
6
+
7
+
8
+ class OllamaDiscoveryError(RuntimeError):
9
+ """Raised when the Ollama model list cannot be retrieved or parsed."""
10
+
11
+
12
+ def list_ollama_models(host: str = "http://localhost:11434", timeout: float = 3.0) -> list[str]:
13
+ """Return every model name reported by an Ollama server's /api/tags endpoint."""
14
+ url = f"{host.rstrip('/')}/api/tags"
15
+ try:
16
+ response = requests.get(url, timeout=timeout)
17
+ response.raise_for_status()
18
+ payload = response.json()
19
+ except requests.RequestException as exc:
20
+ raise OllamaDiscoveryError(f"Could not connect to Ollama at {host}: {exc}") from exc
21
+ except ValueError as exc:
22
+ raise OllamaDiscoveryError("Ollama returned invalid JSON while listing installed models.") from exc
23
+
24
+ models = payload.get("models") if isinstance(payload, dict) else None
25
+ if not isinstance(models, list):
26
+ raise OllamaDiscoveryError("Ollama's /api/tags response did not contain a models list.")
27
+
28
+ names = sorted(
29
+ {
30
+ model["name"].strip()
31
+ for model in models
32
+ if isinstance(model, dict) and isinstance(model.get("name"), str) and model["name"].strip()
33
+ },
34
+ key=str.casefold,
35
+ )
36
+ return names
papersift/keywords.py ADDED
@@ -0,0 +1,40 @@
1
+ """Turn a project description into arXiv search keywords with an LLM."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ from papersift.providers import LLMProvider
8
+ from papersift.utils import extract_json
9
+
10
+
11
+ def parse_keywords(text: str, limit: int = 12) -> list[str]:
12
+ """Accept a JSON list, or a comma/newline separated list, and clean it up."""
13
+ items: list[str]
14
+ try:
15
+ data = extract_json(text)
16
+ if isinstance(data, dict):
17
+ data = next((v for v in data.values() if isinstance(v, list)), [])
18
+ items = [str(x) for x in data] if isinstance(data, list) else []
19
+ except ValueError:
20
+ items = re.split(r"[,\n;]", text)
21
+ cleaned: list[str] = []
22
+ seen: set[str] = set()
23
+ for item in items:
24
+ keyword = re.sub(r"^\s*(?:[-*\u2022]|\d+[.)])\s*", "", item).strip().strip("\"'`")
25
+ if keyword and keyword.lower() not in seen:
26
+ seen.add(keyword.lower())
27
+ cleaned.append(keyword)
28
+ return cleaned[:limit]
29
+
30
+
31
+ def generate_keywords(provider: LLMProvider, project_title: str, count: int = 6) -> list[str]:
32
+ prompt = (
33
+ f"Generate {count} specific, technical academic search keywords or short phrases "
34
+ f"for finding research papers related to this project: '{project_title}'.\n"
35
+ "Return ONLY a JSON array of strings, nothing else."
36
+ )
37
+ keywords = parse_keywords(provider.complete(prompt, json_mode=True), limit=count * 2)
38
+ if not keywords:
39
+ raise ValueError("The model did not return any usable keywords")
40
+ return keywords
papersift/models.py ADDED
@@ -0,0 +1,35 @@
1
+ """Data objects shared across papersift."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from datetime import datetime
7
+
8
+
9
+ @dataclass
10
+ class Paper:
11
+ arxiv_id: str
12
+ title: str
13
+ abstract: str
14
+ authors: list[str]
15
+ published: datetime
16
+ url: str
17
+ pdf_url: str
18
+ categories: list[str] = field(default_factory=list)
19
+
20
+ @property
21
+ def year(self) -> int:
22
+ return self.published.year
23
+
24
+ def to_dict(self) -> dict:
25
+ return {
26
+ "arxiv_id": self.arxiv_id,
27
+ "title": self.title,
28
+ "abstract": self.abstract,
29
+ "authors": list(self.authors),
30
+ "published_date": self.published.strftime("%Y-%m-%d"),
31
+ "year": self.year,
32
+ "url": self.url,
33
+ "pdf_url": self.pdf_url,
34
+ "categories": list(self.categories),
35
+ }
@@ -0,0 +1,22 @@
1
+ """LLM providers behind one common interface."""
2
+
3
+ from papersift.providers.base import LLMProvider, ProviderError, RateLimitError
4
+ from papersift.providers.fallback import FallbackProvider
5
+ from papersift.providers.gemini import GeminiProvider
6
+ from papersift.providers.mock import MockProvider
7
+ from papersift.providers.ollama import OllamaProvider
8
+ from papersift.providers.openai_compat import OpenAICompatProvider
9
+ from papersift.providers.registry import available_providers, create_provider
10
+
11
+ __all__ = [
12
+ "FallbackProvider",
13
+ "GeminiProvider",
14
+ "LLMProvider",
15
+ "MockProvider",
16
+ "OllamaProvider",
17
+ "OpenAICompatProvider",
18
+ "ProviderError",
19
+ "RateLimitError",
20
+ "available_providers",
21
+ "create_provider",
22
+ ]
@@ -0,0 +1,64 @@
1
+ """Shared HTTP helper: POST with retry and exponential backoff."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from collections.abc import Callable
7
+
8
+ import requests
9
+
10
+ from papersift.providers.base import ProviderError, RateLimitError
11
+
12
+ MAX_BACKOFF_SECONDS = 60.0
13
+
14
+
15
+ def _retry_after(response, default: float) -> float:
16
+ """Use the server's Retry-After header (seconds) when it is a number."""
17
+ header = response.headers.get("Retry-After") if response.headers else None
18
+ try:
19
+ return min(float(header), MAX_BACKOFF_SECONDS)
20
+ except (TypeError, ValueError):
21
+ return default
22
+
23
+
24
+ def post_with_retry(
25
+ session,
26
+ url: str,
27
+ *,
28
+ headers: dict[str, str],
29
+ json: dict,
30
+ timeout: float,
31
+ max_retries: int = 4,
32
+ backoff: float = 2.0,
33
+ sleep: Callable[[float], None] = time.sleep,
34
+ ):
35
+ """POST ``json`` to ``url`` and return the successful response.
36
+
37
+ Retries on network errors, HTTP 429 and HTTP 5xx, waiting
38
+ ``backoff * 2**attempt`` seconds (or the server's Retry-After).
39
+ Other HTTP errors (400, 401, 403, 404...) fail immediately because
40
+ retrying will not fix them.
41
+ """
42
+ last_error: str = "unknown error"
43
+ rate_limited = False
44
+ for attempt in range(max_retries + 1):
45
+ delay = min(backoff * (2**attempt), MAX_BACKOFF_SECONDS)
46
+ try:
47
+ response = session.post(url, headers=headers, json=json, timeout=timeout)
48
+ except requests.RequestException as exc:
49
+ last_error = f"network error: {exc}"
50
+ rate_limited = False
51
+ else:
52
+ status = response.status_code
53
+ if status == 200:
54
+ return response
55
+ if status == 429 or status >= 500:
56
+ last_error = f"HTTP {status}: {response.text[:200]}"
57
+ rate_limited = status == 429
58
+ delay = _retry_after(response, delay)
59
+ else:
60
+ raise ProviderError(f"HTTP {status}: {response.text[:300]}")
61
+ if attempt < max_retries:
62
+ sleep(delay)
63
+ message = f"Giving up after {max_retries + 1} attempts ({last_error})"
64
+ raise (RateLimitError if rate_limited else ProviderError)(message)
@@ -0,0 +1,50 @@
1
+ """Common interface that every LLM provider implements."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from abc import ABC, abstractmethod
6
+
7
+ Message = dict[str, str] # {"role": "system" | "user" | "assistant", "content": "..."}
8
+
9
+
10
+ class ProviderError(Exception):
11
+ """Raised when an LLM provider fails (network, auth, bad response...)."""
12
+
13
+
14
+ class RateLimitError(ProviderError):
15
+ """Raised when a provider keeps answering 'too many requests'."""
16
+
17
+
18
+ class LLMProvider(ABC):
19
+ """Minimal chat interface used by the rest of papersift.
20
+
21
+ Implementations only have to provide ``chat``; ``complete`` is a
22
+ convenience wrapper for the common "one system prompt + one user prompt"
23
+ case.
24
+ """
25
+
26
+ name: str = "provider"
27
+
28
+ @abstractmethod
29
+ def chat(
30
+ self,
31
+ messages: list[Message],
32
+ *,
33
+ json_mode: bool = False,
34
+ temperature: float = 0.0,
35
+ ) -> str:
36
+ """Send a conversation and return the assistant's text reply."""
37
+
38
+ def complete(
39
+ self,
40
+ prompt: str,
41
+ *,
42
+ system: str | None = None,
43
+ json_mode: bool = False,
44
+ temperature: float = 0.0,
45
+ ) -> str:
46
+ messages: list[Message] = []
47
+ if system:
48
+ messages.append({"role": "system", "content": system})
49
+ messages.append({"role": "user", "content": prompt})
50
+ return self.chat(messages, json_mode=json_mode, temperature=temperature)
@@ -0,0 +1,29 @@
1
+ """Try several providers in order until one works."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+
7
+ from papersift.providers.base import LLMProvider, ProviderError
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ class FallbackProvider(LLMProvider):
13
+ """Calls ``providers`` in order; moves to the next one when a call fails."""
14
+
15
+ def __init__(self, providers: list[LLMProvider]):
16
+ if not providers:
17
+ raise ValueError("FallbackProvider needs at least one provider")
18
+ self.providers = providers
19
+ self.name = "fallback(" + ",".join(p.name for p in providers) + ")"
20
+
21
+ def chat(self, messages, *, json_mode=False, temperature=0.0):
22
+ errors: list[str] = []
23
+ for provider in self.providers:
24
+ try:
25
+ return provider.chat(messages, json_mode=json_mode, temperature=temperature)
26
+ except ProviderError as exc:
27
+ logger.warning("Provider %s failed (%s); trying the next one", provider.name, exc)
28
+ errors.append(f"{provider.name}: {exc}")
29
+ raise ProviderError("All providers failed: " + " | ".join(errors))