papersift 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- papersift/__init__.py +8 -0
- papersift/cli.py +365 -0
- papersift/core.py +216 -0
- papersift/download.py +66 -0
- papersift/gui/__init__.py +1 -0
- papersift/gui/app.py +328 -0
- papersift/gui/ollama.py +36 -0
- papersift/keywords.py +40 -0
- papersift/models.py +35 -0
- papersift/providers/__init__.py +22 -0
- papersift/providers/_http.py +64 -0
- papersift/providers/base.py +50 -0
- papersift/providers/fallback.py +29 -0
- papersift/providers/gemini.py +78 -0
- papersift/providers/mock.py +35 -0
- papersift/providers/ollama.py +50 -0
- papersift/providers/openai_compat.py +70 -0
- papersift/providers/registry.py +59 -0
- papersift/ranking/__init__.py +6 -0
- papersift/ranking/abstract.py +21 -0
- papersift/ranking/fulltext.py +127 -0
- papersift/ranking/rubric.py +122 -0
- papersift/sources/__init__.py +19 -0
- papersift/sources/arxiv.py +78 -0
- papersift/store.py +90 -0
- papersift/utils.py +38 -0
- papersift-0.1.0.dist-info/METADATA +249 -0
- papersift-0.1.0.dist-info/RECORD +32 -0
- papersift-0.1.0.dist-info/WHEEL +5 -0
- papersift-0.1.0.dist-info/entry_points.txt +2 -0
- papersift-0.1.0.dist-info/licenses/LICENSE +21 -0
- papersift-0.1.0.dist-info/top_level.txt +1 -0
papersift/gui/app.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Streamlit interface for the PaperSift research pipeline.
|
|
2
|
+
|
|
3
|
+
Run with: streamlit run papersift/gui/app.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
import streamlit as st
|
|
12
|
+
|
|
13
|
+
from papersift.core import Scout, ScoutResult
|
|
14
|
+
from papersift.gui.ollama import OllamaDiscoveryError, list_ollama_models
|
|
15
|
+
from papersift.providers import ProviderError, available_providers, create_provider
|
|
16
|
+
from papersift.ranking import AssessmentError
|
|
17
|
+
from papersift.sources import SourceError
|
|
18
|
+
from papersift.ranking.fulltext import FulltextError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
PROVIDER_LABELS = {
|
|
22
|
+
"ollama": "Ollama (local)",
|
|
23
|
+
"gemini": "Google Gemini",
|
|
24
|
+
"groq": "Groq",
|
|
25
|
+
"openrouter": "OpenRouter",
|
|
26
|
+
"together": "Together AI",
|
|
27
|
+
"cerebras": "Cerebras",
|
|
28
|
+
"mistral": "Mistral",
|
|
29
|
+
"openai": "OpenAI",
|
|
30
|
+
"openai_compat": "Custom OpenAI-compatible endpoint",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _provider_label(provider_name: str) -> str:
|
|
35
|
+
return PROVIDER_LABELS.get(provider_name, provider_name.replace("_", " ").title())
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@st.cache_data(ttl=30, show_spinner=False)
|
|
39
|
+
def _cached_ollama_models(host: str) -> list[str]:
|
|
40
|
+
return list_ollama_models(host)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _result_frame(results: list[ScoutResult]) -> pd.DataFrame:
|
|
44
|
+
rows = []
|
|
45
|
+
for result in results:
|
|
46
|
+
paper = result.paper
|
|
47
|
+
assessment = result.abstract_assessment
|
|
48
|
+
rows.append(
|
|
49
|
+
{
|
|
50
|
+
"Title": paper.title,
|
|
51
|
+
"Year": paper.year,
|
|
52
|
+
"Score": result.score,
|
|
53
|
+
"Recommendation": assessment.recommendation,
|
|
54
|
+
"Reasoning": assessment.rubric.reasoning,
|
|
55
|
+
"Authors": ", ".join(paper.authors),
|
|
56
|
+
"arXiv": paper.url,
|
|
57
|
+
"PDF": paper.pdf_url,
|
|
58
|
+
"_result": result,
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
return pd.DataFrame(rows)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _show_result_details(result: ScoutResult) -> None:
|
|
65
|
+
paper = result.paper
|
|
66
|
+
assessment = result.abstract_assessment
|
|
67
|
+
rubric = assessment.rubric
|
|
68
|
+
|
|
69
|
+
st.markdown(f"#### {paper.title}")
|
|
70
|
+
st.caption(f"{paper.year} · {', '.join(paper.authors) or 'Authors unavailable'} · arXiv:{paper.arxiv_id}")
|
|
71
|
+
st.write(paper.abstract)
|
|
72
|
+
|
|
73
|
+
score_cols = st.columns(4)
|
|
74
|
+
score_cols[0].metric("Final score", f"{result.score}/100")
|
|
75
|
+
score_cols[1].metric("Topic match", f"{rubric.topic_match}/3")
|
|
76
|
+
score_cols[2].metric("Method match", f"{rubric.method_match}/3")
|
|
77
|
+
score_cols[3].metric("Novelty", f"{rubric.novelty}/3")
|
|
78
|
+
|
|
79
|
+
st.markdown(f"**Recommendation:** {assessment.recommendation}")
|
|
80
|
+
if rubric.reasoning:
|
|
81
|
+
st.write(rubric.reasoning)
|
|
82
|
+
|
|
83
|
+
left, right = st.columns(2)
|
|
84
|
+
with left:
|
|
85
|
+
st.markdown("**Key strengths**")
|
|
86
|
+
if rubric.key_strengths:
|
|
87
|
+
for item in rubric.key_strengths:
|
|
88
|
+
st.markdown(f"- {item}")
|
|
89
|
+
else:
|
|
90
|
+
st.caption("No strengths were returned.")
|
|
91
|
+
with right:
|
|
92
|
+
st.markdown("**Potential limitations**")
|
|
93
|
+
if rubric.potential_limitations:
|
|
94
|
+
for item in rubric.potential_limitations:
|
|
95
|
+
st.markdown(f"- {item}")
|
|
96
|
+
else:
|
|
97
|
+
st.caption("No limitations were returned.")
|
|
98
|
+
|
|
99
|
+
if result.fulltext_assessment:
|
|
100
|
+
fulltext = result.fulltext_assessment
|
|
101
|
+
with st.expander("Full-text assessment"):
|
|
102
|
+
st.write(f"Score: **{fulltext.score}/100** · {fulltext.recommendation}")
|
|
103
|
+
st.write(fulltext.rubric.reasoning)
|
|
104
|
+
|
|
105
|
+
links = st.columns(3)
|
|
106
|
+
links[0].link_button("Open arXiv page", paper.url, use_container_width=True)
|
|
107
|
+
if result.pdf_path and Path(result.pdf_path).exists():
|
|
108
|
+
with open(result.pdf_path, "rb") as pdf_file:
|
|
109
|
+
links[1].download_button(
|
|
110
|
+
"Download saved PDF",
|
|
111
|
+
data=pdf_file,
|
|
112
|
+
file_name=Path(result.pdf_path).name,
|
|
113
|
+
mime="application/pdf",
|
|
114
|
+
use_container_width=True,
|
|
115
|
+
)
|
|
116
|
+
else:
|
|
117
|
+
links[1].link_button("Open PDF", paper.pdf_url, use_container_width=True)
|
|
118
|
+
if result.metadata_path and Path(result.metadata_path).exists():
|
|
119
|
+
with open(result.metadata_path, "rb") as metadata_file:
|
|
120
|
+
links[2].download_button(
|
|
121
|
+
"Download metadata",
|
|
122
|
+
data=metadata_file,
|
|
123
|
+
file_name=Path(result.metadata_path).name,
|
|
124
|
+
mime="application/json",
|
|
125
|
+
use_container_width=True,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def main() -> None:
|
|
130
|
+
st.set_page_config(page_title="PaperSift", page_icon="📚", layout="wide")
|
|
131
|
+
st.title("PaperSift")
|
|
132
|
+
st.caption("Find the papers that matter to your research, automatically.")
|
|
133
|
+
|
|
134
|
+
settings_tab, results_tab = st.tabs(["Settings", "Research results"])
|
|
135
|
+
|
|
136
|
+
with settings_tab:
|
|
137
|
+
st.header("Research agent configuration")
|
|
138
|
+
st.write("Configure a research topic, an AI provider, and the search parameters.")
|
|
139
|
+
|
|
140
|
+
project_title = st.text_input(
|
|
141
|
+
"Research project",
|
|
142
|
+
placeholder="e.g. Self-improving generative AI agents",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
provider_names = available_providers()
|
|
146
|
+
provider_name = st.selectbox(
|
|
147
|
+
"Provider",
|
|
148
|
+
options=provider_names,
|
|
149
|
+
format_func=_provider_label,
|
|
150
|
+
)
|
|
151
|
+
api_key = ""
|
|
152
|
+
base_url = ""
|
|
153
|
+
if provider_name == "ollama":
|
|
154
|
+
base_url = st.text_input(
|
|
155
|
+
"Ollama host",
|
|
156
|
+
value="http://localhost:11434",
|
|
157
|
+
help="The address of the Ollama server.",
|
|
158
|
+
)
|
|
159
|
+
st.caption("Ollama runs locally and does not require an API key.")
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
installed_models = _cached_ollama_models(base_url.strip() or "http://localhost:11434")
|
|
163
|
+
except OllamaDiscoveryError as exc:
|
|
164
|
+
installed_models = []
|
|
165
|
+
st.warning(f"Could not discover Ollama models: {exc}")
|
|
166
|
+
|
|
167
|
+
if installed_models:
|
|
168
|
+
model_options = [*installed_models, "Enter a model name manually"]
|
|
169
|
+
selected_model = st.selectbox("Installed Ollama model", model_options)
|
|
170
|
+
if selected_model == "Enter a model name manually":
|
|
171
|
+
model_name = st.text_input("Model name", placeholder="e.g. llama3.1:8b")
|
|
172
|
+
else:
|
|
173
|
+
model_name = selected_model
|
|
174
|
+
else:
|
|
175
|
+
st.info("No installed models were detected. Start Ollama and pull a model, or enter its name manually.")
|
|
176
|
+
model_name = st.text_input("Model name", placeholder="e.g. llama3.1:8b")
|
|
177
|
+
|
|
178
|
+
if st.button("Refresh installed models", key="refresh_ollama_models"):
|
|
179
|
+
_cached_ollama_models.clear()
|
|
180
|
+
st.rerun()
|
|
181
|
+
elif provider_name == "openai_compat":
|
|
182
|
+
model_name = st.text_input(
|
|
183
|
+
"Model name",
|
|
184
|
+
placeholder="e.g. provider-model-name",
|
|
185
|
+
)
|
|
186
|
+
base_url = st.text_input(
|
|
187
|
+
"Base URL",
|
|
188
|
+
placeholder="https://your-provider.example/v1",
|
|
189
|
+
)
|
|
190
|
+
api_key = st.text_input("API key", type="password")
|
|
191
|
+
else:
|
|
192
|
+
model_name = st.text_input(
|
|
193
|
+
"Model name",
|
|
194
|
+
placeholder="e.g. gemini-2.5-flash, gpt-4.1-mini",
|
|
195
|
+
)
|
|
196
|
+
api_key = st.text_input("API key", type="password")
|
|
197
|
+
|
|
198
|
+
st.subheader("Search parameters")
|
|
199
|
+
relevance_threshold = st.slider(
|
|
200
|
+
"Minimum relevance score",
|
|
201
|
+
min_value=0,
|
|
202
|
+
max_value=100,
|
|
203
|
+
value=80,
|
|
204
|
+
help="Only papers meeting this final score are included in the results.",
|
|
205
|
+
)
|
|
206
|
+
days_to_search = st.number_input(
|
|
207
|
+
"Search window (days)", min_value=1, max_value=3650, value=365, step=1
|
|
208
|
+
)
|
|
209
|
+
max_results = st.number_input(
|
|
210
|
+
"Maximum candidate papers", min_value=1, max_value=500, value=100, step=10
|
|
211
|
+
)
|
|
212
|
+
fulltext = st.checkbox(
|
|
213
|
+
"Enable full-text assessment",
|
|
214
|
+
value=False,
|
|
215
|
+
help="Downloads and evaluates the full text of the highest-scoring candidates. Requires PyMuPDF.",
|
|
216
|
+
)
|
|
217
|
+
output_dir = st.text_input("Output directory", value="papersift_output")
|
|
218
|
+
|
|
219
|
+
submitted = st.button("Run research agent", type="primary", use_container_width=True)
|
|
220
|
+
|
|
221
|
+
if submitted:
|
|
222
|
+
if not project_title.strip():
|
|
223
|
+
st.error("Enter a research project title.")
|
|
224
|
+
elif not model_name.strip():
|
|
225
|
+
st.error("Enter a model name.")
|
|
226
|
+
elif provider_name != "ollama" and not api_key.strip():
|
|
227
|
+
st.error("Enter the API key for the selected provider.")
|
|
228
|
+
elif provider_name == "openai_compat" and not base_url.strip():
|
|
229
|
+
st.error("Enter the base URL for the custom OpenAI-compatible provider.")
|
|
230
|
+
elif provider_name == "ollama" and not base_url.strip():
|
|
231
|
+
st.error("Enter the Ollama host URL.")
|
|
232
|
+
else:
|
|
233
|
+
try:
|
|
234
|
+
provider = create_provider(
|
|
235
|
+
provider_name,
|
|
236
|
+
model_name.strip(),
|
|
237
|
+
api_key=api_key.strip() or None,
|
|
238
|
+
base_url=base_url.strip() or None,
|
|
239
|
+
)
|
|
240
|
+
scout = Scout(
|
|
241
|
+
project_title=project_title.strip(),
|
|
242
|
+
provider=provider,
|
|
243
|
+
relevance_threshold=int(relevance_threshold),
|
|
244
|
+
since_days=int(days_to_search),
|
|
245
|
+
max_results=int(max_results),
|
|
246
|
+
output_dir=output_dir.strip() or "papersift_output",
|
|
247
|
+
fulltext=fulltext,
|
|
248
|
+
)
|
|
249
|
+
with st.spinner("PaperSift is generating search keywords, searching arXiv, and assessing papers..."):
|
|
250
|
+
results = scout.run()
|
|
251
|
+
st.session_state["papersift_results"] = results
|
|
252
|
+
st.session_state["papersift_project_title"] = project_title.strip()
|
|
253
|
+
st.session_state["papersift_run_config"] = {
|
|
254
|
+
"provider": provider_name,
|
|
255
|
+
"model": model_name.strip(),
|
|
256
|
+
"threshold": int(relevance_threshold),
|
|
257
|
+
"days": int(days_to_search),
|
|
258
|
+
"max_results": int(max_results),
|
|
259
|
+
"fulltext": fulltext,
|
|
260
|
+
"output_dir": output_dir.strip() or "papersift_output",
|
|
261
|
+
}
|
|
262
|
+
st.success(f"Research completed. {len(results)} relevant paper(s) found.")
|
|
263
|
+
st.info("API credentials are used to create the provider and are not included in the saved run configuration.")
|
|
264
|
+
except (ProviderError, SourceError, AssessmentError, FulltextError, ValueError) as exc:
|
|
265
|
+
st.error(f"Research could not be completed: {exc}")
|
|
266
|
+
except Exception as exc: # noqa: BLE001 - keep unexpected runtime failures visible in the UI
|
|
267
|
+
st.exception(exc)
|
|
268
|
+
|
|
269
|
+
with results_tab:
|
|
270
|
+
st.header("Research results")
|
|
271
|
+
results = st.session_state.get("papersift_results", [])
|
|
272
|
+
project_title = st.session_state.get("papersift_project_title")
|
|
273
|
+
|
|
274
|
+
if not results:
|
|
275
|
+
st.info("No research results yet. Configure your project and run the research agent from the Settings tab.")
|
|
276
|
+
else:
|
|
277
|
+
st.caption(f"Project: {project_title}")
|
|
278
|
+
df = _result_frame(results)
|
|
279
|
+
|
|
280
|
+
metric_cols = st.columns(3)
|
|
281
|
+
metric_cols[0].metric("Papers retained", len(results))
|
|
282
|
+
metric_cols[1].metric("Average score", f"{df['Score'].mean():.1f}/100")
|
|
283
|
+
metric_cols[2].metric("Highest score", f"{df['Score'].max()}/100")
|
|
284
|
+
|
|
285
|
+
st.subheader("Filter and sort")
|
|
286
|
+
filter_cols = st.columns([2, 2, 1])
|
|
287
|
+
min_score = filter_cols[0].slider(
|
|
288
|
+
"Minimum score", 0, 100, 0, key="results_min_score"
|
|
289
|
+
)
|
|
290
|
+
sort_by = filter_cols[1].selectbox(
|
|
291
|
+
"Sort by", ["Score", "Year", "Title"], key="results_sort_by"
|
|
292
|
+
)
|
|
293
|
+
descending = filter_cols[2].checkbox(
|
|
294
|
+
"Descending", value=True, key="results_descending"
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
filtered = df[df["Score"] >= min_score].sort_values(
|
|
298
|
+
by=sort_by, ascending=not descending
|
|
299
|
+
)
|
|
300
|
+
st.caption(f"Showing {len(filtered)} of {len(df)} retained papers.")
|
|
301
|
+
display_df = filtered.drop(columns=["_result"])
|
|
302
|
+
st.dataframe(
|
|
303
|
+
display_df,
|
|
304
|
+
hide_index=True,
|
|
305
|
+
use_container_width=True,
|
|
306
|
+
column_config={
|
|
307
|
+
"Title": st.column_config.TextColumn("Paper title", width="large"),
|
|
308
|
+
"Score": st.column_config.ProgressColumn(
|
|
309
|
+
"Relevance score", min_value=0, max_value=100, format="%d/100"
|
|
310
|
+
),
|
|
311
|
+
"arXiv": st.column_config.LinkColumn("arXiv", display_text="Open paper"),
|
|
312
|
+
"PDF": st.column_config.LinkColumn("PDF / saved path"),
|
|
313
|
+
"Reasoning": st.column_config.TextColumn("Assessment reasoning", width="large"),
|
|
314
|
+
},
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
st.subheader("Paper assessments")
|
|
318
|
+
for _, row in filtered.iterrows():
|
|
319
|
+
result = row["_result"]
|
|
320
|
+
with st.expander(f"{result.score}/100 · {result.paper.title}"):
|
|
321
|
+
_show_result_details(result)
|
|
322
|
+
|
|
323
|
+
st.divider()
|
|
324
|
+
st.caption("PaperSift · Research assistant")
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
if __name__ == "__main__":
|
|
328
|
+
main()
|
papersift/gui/ollama.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Helpers for discovering locally installed Ollama models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class OllamaDiscoveryError(RuntimeError):
|
|
9
|
+
"""Raised when the Ollama model list cannot be retrieved or parsed."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def list_ollama_models(host: str = "http://localhost:11434", timeout: float = 3.0) -> list[str]:
|
|
13
|
+
"""Return every model name reported by an Ollama server's /api/tags endpoint."""
|
|
14
|
+
url = f"{host.rstrip('/')}/api/tags"
|
|
15
|
+
try:
|
|
16
|
+
response = requests.get(url, timeout=timeout)
|
|
17
|
+
response.raise_for_status()
|
|
18
|
+
payload = response.json()
|
|
19
|
+
except requests.RequestException as exc:
|
|
20
|
+
raise OllamaDiscoveryError(f"Could not connect to Ollama at {host}: {exc}") from exc
|
|
21
|
+
except ValueError as exc:
|
|
22
|
+
raise OllamaDiscoveryError("Ollama returned invalid JSON while listing installed models.") from exc
|
|
23
|
+
|
|
24
|
+
models = payload.get("models") if isinstance(payload, dict) else None
|
|
25
|
+
if not isinstance(models, list):
|
|
26
|
+
raise OllamaDiscoveryError("Ollama's /api/tags response did not contain a models list.")
|
|
27
|
+
|
|
28
|
+
names = sorted(
|
|
29
|
+
{
|
|
30
|
+
model["name"].strip()
|
|
31
|
+
for model in models
|
|
32
|
+
if isinstance(model, dict) and isinstance(model.get("name"), str) and model["name"].strip()
|
|
33
|
+
},
|
|
34
|
+
key=str.casefold,
|
|
35
|
+
)
|
|
36
|
+
return names
|
papersift/keywords.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Turn a project description into arXiv search keywords with an LLM."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from papersift.providers import LLMProvider
|
|
8
|
+
from papersift.utils import extract_json
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def parse_keywords(text: str, limit: int = 12) -> list[str]:
|
|
12
|
+
"""Accept a JSON list, or a comma/newline separated list, and clean it up."""
|
|
13
|
+
items: list[str]
|
|
14
|
+
try:
|
|
15
|
+
data = extract_json(text)
|
|
16
|
+
if isinstance(data, dict):
|
|
17
|
+
data = next((v for v in data.values() if isinstance(v, list)), [])
|
|
18
|
+
items = [str(x) for x in data] if isinstance(data, list) else []
|
|
19
|
+
except ValueError:
|
|
20
|
+
items = re.split(r"[,\n;]", text)
|
|
21
|
+
cleaned: list[str] = []
|
|
22
|
+
seen: set[str] = set()
|
|
23
|
+
for item in items:
|
|
24
|
+
keyword = re.sub(r"^\s*(?:[-*\u2022]|\d+[.)])\s*", "", item).strip().strip("\"'`")
|
|
25
|
+
if keyword and keyword.lower() not in seen:
|
|
26
|
+
seen.add(keyword.lower())
|
|
27
|
+
cleaned.append(keyword)
|
|
28
|
+
return cleaned[:limit]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def generate_keywords(provider: LLMProvider, project_title: str, count: int = 6) -> list[str]:
|
|
32
|
+
prompt = (
|
|
33
|
+
f"Generate {count} specific, technical academic search keywords or short phrases "
|
|
34
|
+
f"for finding research papers related to this project: '{project_title}'.\n"
|
|
35
|
+
"Return ONLY a JSON array of strings, nothing else."
|
|
36
|
+
)
|
|
37
|
+
keywords = parse_keywords(provider.complete(prompt, json_mode=True), limit=count * 2)
|
|
38
|
+
if not keywords:
|
|
39
|
+
raise ValueError("The model did not return any usable keywords")
|
|
40
|
+
return keywords
|
papersift/models.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Data objects shared across papersift."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Paper:
|
|
11
|
+
arxiv_id: str
|
|
12
|
+
title: str
|
|
13
|
+
abstract: str
|
|
14
|
+
authors: list[str]
|
|
15
|
+
published: datetime
|
|
16
|
+
url: str
|
|
17
|
+
pdf_url: str
|
|
18
|
+
categories: list[str] = field(default_factory=list)
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def year(self) -> int:
|
|
22
|
+
return self.published.year
|
|
23
|
+
|
|
24
|
+
def to_dict(self) -> dict:
|
|
25
|
+
return {
|
|
26
|
+
"arxiv_id": self.arxiv_id,
|
|
27
|
+
"title": self.title,
|
|
28
|
+
"abstract": self.abstract,
|
|
29
|
+
"authors": list(self.authors),
|
|
30
|
+
"published_date": self.published.strftime("%Y-%m-%d"),
|
|
31
|
+
"year": self.year,
|
|
32
|
+
"url": self.url,
|
|
33
|
+
"pdf_url": self.pdf_url,
|
|
34
|
+
"categories": list(self.categories),
|
|
35
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""LLM providers behind one common interface."""
|
|
2
|
+
|
|
3
|
+
from papersift.providers.base import LLMProvider, ProviderError, RateLimitError
|
|
4
|
+
from papersift.providers.fallback import FallbackProvider
|
|
5
|
+
from papersift.providers.gemini import GeminiProvider
|
|
6
|
+
from papersift.providers.mock import MockProvider
|
|
7
|
+
from papersift.providers.ollama import OllamaProvider
|
|
8
|
+
from papersift.providers.openai_compat import OpenAICompatProvider
|
|
9
|
+
from papersift.providers.registry import available_providers, create_provider
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"FallbackProvider",
|
|
13
|
+
"GeminiProvider",
|
|
14
|
+
"LLMProvider",
|
|
15
|
+
"MockProvider",
|
|
16
|
+
"OllamaProvider",
|
|
17
|
+
"OpenAICompatProvider",
|
|
18
|
+
"ProviderError",
|
|
19
|
+
"RateLimitError",
|
|
20
|
+
"available_providers",
|
|
21
|
+
"create_provider",
|
|
22
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Shared HTTP helper: POST with retry and exponential backoff."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from papersift.providers.base import ProviderError, RateLimitError
|
|
11
|
+
|
|
12
|
+
MAX_BACKOFF_SECONDS = 60.0
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _retry_after(response, default: float) -> float:
|
|
16
|
+
"""Use the server's Retry-After header (seconds) when it is a number."""
|
|
17
|
+
header = response.headers.get("Retry-After") if response.headers else None
|
|
18
|
+
try:
|
|
19
|
+
return min(float(header), MAX_BACKOFF_SECONDS)
|
|
20
|
+
except (TypeError, ValueError):
|
|
21
|
+
return default
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def post_with_retry(
|
|
25
|
+
session,
|
|
26
|
+
url: str,
|
|
27
|
+
*,
|
|
28
|
+
headers: dict[str, str],
|
|
29
|
+
json: dict,
|
|
30
|
+
timeout: float,
|
|
31
|
+
max_retries: int = 4,
|
|
32
|
+
backoff: float = 2.0,
|
|
33
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
34
|
+
):
|
|
35
|
+
"""POST ``json`` to ``url`` and return the successful response.
|
|
36
|
+
|
|
37
|
+
Retries on network errors, HTTP 429 and HTTP 5xx, waiting
|
|
38
|
+
``backoff * 2**attempt`` seconds (or the server's Retry-After).
|
|
39
|
+
Other HTTP errors (400, 401, 403, 404...) fail immediately because
|
|
40
|
+
retrying will not fix them.
|
|
41
|
+
"""
|
|
42
|
+
last_error: str = "unknown error"
|
|
43
|
+
rate_limited = False
|
|
44
|
+
for attempt in range(max_retries + 1):
|
|
45
|
+
delay = min(backoff * (2**attempt), MAX_BACKOFF_SECONDS)
|
|
46
|
+
try:
|
|
47
|
+
response = session.post(url, headers=headers, json=json, timeout=timeout)
|
|
48
|
+
except requests.RequestException as exc:
|
|
49
|
+
last_error = f"network error: {exc}"
|
|
50
|
+
rate_limited = False
|
|
51
|
+
else:
|
|
52
|
+
status = response.status_code
|
|
53
|
+
if status == 200:
|
|
54
|
+
return response
|
|
55
|
+
if status == 429 or status >= 500:
|
|
56
|
+
last_error = f"HTTP {status}: {response.text[:200]}"
|
|
57
|
+
rate_limited = status == 429
|
|
58
|
+
delay = _retry_after(response, delay)
|
|
59
|
+
else:
|
|
60
|
+
raise ProviderError(f"HTTP {status}: {response.text[:300]}")
|
|
61
|
+
if attempt < max_retries:
|
|
62
|
+
sleep(delay)
|
|
63
|
+
message = f"Giving up after {max_retries + 1} attempts ({last_error})"
|
|
64
|
+
raise (RateLimitError if rate_limited else ProviderError)(message)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Common interface that every LLM provider implements."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
Message = dict[str, str] # {"role": "system" | "user" | "assistant", "content": "..."}
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ProviderError(Exception):
|
|
11
|
+
"""Raised when an LLM provider fails (network, auth, bad response...)."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RateLimitError(ProviderError):
|
|
15
|
+
"""Raised when a provider keeps answering 'too many requests'."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class LLMProvider(ABC):
|
|
19
|
+
"""Minimal chat interface used by the rest of papersift.
|
|
20
|
+
|
|
21
|
+
Implementations only have to provide ``chat``; ``complete`` is a
|
|
22
|
+
convenience wrapper for the common "one system prompt + one user prompt"
|
|
23
|
+
case.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
name: str = "provider"
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def chat(
|
|
30
|
+
self,
|
|
31
|
+
messages: list[Message],
|
|
32
|
+
*,
|
|
33
|
+
json_mode: bool = False,
|
|
34
|
+
temperature: float = 0.0,
|
|
35
|
+
) -> str:
|
|
36
|
+
"""Send a conversation and return the assistant's text reply."""
|
|
37
|
+
|
|
38
|
+
def complete(
|
|
39
|
+
self,
|
|
40
|
+
prompt: str,
|
|
41
|
+
*,
|
|
42
|
+
system: str | None = None,
|
|
43
|
+
json_mode: bool = False,
|
|
44
|
+
temperature: float = 0.0,
|
|
45
|
+
) -> str:
|
|
46
|
+
messages: list[Message] = []
|
|
47
|
+
if system:
|
|
48
|
+
messages.append({"role": "system", "content": system})
|
|
49
|
+
messages.append({"role": "user", "content": prompt})
|
|
50
|
+
return self.chat(messages, json_mode=json_mode, temperature=temperature)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Try several providers in order until one works."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from papersift.providers.base import LLMProvider, ProviderError
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FallbackProvider(LLMProvider):
|
|
13
|
+
"""Calls ``providers`` in order; moves to the next one when a call fails."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, providers: list[LLMProvider]):
|
|
16
|
+
if not providers:
|
|
17
|
+
raise ValueError("FallbackProvider needs at least one provider")
|
|
18
|
+
self.providers = providers
|
|
19
|
+
self.name = "fallback(" + ",".join(p.name for p in providers) + ")"
|
|
20
|
+
|
|
21
|
+
def chat(self, messages, *, json_mode=False, temperature=0.0):
|
|
22
|
+
errors: list[str] = []
|
|
23
|
+
for provider in self.providers:
|
|
24
|
+
try:
|
|
25
|
+
return provider.chat(messages, json_mode=json_mode, temperature=temperature)
|
|
26
|
+
except ProviderError as exc:
|
|
27
|
+
logger.warning("Provider %s failed (%s); trying the next one", provider.name, exc)
|
|
28
|
+
errors.append(f"{provider.name}: {exc}")
|
|
29
|
+
raise ProviderError("All providers failed: " + " | ".join(errors))
|