paper-firehose 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- paper_firehose-0.1.0/LICENSE +21 -0
- paper_firehose-0.1.0/PKG-INFO +281 -0
- paper_firehose-0.1.0/README.md +230 -0
- paper_firehose-0.1.0/pyproject.toml +57 -0
- paper_firehose-0.1.0/setup.cfg +4 -0
- paper_firehose-0.1.0/src/paper_firehose/__init__.py +294 -0
- paper_firehose-0.1.0/src/paper_firehose/cli.py +325 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/__init__.py +1 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/abstracts.py +556 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/email_list.py +224 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/filter.py +160 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/generate_html.py +111 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/pqa_summary.py +836 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/rank.py +344 -0
- paper_firehose-0.1.0/src/paper_firehose/commands/summarize.py +374 -0
- paper_firehose-0.1.0/src/paper_firehose/core/__init__.py +1 -0
- paper_firehose-0.1.0/src/paper_firehose/core/config.py +347 -0
- paper_firehose-0.1.0/src/paper_firehose/core/database.py +681 -0
- paper_firehose-0.1.0/src/paper_firehose/core/paths.py +137 -0
- paper_firehose-0.1.0/src/paper_firehose/processors/__init__.py +1 -0
- paper_firehose-0.1.0/src/paper_firehose/processors/emailer.py +416 -0
- paper_firehose-0.1.0/src/paper_firehose/processors/feed_processor.py +199 -0
- paper_firehose-0.1.0/src/paper_firehose/processors/html_generator.py +658 -0
- paper_firehose-0.1.0/src/paper_firehose/processors/st_ranker.py +84 -0
- paper_firehose-0.1.0/src/paper_firehose/system/config/config.yaml +225 -0
- paper_firehose-0.1.0/src/paper_firehose/system/config/secrets/README.md +12 -0
- paper_firehose-0.1.0/src/paper_firehose/system/config/secrets/mailing_lists.yaml +11 -0
- paper_firehose-0.1.0/src/paper_firehose/system/config/topics/perovskites.yaml +105 -0
- paper_firehose-0.1.0/src/paper_firehose/system/templates/html_template.html +33 -0
- paper_firehose-0.1.0/src/paper_firehose/system/templates/llmsummary_template.html +167 -0
- paper_firehose-0.1.0/src/paper_firehose/system/templates/ranked_template.html +33 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/PKG-INFO +281 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/SOURCES.txt +39 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/dependency_links.txt +1 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/entry_points.txt +2 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/requires.txt +11 -0
- paper_firehose-0.1.0/src/paper_firehose.egg-info/top_level.txt +1 -0
- paper_firehose-0.1.0/tests/test_config_manager.py +80 -0
- paper_firehose-0.1.0/tests/test_paths.py +44 -0
- paper_firehose-0.1.0/tests/test_pipeline.py +148 -0
- paper_firehose-0.1.0/tests/test_summarize_api_key.py +34 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 paper-firehose, Peter Nemes-Incze
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: paper_firehose
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Filter, rank, and summarize research-paper RSS feeds.
|
|
5
|
+
Author: Peter Nemes-Incze
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 paper-firehose, Peter Nemes-Incze
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Keywords: rss,llm,research,papers,arxiv,summaries
|
|
29
|
+
Classifier: Intended Audience :: Science/Research
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Operating System :: OS Independent
|
|
32
|
+
Classifier: Programming Language :: Python :: 3
|
|
33
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
36
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
37
|
+
Requires-Python: >=3.10
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
License-File: LICENSE
|
|
40
|
+
Requires-Dist: feedparser
|
|
41
|
+
Requires-Dist: PyYAML
|
|
42
|
+
Requires-Dist: click
|
|
43
|
+
Requires-Dist: sentence-transformers>=2.2.2
|
|
44
|
+
Requires-Dist: requests
|
|
45
|
+
Requires-Dist: openai>=1.30.0
|
|
46
|
+
Requires-Dist: paper-qa
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
49
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
50
|
+
Dynamic: license-file
|
|
51
|
+
|
|
52
|
+
# Paper Firehose
|
|
53
|
+
|
|
54
|
+
- Fetches academic RSS feeds, filters entries with per-topic regex, and writes results into SQLite databases. HTML pages with the results are rendered directly from the database.
|
|
55
|
+
- Results are ranked by cosine similarity to a set of user defined keywords. Configurable list of authors can get a ranking boost. So papers from your friends / competitors can be boosted in ranking.
|
|
56
|
+
- Highest ranked results are summarized by an LLM. For summarization to work, you need an OpenAI API key. Full text summarization uses [Paper-qa](https://github.com/Future-House/paper-qa).
|
|
57
|
+
- New search terms can be created by simply adding a yaml config file under: `config/topics/your_topic_name.yaml`. Look at the other topics for guidance.
|
|
58
|
+
- The repository ships a self-contained `system/` bundle (configs, templates, sample topics). On first run these files are copied into your runtime data directory (default `~/.paper_firehose` or the path in `PAPER_FIREHOSE_DATA_DIR`) so you can customize them with your own search terms, RSS feed sources, etc.
|
|
59
|
+
- GitHub Actions–only YAML config lives in `github_actions_config/`. When the workflow sets `PAPER_FIREHOSE_DATA_DIR`, those files are synced into `${PAPER_FIREHOSE_DATA_DIR}/config/`, keeping the published package generic while letting GitHub Actions runs use repo-specific overrides. So if you want to use this via GA, set your preferences in this directory.
|
|
60
|
+
- Daily automation runs through GitHub Actions (`.github/workflows/pages.yml`). The workflow restores database snapshots from the `data` branch, executes the CLI pipeline (`filter → rank → abstracts → html` plus optional `pqa_summary`/`email`), and publishes refreshed HTML/SQLite artifacts back to GitHub Pages and the `data` branch.
|
|
61
|
+
|
|
62
|
+
Written using Python 3.11.
|
|
63
|
+
For dependencies check `requirements.txt`.
|
|
64
|
+
OpenAI API key is searched for in the `openaikulcs.env` file in the repo root or `OPENAI_API_KEY` environment variable.
|
|
65
|
+
|
|
66
|
+
## CLI
|
|
67
|
+
|
|
68
|
+
- **Filter**: fetch from RSS feed list, dedup, match regex, write DBs, render HTML
|
|
69
|
+
- `python cli/main.py -v filter [--topic TOPIC]`
|
|
70
|
+
- Backs up `all_feed_entries.db` and `matched_entries_history.db` (keeps 3 latest backups).
|
|
71
|
+
- Clears `papers.db` working table before processing this run.
|
|
72
|
+
- Fetches configured feeds for each topic, dedups by title against `all_feed_entries.db`, filters by topic regex.
|
|
73
|
+
- Writes matches to `papers.db` (`status='filtered'`); optionally archives to `matched_entries_history.db` if `output.archive: true`.
|
|
74
|
+
- Saves ALL processed entries (matched and non-matched) to `all_feed_entries.db` for future dedup.
|
|
75
|
+
- Renders per-topic HTML from `papers.db`.
|
|
76
|
+
|
|
77
|
+
- **Rank**: ranks fetched papers by title, wrt keywords in yaml config (optional)
|
|
78
|
+
- `python cli/main.py rank [--topic TOPIC]`
|
|
79
|
+
- Computes and writes `rank_score` for `papers.db` entries using sentence-transformers. HTML files with ranked entries are generated.
|
|
80
|
+
- Model selection: if `models/all-MiniLM-L6-v2` exists, it is used; otherwise it falls back to the Hugging Face repo id `all-MiniLM-L6-v2` and downloads once into cache. You can vendor the model with `python scripts/vendor_model.py`.
|
|
81
|
+
- Scoring details: applies a small penalty for `ranking.negative_queries` matches (title/summary). Optional boosts: per-topic `ranking.preferred_authors` with `ranking.priority_author_boost`, and global `priority_journal_boost` for feeds listed in `priority_journals`.
|
|
82
|
+
|
|
83
|
+
- **Abstracts**: try to fetch abstracts
|
|
84
|
+
- `python cli/main.py abstracts [--topic TOPIC] [--mailto you@example.com] [--limit N] [--rps 1.0]`
|
|
85
|
+
- Order: 1) Fill arXiv/cond-mat abstracts from `summary` (no threshold) 2) Above-threshold: Crossref (DOI, then title) 3) Above-threshold: Semantic Scholar → OpenAlex → PubMed
|
|
86
|
+
- Threshold: topic `abstract_fetch.rank_threshold` else global `defaults.rank_threshold`.
|
|
87
|
+
- Only topics with `abstract_fetch.enabled: true` are processed.
|
|
88
|
+
- Writes to both `papers.db.entries.abstract` and `matched_entries_history.db.matched_entries.abstract`.
|
|
89
|
+
- Rate limiting: descriptive User-Agent (includes `--mailto` or `$MAILTO`), respects Retry-After; default ~1 req/sec via `--rps`.
|
|
90
|
+
- Populates the `entries.abstract` column; leaves other fields unchanged.
|
|
91
|
+
- Contact email: if `--mailto` is not provided, the command reads `$MAILTO` from the environment; if unset, it uses a safe default.
|
|
92
|
+
|
|
93
|
+
- **Summarize** (optional)
|
|
94
|
+
- `python cli/main.py summarize [--topic TOPIC] [--rps 0.5]`
|
|
95
|
+
- Selects top entries per topic based on `llm_summary.score_cutoff` and `llm_summary.top_n`, builds input strictly from `title + abstract` (skips entries without an abstract), and calls the configured OpenAI chat model.
|
|
96
|
+
- Writes summaries to `papers.db.entries.llm_summary` and, when present, `matched_entries_history.db.matched_entries.llm_summary`.
|
|
97
|
+
- Uses `config.llm` settings. Supports JSON or plain-text responses; JSON is preferred and rendered with headings.
|
|
98
|
+
- Note: This command only updates databases. Use `html` to render pages.
|
|
99
|
+
|
|
100
|
+
- **pqa_summary** (PDF-based summarization)
|
|
101
|
+
- `python cli/main.py pqa_summary [--topic TOPIC]`
|
|
102
|
+
- Selects preprints from arXiv in `papers.db` with `rank_score >= config.paperqa.download_rank_threshold`, detects arXiv IDs, and downloads PDFs (polite arXiv API usage).
|
|
103
|
+
- Runs paper-qa to summarize full text into JSON keys: `summary`, `methods`.
|
|
104
|
+
- Writes summaries to `papers.db.entries.paper_qa_summary` only for the specific topic row the item was selected under (no longer cross-updating all topics for the same entry id), and to `matched_entries_history.db.matched_entries.paper_qa_summary`.
|
|
105
|
+
- Prunes archived PDFs older than ~30 days from `assets/paperqa_archive/` after each run to keep storage manageable.
|
|
106
|
+
- Note: This command only updates databases. Use `html` to render pages.
|
|
107
|
+
|
|
108
|
+
- **HTML** (render only; no fetching)
|
|
109
|
+
- `python cli/main.py html [--topic TOPIC]`
|
|
110
|
+
- Reads from `papers.db` and generates, per topic:
|
|
111
|
+
- Filtered page: `output.filename` (if configured)
|
|
112
|
+
- Ranked page: `output.filename_ranked` (if configured and entries exist)
|
|
113
|
+
- Summary page: `output.filename_summary` (if configured). Content priority: PDF summaries → LLM summaries → abstract-only fallback; always ordered by rank.
|
|
114
|
+
|
|
115
|
+
### Summary pages
|
|
116
|
+
|
|
117
|
+
- When `output.filename_summary` is set for a topic, summary pages prefer content in this order:
|
|
118
|
+
1) `paper_qa_summary` (PDF-based)
|
|
119
|
+
2) `llm_summary`
|
|
120
|
+
3) Fallback to ranked fields (abstract → summary)
|
|
121
|
+
- Entries are ordered by descending `rank_score`.
|
|
122
|
+
|
|
123
|
+
- **Purge**
|
|
124
|
+
- `python cli/main.py purge --days N` removes entries with `published_date` within the most recent N days in the seen entries DB.
|
|
125
|
+
- `python cli/main.py purge --all` deletes all DB files and reinitializes schemas (no confirmation prompt).
|
|
126
|
+
|
|
127
|
+
- **Status**
|
|
128
|
+
- `python cli/main.py status`
|
|
129
|
+
- Validates config, lists topics/feeds, and shows DB paths.
|
|
130
|
+
|
|
131
|
+
- **Email** (Mailman digest)
|
|
132
|
+
- `python cli/main.py email [--topic TOPIC] [--mode auto|ranked] [--limit N] [--dry-run]`
|
|
133
|
+
- Builds an email‑friendly HTML digest from `papers.db` and sends via SMTP (SSL).
|
|
134
|
+
- `--mode auto` renders a ranked‑style list directly from `papers.db`; `--mode ranked` embeds the pre‑generated ranked HTML if present, otherwise falls back to the ranked‑style list.
|
|
135
|
+
- `--dry-run` writes a preview HTML to `assets/` instead of sending.
|
|
136
|
+
- Per‑recipient routing: `python cli/main.py email --recipients config/secrets/mailing_lists.yaml` or set `config.email.recipients_file`.
|
|
137
|
+
|
|
138
|
+
### Email configuration
|
|
139
|
+
|
|
140
|
+
Add an `email` section in `config/config.yaml` (secrets in a separate file):
|
|
141
|
+
|
|
142
|
+
```
|
|
143
|
+
email:
|
|
144
|
+
to: "LIST_ADDRESS@yourdomain" # Mailman list address
|
|
145
|
+
subject_prefix: "Paper Firehose" # Optional
|
|
146
|
+
from: "mail@xyz.com" # Defaults to smtp.username
|
|
147
|
+
smtp:
|
|
148
|
+
host: "mail.xyz.com"
|
|
149
|
+
port: 465
|
|
150
|
+
username: "youraccount"
|
|
151
|
+
password_file: "config/secrets/email_password.txt" # store only the password here
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Notes
|
|
155
|
+
- Store the SMTP password in `config/secrets/email_password.txt` (gitignored).
|
|
156
|
+
- The command prefers to be run after `filter`, `rank`, `abstracts`, and `summarize` so it can include LLM summaries when available.
|
|
157
|
+
|
|
158
|
+
Per‑recipient YAML (config/secrets/mailing_lists.yaml)
|
|
159
|
+
Example:
|
|
160
|
+
```
|
|
161
|
+
recipients:
|
|
162
|
+
- to: "materials-list@nemeslab.com"
|
|
163
|
+
topics: ["primary", "perovskites"]
|
|
164
|
+
min_rank_score: 0.40
|
|
165
|
+
- to: "2d-list@nemeslab.com"
|
|
166
|
+
topics: ["rg", "2d_metals"]
|
|
167
|
+
min_rank_score: 0.35
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Python API
|
|
171
|
+
|
|
172
|
+
You can call the main steps programmatically via `paper_firehose`.
|
|
173
|
+
|
|
174
|
+
Basics
|
|
175
|
+
- `import paper_firehose as pf`
|
|
176
|
+
- All functions default to `config/config.yaml`; override with `config_path="..."`.
|
|
177
|
+
|
|
178
|
+
Functions
|
|
179
|
+
- `pf.filter(topic=None, config_path=None)`: Runs the filter step for one topic or all.
|
|
180
|
+
- `pf.rank(topic=None, config_path=None)`: Computes and writes `rank_score` for entries.
|
|
181
|
+
- `pf.abstracts(topic=None, *, mailto=None, limit=None, rps=None, config_path=None)`: Fetches abstracts for above‑threshold entries and writes to DBs.
|
|
182
|
+
- `pf.summarize(topic=None, *, rps=None, config_path=None)`: Runs LLM summarization for top‐ranked entries per topic, writing JSON (or text) to `llm_summary`.
|
|
183
|
+
- `pf.pqa_summary(topic=None, *, rps=None, limit=None, arxiv=None, entry_ids=None, use_history=False, history_date=None, history_feed_like=None, config_path=None)`: Runs the paper-qa PDF summarizer with the same parameters as the CLI command.
|
|
184
|
+
- `pf.html(topic=None, output_path=None, config_path=None)`: Regenerates the HTML pages directly from `papers.db`. When `topic` is omitted, all topics are rendered to their configured filenames.
|
|
185
|
+
- `pf.email(topic=None, *, mode='auto', limit=None, recipients_file=None, dry_run=False, config_path=None)`: Sends the digest email (or writes a preview on `dry_run`).
|
|
186
|
+
- `pf.purge(days=None, all_data=False, config_path=None)`: Purges entries based on publication date. When `days` is provided, removes entries from the most recent N days (including today) across all DBs; when `all_data=True`, reinitializes all DBs.
|
|
187
|
+
- `pf.status(config_path=None) -> dict`: Returns configuration validity, available topics, enabled feed count, and database paths.
|
|
188
|
+
|
|
189
|
+
## History Viewer
|
|
190
|
+
|
|
191
|
+
- `history_viewer.html` is a static browser viewer for `assets/matched_entries_history.db` (table `matched_entries`).
|
|
192
|
+
- By default it auto-loads the latest history DB from GitHub:
|
|
193
|
+
- Displayed: `https://github.com/zrbyte/paper-firehose/tree/data/assets/matched_entries_history.latest.db`
|
|
194
|
+
- The viewer automatically normalizes GitHub page links to their raw content (e.g., `raw.githubusercontent.com`) before fetching.
|
|
195
|
+
- You can override with a query param or local file:
|
|
196
|
+
- `history_viewer.html?db=<url>` to load a specific remote DB
|
|
197
|
+
- Use the file input or drag-and-drop a local `matched_entries_history.db`
|
|
198
|
+
|
|
199
|
+
- `history_viewer_cards.html` provides a cleaner, card‑style view of history entries with just the key fields (title, authors, feed name, abstract, matched date). It supports the same controls and query params as `history_viewer.html` (topic, order, search, `?db=<url>` and file drag‑and‑drop) but focuses on readability instead of tabular data.
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
## Continuous delivery via GitHub Actions
|
|
203
|
+
|
|
204
|
+
The workflow **Build and Deploy (Runtime Data Dir Test)** defined in `.github/workflows/pages.yml` drives the hosted pipeline:
|
|
205
|
+
|
|
206
|
+
- Checks out the repository and prepares a runtime directory at `$GITHUB_WORKSPACE/.paper_firehose` (matching local defaults).
|
|
207
|
+
- Restores cached SQLite databases from the `data` branch so runs build on prior state rather than starting from empty tables.
|
|
208
|
+
- Installs dependencies, seeds secrets, and runs the CLI sequence: `filter`, `rank`, `abstracts`, optional `pqa_summary`, optional `email`, and finally `html` to refresh HTML artifacts.
|
|
209
|
+
- Publishes generated HTML via GitHub Pages and commits the updated databases plus rotated history snapshots back to the `data` branch.
|
|
210
|
+
- Accepts workflow dispatch inputs (`run_pqa`, `run_email`) so heavier steps can be toggled without editing the workflow. Useful for debugging.
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
## Architecture overview
|
|
214
|
+
|
|
215
|
+
- Three-DB architecture:
|
|
216
|
+
- `assets/all_feed_entries.db`: Every fetched item (for deduplication).
|
|
217
|
+
- `assets/matched_entries_history.db`: All matched items across topics and runs (historical archive).
|
|
218
|
+
- `assets/papers.db`: Current-run working set (filtered → ranked → summarized).
|
|
219
|
+
- YAML-driven configuration for feeds and topics.
|
|
220
|
+
- HTML generated from `papers.db` so you can re-render without refetching.
|
|
221
|
+
- Optional LLM summarization writes JSON summaries to DB and renders dedicated summary pages.
|
|
222
|
+
|
|
223
|
+
### Bundled system assets
|
|
224
|
+
|
|
225
|
+
- The directory `src/paper_firehose/system/` is the canonical source for starter config files, topic examples, HTML templates, and vendored models.
|
|
226
|
+
- `paper_firehose.core.paths.ensure_data_dir()` provisions a runtime directory (default `~/.paper_firehose`). Missing files are copied from the system bundle exactly once, enabling local overrides that survive upgrades.
|
|
227
|
+
- To refresh a single asset, delete it from the runtime directory and rerun any CLI command; the latest version from `system/` will be re-seeded automatically.
|
|
228
|
+
|
|
229
|
+
### Databases
|
|
230
|
+
|
|
231
|
+
- `all_feed_entries.db` (table `feed_entries`)
|
|
232
|
+
- Keys: `entry_id` (pk), `feed_name` (display name from `config.yaml`), `title`, `link`.
|
|
233
|
+
- Metadata: `summary`, `authors`, `published_date`, `first_seen`, `last_seen`, `raw_data` (JSON).
|
|
234
|
+
- Used only for dedup; populated after filtering completes.
|
|
235
|
+
|
|
236
|
+
- `matched_entries_history.db` (table `matched_entries`)
|
|
237
|
+
- Keys: `entry_id` (pk), `feed_name`, `topics` (CSV of topic names).
|
|
238
|
+
- Metadata: `title`, `link`, `summary`, `authors`, `abstract` (nullable), `doi` (nullable), `published_date`, `matched_date`, `raw_data` (JSON), `llm_summary` (nullable), `paper_qa_summary` (nullable).
|
|
239
|
+
- Written only when a topic’s `output.archive: true`.
|
|
240
|
+
|
|
241
|
+
- `papers.db` (table `entries`)
|
|
242
|
+
- Primary key: composite `PRIMARY KEY(id, topic)` so the same entry can appear once per topic.
|
|
243
|
+
- Columns: `id`, `topic`, `feed_name` (display name), `title`, `link`, `summary`, `authors`, `abstract` (nullable), `doi` (nullable), `published_date`, `discovered_date`, `status` (`filtered|ranked|summarized`), `rank_score`, `rank_reasoning`, `llm_summary`, `raw_data` (JSON).
|
|
244
|
+
|
|
245
|
+
Notes
|
|
246
|
+
- `feed_name` is the human-readable name from `config.yaml -> feeds.<key>.name` (e.g., "Nature Physics").
|
|
247
|
+
- `doi` is best-effort, fetched from RSS feed and can be found in `doi`, `dc:identifier`, `prism:doi`, `id`, `link`, `summary`, `summary_detail.value`, or embedded `content[].value`. arXiv feeds may not include DOIs; no external lookup is performed (by design for now).
|
|
248
|
+
- `abstract` populated via Crossref API or publisher APIs.
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
## Configuration
|
|
252
|
+
|
|
253
|
+
- `config/config.yaml` (feeds, DB paths, defaults)
|
|
254
|
+
- Each feed has a key and a display `name`; the key is used in topic files, the name is stored in DBs.
|
|
255
|
+
- `paperqa`: settings for the arXiv downloader (Phase 1)
|
|
256
|
+
- `download_rank_threshold`: minimum `rank_score` to download (default 0.35)
|
|
257
|
+
- `rps`: requests/second throttle (default 0.3; ~1 request/3.3s per arXiv API guidance)
|
|
258
|
+
- `max_retries`: per-item retry attempts on transient errors
|
|
259
|
+
- `prompt`: paper-qa question used for summarization; should instruct the model to return only JSON with keys `summary`, `methods` (supports `{ranking_query}` placeholder)
|
|
260
|
+
- `config/topics/<topic>.yaml`
|
|
261
|
+
- `feeds`: list of feed keys from `config.yaml`.
|
|
262
|
+
- `filter.pattern` and `filter.fields`: regex and fields to match (defaults include `title` and `summary`).
|
|
263
|
+
- `ranking`: optional `query`, `model`, cutoffs, etc. (for the rank command).
|
|
264
|
+
- Optional: `negative_queries` (list), `preferred_authors` (list of names), `priority_author_boost` (float, e.g., 0.1).
|
|
265
|
+
- `output.filename` and `output.filename_ranked`: HTML output; `archive: true` enables history DB writes.
|
|
266
|
+
- `llm_summary`: topic-level controls for LLM summarization.
|
|
267
|
+
- `enabled: true|false`
|
|
268
|
+
- `prompt`: instruction given to the model. You can reference `{ranking_query}` and it will be replaced with the topic’s `ranking.query`.
|
|
269
|
+
- `score_cutoff`: minimum `rank_score` to consider (0.0–1.0)
|
|
270
|
+
- `top_n`: hard cap on the number of entries considered (after filtering by score)
|
|
271
|
+
- Works together with global `config.llm` below.
|
|
272
|
+
|
|
273
|
+
- `config.llm` (global model settings)
|
|
274
|
+
- `model`: preferred chat model id
|
|
275
|
+
- `model_fallback`: secondary model if the primary is unsupported/unavailable
|
|
276
|
+
- `api_key_env`: environment variable name to read if `openaikulcs.env` is missing
|
|
277
|
+
- `rps`: default requests/second throttle for summarization
|
|
278
|
+
- `max_retries`: retry attempts per item on transient errors
|
|
279
|
+
- Optional GPT‑5 parameters: `verbosity`, `reasoning_effort` (used when the model starts with `gpt-5`)
|
|
280
|
+
|
|
281
|
+
Thank you to arXiv for use of its open access interoperability. The script does not serve pdf's or arxiv preprints, but serves up a link to the filtered article.
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
# Paper Firehose
|
|
2
|
+
|
|
3
|
+
- Fetches academic RSS feeds, filters entries with per-topic regex, and writes results into SQLite databases. HTML pages with the results are rendered directly from the database.
|
|
4
|
+
- Results are ranked by cosine similarity to a set of user defined keywords. Configurable list of authors can get a ranking boost. So papers from your friends / competitors can be boosted in ranking.
|
|
5
|
+
- Highest ranked results are summarized by an LLM. For summarization to work, you need an OpenAI API key. Full text summarization uses [Paper-qa](https://github.com/Future-House/paper-qa).
|
|
6
|
+
- New search terms can be created by simply adding a yaml config file under: `config/topics/your_topic_name.yaml`. Look at the other topics for guidance.
|
|
7
|
+
- The repository ships a self-contained `system/` bundle (configs, templates, sample topics). On first run these files are copied into your runtime data directory (default `~/.paper_firehose` or the path in `PAPER_FIREHOSE_DATA_DIR`) so you can customize them with your own search terms, RSS feed sources, etc.
|
|
8
|
+
- GitHub Actions–only YAML config lives in `github_actions_config/`. When the workflow sets `PAPER_FIREHOSE_DATA_DIR`, those files are synced into `${PAPER_FIREHOSE_DATA_DIR}/config/`, keeping the published package generic while letting GitHub Actions runs use repo-specific overrides. So if you want to use this via GA, set your preferences in this directory.
|
|
9
|
+
- Daily automation runs through GitHub Actions (`.github/workflows/pages.yml`). The workflow restores database snapshots from the `data` branch, executes the CLI pipeline (`filter → rank → abstracts → html` plus optional `pqa_summary`/`email`), and publishes refreshed HTML/SQLite artifacts back to GitHub Pages and the `data` branch.
|
|
10
|
+
|
|
11
|
+
Written using Python 3.11.
|
|
12
|
+
For dependencies check `requirements.txt`.
|
|
13
|
+
OpenAI API key is searched for in the `openaikulcs.env` file in the repo root or `OPENAI_API_KEY` environment variable.
|
|
14
|
+
|
|
15
|
+
## CLI
|
|
16
|
+
|
|
17
|
+
- **Filter**: fetch from RSS feed list, dedup, match regex, write DBs, render HTML
|
|
18
|
+
- `python cli/main.py -v filter [--topic TOPIC]`
|
|
19
|
+
- Backs up `all_feed_entries.db` and `matched_entries_history.db` (keeps 3 latest backups).
|
|
20
|
+
- Clears `papers.db` working table before processing this run.
|
|
21
|
+
- Fetches configured feeds for each topic, dedups by title against `all_feed_entries.db`, filters by topic regex.
|
|
22
|
+
- Writes matches to `papers.db` (`status='filtered'`); optionally archives to `matched_entries_history.db` if `output.archive: true`.
|
|
23
|
+
- Saves ALL processed entries (matched and non-matched) to `all_feed_entries.db` for future dedup.
|
|
24
|
+
- Renders per-topic HTML from `papers.db`.
|
|
25
|
+
|
|
26
|
+
- **Rank**: ranks fetched papers by title, wrt keywords in yaml config (optional)
|
|
27
|
+
- `python cli/main.py rank [--topic TOPIC]`
|
|
28
|
+
- Computes and writes `rank_score` for `papers.db` entries using sentence-transformers. HTML files with ranked entries are generated.
|
|
29
|
+
- Model selection: if `models/all-MiniLM-L6-v2` exists, it is used; otherwise it falls back to the Hugging Face repo id `all-MiniLM-L6-v2` and downloads once into cache. You can vendor the model with `python scripts/vendor_model.py`.
|
|
30
|
+
- Scoring details: applies a small penalty for `ranking.negative_queries` matches (title/summary). Optional boosts: per-topic `ranking.preferred_authors` with `ranking.priority_author_boost`, and global `priority_journal_boost` for feeds listed in `priority_journals`.
|
|
31
|
+
|
|
32
|
+
- **Abstracts**: try to fetch abstracts
|
|
33
|
+
- `python cli/main.py abstracts [--topic TOPIC] [--mailto you@example.com] [--limit N] [--rps 1.0]`
|
|
34
|
+
- Order: 1) Fill arXiv/cond-mat abstracts from `summary` (no threshold) 2) Above-threshold: Crossref (DOI, then title) 3) Above-threshold: Semantic Scholar → OpenAlex → PubMed
|
|
35
|
+
- Threshold: topic `abstract_fetch.rank_threshold` else global `defaults.rank_threshold`.
|
|
36
|
+
- Only topics with `abstract_fetch.enabled: true` are processed.
|
|
37
|
+
- Writes to both `papers.db.entries.abstract` and `matched_entries_history.db.matched_entries.abstract`.
|
|
38
|
+
- Rate limiting: descriptive User-Agent (includes `--mailto` or `$MAILTO`), respects Retry-After; default ~1 req/sec via `--rps`.
|
|
39
|
+
- Populates the `entries.abstract` column; leaves other fields unchanged.
|
|
40
|
+
- Contact email: if `--mailto` is not provided, the command reads `$MAILTO` from the environment; if unset, it uses a safe default.
|
|
41
|
+
|
|
42
|
+
- **Summarize** (optional)
|
|
43
|
+
- `python cli/main.py summarize [--topic TOPIC] [--rps 0.5]`
|
|
44
|
+
- Selects top entries per topic based on `llm_summary.score_cutoff` and `llm_summary.top_n`, builds input strictly from `title + abstract` (skips entries without an abstract), and calls the configured OpenAI chat model.
|
|
45
|
+
- Writes summaries to `papers.db.entries.llm_summary` and, when present, `matched_entries_history.db.matched_entries.llm_summary`.
|
|
46
|
+
- Uses `config.llm` settings. Supports JSON or plain-text responses; JSON is preferred and rendered with headings.
|
|
47
|
+
- Note: This command only updates databases. Use `html` to render pages.
|
|
48
|
+
|
|
49
|
+
- **pqa_summary** (PDF-based summarization)
|
|
50
|
+
- `python cli/main.py pqa_summary [--topic TOPIC]`
|
|
51
|
+
- Selects preprints from arXiv in `papers.db` with `rank_score >= config.paperqa.download_rank_threshold`, detects arXiv IDs, and downloads PDFs (polite arXiv API usage).
|
|
52
|
+
- Runs paper-qa to summarize full text into JSON keys: `summary`, `methods`.
|
|
53
|
+
- Writes summaries to `papers.db.entries.paper_qa_summary` only for the specific topic row the item was selected under (no longer cross-updating all topics for the same entry id), and to `matched_entries_history.db.matched_entries.paper_qa_summary`.
|
|
54
|
+
- Prunes archived PDFs older than ~30 days from `assets/paperqa_archive/` after each run to keep storage manageable.
|
|
55
|
+
- Note: This command only updates databases. Use `html` to render pages.
|
|
56
|
+
|
|
57
|
+
- **HTML** (render only; no fetching)
|
|
58
|
+
- `python cli/main.py html [--topic TOPIC]`
|
|
59
|
+
- Reads from `papers.db` and generates, per topic:
|
|
60
|
+
- Filtered page: `output.filename` (if configured)
|
|
61
|
+
- Ranked page: `output.filename_ranked` (if configured and entries exist)
|
|
62
|
+
- Summary page: `output.filename_summary` (if configured). Content priority: PDF summaries → LLM summaries → abstract-only fallback; always ordered by rank.
|
|
63
|
+
|
|
64
|
+
### Summary pages
|
|
65
|
+
|
|
66
|
+
- When `output.filename_summary` is set for a topic, summary pages prefer content in this order:
|
|
67
|
+
1) `paper_qa_summary` (PDF-based)
|
|
68
|
+
2) `llm_summary`
|
|
69
|
+
3) Fallback to ranked fields (abstract → summary)
|
|
70
|
+
- Entries are ordered by descending `rank_score`.
|
|
71
|
+
|
|
72
|
+
- **Purge**
|
|
73
|
+
- `python cli/main.py purge --days N` removes entries with `published_date` within the most recent N days in the seen entries DB.
|
|
74
|
+
- `python cli/main.py purge --all` deletes all DB files and reinitializes schemas (no confirmation prompt).
|
|
75
|
+
|
|
76
|
+
- **Status**
|
|
77
|
+
- `python cli/main.py status`
|
|
78
|
+
- Validates config, lists topics/feeds, and shows DB paths.
|
|
79
|
+
|
|
80
|
+
- **Email** (Mailman digest)
|
|
81
|
+
- `python cli/main.py email [--topic TOPIC] [--mode auto|ranked] [--limit N] [--dry-run]`
|
|
82
|
+
- Builds an email‑friendly HTML digest from `papers.db` and sends via SMTP (SSL).
|
|
83
|
+
- `--mode auto` renders a ranked‑style list directly from `papers.db`; `--mode ranked` embeds the pre‑generated ranked HTML if present, otherwise falls back to the ranked‑style list.
|
|
84
|
+
- `--dry-run` writes a preview HTML to `assets/` instead of sending.
|
|
85
|
+
- Per‑recipient routing: `python cli/main.py email --recipients config/secrets/mailing_lists.yaml` or set `config.email.recipients_file`.
|
|
86
|
+
|
|
87
|
+
### Email configuration
|
|
88
|
+
|
|
89
|
+
Add an `email` section in `config/config.yaml` (secrets in a separate file):
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
email:
|
|
93
|
+
to: "LIST_ADDRESS@yourdomain" # Mailman list address
|
|
94
|
+
subject_prefix: "Paper Firehose" # Optional
|
|
95
|
+
from: "mail@xyz.com" # Defaults to smtp.username
|
|
96
|
+
smtp:
|
|
97
|
+
host: "mail.xyz.com"
|
|
98
|
+
port: 465
|
|
99
|
+
username: "youraccount"
|
|
100
|
+
password_file: "config/secrets/email_password.txt" # store only the password here
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Notes
|
|
104
|
+
- Store the SMTP password in `config/secrets/email_password.txt` (gitignored).
|
|
105
|
+
- The command prefers to be run after `filter`, `rank`, `abstracts`, and `summarize` so it can include LLM summaries when available.
|
|
106
|
+
|
|
107
|
+
Per‑recipient YAML (config/secrets/mailing_lists.yaml)
|
|
108
|
+
Example:
|
|
109
|
+
```
|
|
110
|
+
recipients:
|
|
111
|
+
- to: "materials-list@nemeslab.com"
|
|
112
|
+
topics: ["primary", "perovskites"]
|
|
113
|
+
min_rank_score: 0.40
|
|
114
|
+
- to: "2d-list@nemeslab.com"
|
|
115
|
+
topics: ["rg", "2d_metals"]
|
|
116
|
+
min_rank_score: 0.35
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Python API
|
|
120
|
+
|
|
121
|
+
You can call the main steps programmatically via `paper_firehose`.
|
|
122
|
+
|
|
123
|
+
Basics
|
|
124
|
+
- `import paper_firehose as pf`
|
|
125
|
+
- All functions default to `config/config.yaml`; override with `config_path="..."`.
|
|
126
|
+
|
|
127
|
+
Functions
|
|
128
|
+
- `pf.filter(topic=None, config_path=None)`: Runs the filter step for one topic or all.
|
|
129
|
+
- `pf.rank(topic=None, config_path=None)`: Computes and writes `rank_score` for entries.
|
|
130
|
+
- `pf.abstracts(topic=None, *, mailto=None, limit=None, rps=None, config_path=None)`: Fetches abstracts for above‑threshold entries and writes to DBs.
|
|
131
|
+
- `pf.summarize(topic=None, *, rps=None, config_path=None)`: Runs LLM summarization for top‐ranked entries per topic, writing JSON (or text) to `llm_summary`.
|
|
132
|
+
- `pf.pqa_summary(topic=None, *, rps=None, limit=None, arxiv=None, entry_ids=None, use_history=False, history_date=None, history_feed_like=None, config_path=None)`: Runs the paper-qa PDF summarizer with the same parameters as the CLI command.
|
|
133
|
+
- `pf.html(topic=None, output_path=None, config_path=None)`: Regenerates the HTML pages directly from `papers.db`. When `topic` is omitted, all topics are rendered to their configured filenames.
|
|
134
|
+
- `pf.email(topic=None, *, mode='auto', limit=None, recipients_file=None, dry_run=False, config_path=None)`: Sends the digest email (or writes a preview on `dry_run`).
|
|
135
|
+
- `pf.purge(days=None, all_data=False, config_path=None)`: Purges entries based on publication date. When `days` is provided, removes entries from the most recent N days (including today) across all DBs; when `all_data=True`, reinitializes all DBs.
|
|
136
|
+
- `pf.status(config_path=None) -> dict`: Returns configuration validity, available topics, enabled feed count, and database paths.
|
|
137
|
+
|
|
138
|
+
## History Viewer
|
|
139
|
+
|
|
140
|
+
- `history_viewer.html` is a static browser viewer for `assets/matched_entries_history.db` (table `matched_entries`).
|
|
141
|
+
- By default it auto-loads the latest history DB from GitHub:
|
|
142
|
+
- Displayed: `https://github.com/zrbyte/paper-firehose/tree/data/assets/matched_entries_history.latest.db`
|
|
143
|
+
- The viewer automatically normalizes GitHub page links to their raw content (e.g., `raw.githubusercontent.com`) before fetching.
|
|
144
|
+
- You can override with a query param or local file:
|
|
145
|
+
- `history_viewer.html?db=<url>` to load a specific remote DB
|
|
146
|
+
- Use the file input or drag-and-drop a local `matched_entries_history.db`
|
|
147
|
+
|
|
148
|
+
- `history_viewer_cards.html` provides a cleaner, card‑style view of history entries with just the key fields (title, authors, feed name, abstract, matched date). It supports the same controls and query params as `history_viewer.html` (topic, order, search, `?db=<url>` and file drag‑and‑drop) but focuses on readability instead of tabular data.
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
## Continuous delivery via GitHub Actions
|
|
152
|
+
|
|
153
|
+
The workflow **Build and Deploy (Runtime Data Dir Test)** defined in `.github/workflows/pages.yml` drives the hosted pipeline:
|
|
154
|
+
|
|
155
|
+
- Checks out the repository and prepares a runtime directory at `$GITHUB_WORKSPACE/.paper_firehose` (matching local defaults).
|
|
156
|
+
- Restores cached SQLite databases from the `data` branch so runs build on prior state rather than starting from empty tables.
|
|
157
|
+
- Installs dependencies, seeds secrets, and runs the CLI sequence: `filter`, `rank`, `abstracts`, optional `pqa_summary`, optional `email`, and finally `html` to refresh HTML artifacts.
|
|
158
|
+
- Publishes generated HTML via GitHub Pages and commits the updated databases plus rotated history snapshots back to the `data` branch.
|
|
159
|
+
- Accepts workflow dispatch inputs (`run_pqa`, `run_email`) so heavier steps can be toggled without editing the workflow. Useful for debugging.
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
## Architecture overview
|
|
163
|
+
|
|
164
|
+
- Three-DB architecture:
|
|
165
|
+
- `assets/all_feed_entries.db`: Every fetched item (for deduplication).
|
|
166
|
+
- `assets/matched_entries_history.db`: All matched items across topics and runs (historical archive).
|
|
167
|
+
- `assets/papers.db`: Current-run working set (filtered → ranked → summarized).
|
|
168
|
+
- YAML-driven configuration for feeds and topics.
|
|
169
|
+
- HTML generated from `papers.db` so you can re-render without refetching.
|
|
170
|
+
- Optional LLM summarization writes JSON summaries to DB and renders dedicated summary pages.
|
|
171
|
+
|
|
172
|
+
### Bundled system assets
|
|
173
|
+
|
|
174
|
+
- The directory `src/paper_firehose/system/` is the canonical source for starter config files, topic examples, HTML templates, and vendored models.
|
|
175
|
+
- `paper_firehose.core.paths.ensure_data_dir()` provisions a runtime directory (default `~/.paper_firehose`). Missing files are copied from the system bundle exactly once, enabling local overrides that survive upgrades.
|
|
176
|
+
- To refresh a single asset, delete it from the runtime directory and rerun any CLI command; the latest version from `system/` will be re-seeded automatically.
|
|
177
|
+
|
|
178
|
+
### Databases
|
|
179
|
+
|
|
180
|
+
- `all_feed_entries.db` (table `feed_entries`)
|
|
181
|
+
- Keys: `entry_id` (pk), `feed_name` (display name from `config.yaml`), `title`, `link`.
|
|
182
|
+
- Metadata: `summary`, `authors`, `published_date`, `first_seen`, `last_seen`, `raw_data` (JSON).
|
|
183
|
+
- Used only for dedup; populated after filtering completes.
|
|
184
|
+
|
|
185
|
+
- `matched_entries_history.db` (table `matched_entries`)
|
|
186
|
+
- Keys: `entry_id` (pk), `feed_name`, `topics` (CSV of topic names).
|
|
187
|
+
- Metadata: `title`, `link`, `summary`, `authors`, `abstract` (nullable), `doi` (nullable), `published_date`, `matched_date`, `raw_data` (JSON), `llm_summary` (nullable), `paper_qa_summary` (nullable).
|
|
188
|
+
- Written only when a topic’s `output.archive: true`.
|
|
189
|
+
|
|
190
|
+
- `papers.db` (table `entries`)
|
|
191
|
+
- Primary key: composite `PRIMARY KEY(id, topic)` so the same entry can appear once per topic.
|
|
192
|
+
- Columns: `id`, `topic`, `feed_name` (display name), `title`, `link`, `summary`, `authors`, `abstract` (nullable), `doi` (nullable), `published_date`, `discovered_date`, `status` (`filtered|ranked|summarized`), `rank_score`, `rank_reasoning`, `llm_summary`, `raw_data` (JSON).
|
|
193
|
+
|
|
194
|
+
Notes
|
|
195
|
+
- `feed_name` is the human-readable name from `config.yaml -> feeds.<key>.name` (e.g., "Nature Physics").
|
|
196
|
+
- `doi` is best-effort, fetched from RSS feed and can be found in `doi`, `dc:identifier`, `prism:doi`, `id`, `link`, `summary`, `summary_detail.value`, or embedded `content[].value`. arXiv feeds may not include DOIs; no external lookup is performed (by design for now).
|
|
197
|
+
- `abstract` populated via Crossref API or publisher APIs.
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
## Configuration
|
|
201
|
+
|
|
202
|
+
- `config/config.yaml` (feeds, DB paths, defaults)
|
|
203
|
+
- Each feed has a key and a display `name`; the key is used in topic files, the name is stored in DBs.
|
|
204
|
+
- `paperqa`: settings for the arXiv downloader (Phase 1)
|
|
205
|
+
- `download_rank_threshold`: minimum `rank_score` to download (default 0.35)
|
|
206
|
+
- `rps`: requests/second throttle (default 0.3; ~1 request/3.3s per arXiv API guidance)
|
|
207
|
+
- `max_retries`: per-item retry attempts on transient errors
|
|
208
|
+
- `prompt`: paper-qa question used for summarization; should instruct the model to return only JSON with keys `summary`, `methods` (supports `{ranking_query}` placeholder)
|
|
209
|
+
- `config/topics/<topic>.yaml`
|
|
210
|
+
- `feeds`: list of feed keys from `config.yaml`.
|
|
211
|
+
- `filter.pattern` and `filter.fields`: regex and fields to match (defaults include `title` and `summary`).
|
|
212
|
+
- `ranking`: optional `query`, `model`, cutoffs, etc. (for the rank command).
|
|
213
|
+
- Optional: `negative_queries` (list), `preferred_authors` (list of names), `priority_author_boost` (float, e.g., 0.1).
|
|
214
|
+
- `output.filename` and `output.filename_ranked`: HTML output; `archive: true` enables history DB writes.
|
|
215
|
+
- `llm_summary`: topic-level controls for LLM summarization.
|
|
216
|
+
- `enabled: true|false`
|
|
217
|
+
- `prompt`: instruction given to the model. You can reference `{ranking_query}` and it will be replaced with the topic’s `ranking.query`.
|
|
218
|
+
- `score_cutoff`: minimum `rank_score` to consider (0.0–1.0)
|
|
219
|
+
- `top_n`: hard cap on the number of entries considered (after filtering by score)
|
|
220
|
+
- Works together with global `config.llm` below.
|
|
221
|
+
|
|
222
|
+
- `config.llm` (global model settings)
|
|
223
|
+
- `model`: preferred chat model id
|
|
224
|
+
- `model_fallback`: secondary model if the primary is unsupported/unavailable
|
|
225
|
+
- `api_key_env`: environment variable name to read if `openaikulcs.env` is missing
|
|
226
|
+
- `rps`: default requests/second throttle for summarization
|
|
227
|
+
- `max_retries`: retry attempts per item on transient errors
|
|
228
|
+
- Optional GPT‑5 parameters: `verbosity`, `reasoning_effort` (used when the model starts with `gpt-5`)
|
|
229
|
+
|
|
230
|
+
Thank you to arXiv for use of its open access interoperability. The script does not serve pdf's or arxiv preprints, but serves up a link to the filtered article.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = [
|
|
3
|
+
"setuptools>=68",
|
|
4
|
+
"wheel",
|
|
5
|
+
]
|
|
6
|
+
build-backend = "setuptools.build_meta"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "paper_firehose"
|
|
10
|
+
version = "0.1.0"
|
|
11
|
+
description = "Filter, rank, and summarize research-paper RSS feeds."
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
license = { file = "LICENSE" }
|
|
14
|
+
authors = [
|
|
15
|
+
{ name = "Peter Nemes-Incze" },
|
|
16
|
+
]
|
|
17
|
+
requires-python = ">=3.10"
|
|
18
|
+
dependencies = [
|
|
19
|
+
"feedparser",
|
|
20
|
+
"PyYAML",
|
|
21
|
+
"click",
|
|
22
|
+
"sentence-transformers>=2.2.2",
|
|
23
|
+
"requests",
|
|
24
|
+
"openai>=1.30.0",
|
|
25
|
+
"paper-qa",
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Intended Audience :: Science/Research",
|
|
29
|
+
"License :: OSI Approved :: MIT License",
|
|
30
|
+
"Operating System :: OS Independent",
|
|
31
|
+
"Programming Language :: Python :: 3",
|
|
32
|
+
"Programming Language :: Python :: 3.10",
|
|
33
|
+
"Programming Language :: Python :: 3.11",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
35
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
36
|
+
]
|
|
37
|
+
keywords = ["rss", "llm", "research", "papers", "arxiv", "summaries"]
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
dev = [
|
|
41
|
+
"pytest>=7.0",
|
|
42
|
+
"pytest-cov",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
paper-firehose = "paper_firehose.cli:cli"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools]
|
|
49
|
+
include-package-data = true
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.setuptools.package-data]
|
|
55
|
+
paper_firehose = [
|
|
56
|
+
"system/**/*",
|
|
57
|
+
]
|