arxivscanner 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arxivscanner-0.1.0/MANIFEST.in +1 -0
- arxivscanner-0.1.0/PKG-INFO +163 -0
- arxivscanner-0.1.0/README.md +146 -0
- arxivscanner-0.1.0/arxivscanner/__init__.py +3 -0
- arxivscanner-0.1.0/arxivscanner/__main__.py +5 -0
- arxivscanner-0.1.0/arxivscanner/cli.py +101 -0
- arxivscanner-0.1.0/arxivscanner/display.py +133 -0
- arxivscanner-0.1.0/arxivscanner/fetchers.py +264 -0
- arxivscanner-0.1.0/arxivscanner/models.py +49 -0
- arxivscanner-0.1.0/arxivscanner/taxonomy.py +253 -0
- arxivscanner-0.1.0/arxivscanner.egg-info/PKG-INFO +163 -0
- arxivscanner-0.1.0/arxivscanner.egg-info/SOURCES.txt +18 -0
- arxivscanner-0.1.0/arxivscanner.egg-info/dependency_links.txt +1 -0
- arxivscanner-0.1.0/arxivscanner.egg-info/entry_points.txt +2 -0
- arxivscanner-0.1.0/arxivscanner.egg-info/top_level.txt +1 -0
- arxivscanner-0.1.0/pyproject.toml +34 -0
- arxivscanner-0.1.0/setup.cfg +4 -0
- arxivscanner-0.1.0/tests/fixtures/sample_api.xml +35 -0
- arxivscanner-0.1.0/tests/fixtures/sample_rss.xml +49 -0
- arxivscanner-0.1.0/tests/test_parsers.py +68 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
recursive-include tests *.py *.xml
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arxivscanner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fetch and display new arXiv papers for a domain or subdomain, from the terminal.
|
|
5
|
+
Author-email: Subham <shubham.divakar@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/shubham10divakar/arxivscanner
|
|
7
|
+
Project-URL: Issues, https://github.com/shubham10divakar/arxivscanner/issues
|
|
8
|
+
Keywords: arxiv,papers,research,rss,cli
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# arXiv Scanner
|
|
19
|
+
|
|
20
|
+
A command-line tool that fetches and shows new arXiv papers for a chosen **domain** (archive, for example `cs`) and **subdomain** (category, for example `cs.CV`).
|
|
21
|
+
|
|
22
|
+
It uses only the Python standard library, so there is nothing to `pip install`. It needs Python 3.9 or later and runs on Windows, macOS and Linux.
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
Install from GitHub with pip:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install git+https://github.com/shubham10divakar/arxivscanner.git
|
|
30
|
+
arxivscanner --list # check it works: prints every known domain and subdomain
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
This installs an `arxivscanner` command. `python -m arxivscanner …` works the same way. Use `python3 -m pip` on macOS or Linux if `pip` points at an older Python.
|
|
34
|
+
|
|
35
|
+
To work on the code instead, clone the repo and install it in editable mode:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
git clone https://github.com/shubham10divakar/arxivscanner.git
|
|
39
|
+
cd arxivscanner
|
|
40
|
+
pip install -e .
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Without installing, `python run.py …` from the repo folder also works.
|
|
44
|
+
|
|
45
|
+
## How to use it
|
|
46
|
+
|
|
47
|
+
### 1. Interactive mode (easiest)
|
|
48
|
+
|
|
49
|
+
Run the tool with no arguments and answer the prompts:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
arxivscanner
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
1. **Pick a domain.** Type its number (for example `1` for `cs`).
|
|
56
|
+
2. **Pick one or more subdomains.** Type numbers such as `8` or `8,23`, or type `0` for the whole domain. You can also type arXiv codes directly (`cs.CV cs.LG`).
|
|
57
|
+
3. **Pick a mode.** `1` shows today's announcement. `2` shows papers submitted in the last N days, and asks for N.
|
|
58
|
+
|
|
59
|
+
### 2. Command-line flags
|
|
60
|
+
|
|
61
|
+
Pass `-c` with one or more codes to skip the prompts:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
arxivscanner -c cs.CV # today's Computer Vision list
|
|
65
|
+
arxivscanner -c cs.CV --type new # only brand-new submissions (no cross-lists or updates)
|
|
66
|
+
arxivscanner -c cs.CV cs.LG --short # two subdomains, abstracts trimmed
|
|
67
|
+
arxivscanner -c cs # the whole Computer Science domain
|
|
68
|
+
arxivscanner -c cs.CV --mode recent --days 3 # everything submitted in the last 3 days
|
|
69
|
+
arxivscanner -c cs.CV --md cv.md --json cv.json # also save the results to files
|
|
70
|
+
arxivscanner --from-file saved_feed.xml # parse a previously saved RSS/API XML file offline
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
| Flag | Meaning | Default |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| `-c, --cats CODE …` | Domain(s) or subdomain(s), such as `cs.CV cs.LG`, `cs` or `quant-ph`. Leave it out to get the interactive picker. | — |
|
|
76
|
+
| `--mode today\|recent` | `today` is today's announcement (RSS). `recent` is everything submitted in the last `--days` days (API). | `today` |
|
|
77
|
+
| `--days N` | Window size for `--mode recent`, in whole UTC days, including today. | `3` |
|
|
78
|
+
| `--max N` | Maximum number of papers for `--mode recent`. | `500` |
|
|
79
|
+
| `--type T …` | Keep only these announcement types (today mode): `new`, `cross`, `replace`, `replace-cross`. | all |
|
|
80
|
+
| `--short` | Trim each abstract to about 300 characters. | off |
|
|
81
|
+
| `--json FILE` | Also save the results as JSON (all fields plus the abs and PDF URLs). | — |
|
|
82
|
+
| `--md FILE` | Also save the results as a Markdown reading list. | — |
|
|
83
|
+
| `--from-file XML` | Parse a saved RSS or API XML file instead of fetching. | — |
|
|
84
|
+
| `--list` | Print every built-in domain and subdomain, then exit. | — |
|
|
85
|
+
| `--no-color` | Plain output, for example when piping to a file. `NO_COLOR` is also respected. | — |
|
|
86
|
+
| `--version` | Show the version. | — |
|
|
87
|
+
|
|
88
|
+
### Which mode should I use?
|
|
89
|
+
|
|
90
|
+
| Mode | Source | Answers | Notes |
|
|
91
|
+
|---|---|---|---|
|
|
92
|
+
| `today` (default) | `rss.arxiv.org/rss/<cats>` | "What did arXiv announce today?" | Matches arXiv's daily "new" listing. Each paper is tagged `new`, `cross` (cross-listed from another category), `replace` or `replace-cross` (an updated version of an older paper). |
|
|
93
|
+
| `recent` | `export.arxiv.org/api/query` | "What was submitted in the last N days?" | Filters on submission date (UTC). Also includes author comments (page counts, venue) and journal refs. Capped by `--max`. |
|
|
94
|
+
|
|
95
|
+
**When is there something new?** arXiv announces Sunday to Thursday at 20:00 US Eastern time, which is about **05:30 IST the next morning**. There are no announcements on Friday or Saturday nights US Eastern, so the Saturday and Sunday (IST) feeds are empty. On those days, use `--mode recent --days 3`. `cs.CV` usually has 150–300 papers per announcement.
|
|
96
|
+
|
|
97
|
+
### Example output
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
arXiv · cs.CV (Computer Vision and Pattern Recognition)
|
|
101
|
+
Announcement: Mon, 28 Sep 2026 00:00:00 -0400
|
|
102
|
+
3 papers new: 1 cross: 1 replace: 1
|
|
103
|
+
|
|
104
|
+
1. 2609.00001v1 [new]
|
|
105
|
+
Sample Paper A: Looped Vision Transformers for Fine-Grained Recognition
|
|
106
|
+
Alice Author, Bob Builder, Chandra Kumar
|
|
107
|
+
cs.CV, cs.LG · 2026-09-28
|
|
108
|
+
We study looped vision transformers and show strong results on fine-grained benchmarks.
|
|
109
|
+
https://arxiv.org/abs/2609.00001 https://arxiv.org/pdf/2609.00001
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Codes
|
|
113
|
+
|
|
114
|
+
Run `arxivscanner --list` to see all built-in codes. They cover cs, eess, stat, math, q-bio, q-fin, econ, astro-ph, cond-mat, physics, nlin, quant-ph, gr-qc, hep-th and hep-ph. Any other valid arXiv code also works, even if it isn't in the list. Some common ones:
|
|
115
|
+
|
|
116
|
+
| Code | Subject |
|
|
117
|
+
|---|---|
|
|
118
|
+
| `cs.CV` | Computer Vision and Pattern Recognition |
|
|
119
|
+
| `cs.LG` | Machine Learning |
|
|
120
|
+
| `cs.CL` | Computation and Language (NLP) |
|
|
121
|
+
| `cs.AI` | Artificial Intelligence |
|
|
122
|
+
| `cs.RO` | Robotics |
|
|
123
|
+
| `eess.IV` | Image and Video Processing |
|
|
124
|
+
| `stat.ML` | Machine Learning (Statistics) |
|
|
125
|
+
|
|
126
|
+
### Rate limiting
|
|
127
|
+
|
|
128
|
+
The arXiv API sometimes rate-limits in bursts, answering `406` or `429` for a few minutes. The tool retries with back-off for up to about 4 minutes per request and waits 3 seconds between pages, as arXiv's terms ask. If a later page still fails, it keeps the papers it already fetched and prints a warning. If the first request fails, wait a few minutes and run the command again. The RSS feed (`today` mode) is rarely affected.
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
python -m unittest discover -s tests # offline tests, using the XML fixtures in tests/fixtures
|
|
134
|
+
python -m build # builds dist/arxivscanner-<version>.tar.gz and .whl (pip install build)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The version lives in `arxivscanner/__init__.py` (`__version__`).
|
|
138
|
+
|
|
139
|
+
## Design
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
arxivscanner ──► cli.py ──► fetchers.py ──► arXiv (RSS / API)
|
|
143
|
+
│ │
|
|
144
|
+
│ └─► models.Paper (one normalised record)
|
|
145
|
+
├─► taxonomy.py (domain → subdomain names, picker)
|
|
146
|
+
└─► display.py (terminal view, JSON / Markdown export)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
| Module | Role |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `taxonomy.py` | Built-in map of domains and subdomains, plus the interactive picker. |
|
|
152
|
+
| `models.py` | `Paper` dataclass: id, version, title, authors, abstract, categories, primary category, announce type, dates, comment, journal ref, DOI, abs and PDF URLs. |
|
|
153
|
+
| `fetchers.py` | Two sources, one output type. `fetch_today()` reads the RSS feed. `fetch_recent()` pages through the API. Also handles retries with back-off, the 3 s delay between API calls, de-duplication and type filtering. |
|
|
154
|
+
| `display.py` | Colour terminal output (works in Windows 10+ consoles too) plus `export_json` and `export_markdown`. |
|
|
155
|
+
| `cli.py` | Flags and the interactive picker. `--from-file` parses a saved XML file offline. |
|
|
156
|
+
|
|
157
|
+
## Roadmap
|
|
158
|
+
|
|
159
|
+
- **0.1** (current) Fetch, display and export by domain and subdomain.
|
|
160
|
+
- **0.2** Remember papers already seen (a local SQLite or JSON file) so each run shows only unseen papers.
|
|
161
|
+
- **0.3** Keyword or interest filtering and ranking (title and abstract match, later embeddings).
|
|
162
|
+
- **0.4** Daily automation (a scheduled task or cron) and a digest by email, Telegram or HTML.
|
|
163
|
+
- **0.5** Dashboard view with bookmarking.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# arXiv Scanner
|
|
2
|
+
|
|
3
|
+
A command-line tool that fetches and shows new arXiv papers for a chosen **domain** (archive, for example `cs`) and **subdomain** (category, for example `cs.CV`).
|
|
4
|
+
|
|
5
|
+
It uses only the Python standard library, so there is nothing to `pip install`. It needs Python 3.9 or later and runs on Windows, macOS and Linux.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
Install from GitHub with pip:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install git+https://github.com/shubham10divakar/arxivscanner.git
|
|
13
|
+
arxivscanner --list # check it works: prints every known domain and subdomain
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
This installs an `arxivscanner` command. `python -m arxivscanner …` works the same way. Use `python3 -m pip` on macOS or Linux if `pip` points at an older Python.
|
|
17
|
+
|
|
18
|
+
To work on the code instead, clone the repo and install it in editable mode:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
git clone https://github.com/shubham10divakar/arxivscanner.git
|
|
22
|
+
cd arxivscanner
|
|
23
|
+
pip install -e .
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Without installing, `python run.py …` from the repo folder also works.
|
|
27
|
+
|
|
28
|
+
## How to use it
|
|
29
|
+
|
|
30
|
+
### 1. Interactive mode (easiest)
|
|
31
|
+
|
|
32
|
+
Run the tool with no arguments and answer the prompts:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
arxivscanner
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
1. **Pick a domain.** Type its number (for example `1` for `cs`).
|
|
39
|
+
2. **Pick one or more subdomains.** Type numbers such as `8` or `8,23`, or type `0` for the whole domain. You can also type arXiv codes directly (`cs.CV cs.LG`).
|
|
40
|
+
3. **Pick a mode.** `1` shows today's announcement. `2` shows papers submitted in the last N days, and asks for N.
|
|
41
|
+
|
|
42
|
+
### 2. Command-line flags
|
|
43
|
+
|
|
44
|
+
Pass `-c` with one or more codes to skip the prompts:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
arxivscanner -c cs.CV # today's Computer Vision list
|
|
48
|
+
arxivscanner -c cs.CV --type new # only brand-new submissions (no cross-lists or updates)
|
|
49
|
+
arxivscanner -c cs.CV cs.LG --short # two subdomains, abstracts trimmed
|
|
50
|
+
arxivscanner -c cs # the whole Computer Science domain
|
|
51
|
+
arxivscanner -c cs.CV --mode recent --days 3 # everything submitted in the last 3 days
|
|
52
|
+
arxivscanner -c cs.CV --md cv.md --json cv.json # also save the results to files
|
|
53
|
+
arxivscanner --from-file saved_feed.xml # parse a previously saved RSS/API XML file offline
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
| Flag | Meaning | Default |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| `-c, --cats CODE …` | Domain(s) or subdomain(s), such as `cs.CV cs.LG`, `cs` or `quant-ph`. Leave it out to get the interactive picker. | — |
|
|
59
|
+
| `--mode today\|recent` | `today` is today's announcement (RSS). `recent` is everything submitted in the last `--days` days (API). | `today` |
|
|
60
|
+
| `--days N` | Window size for `--mode recent`, in whole UTC days, including today. | `3` |
|
|
61
|
+
| `--max N` | Maximum number of papers for `--mode recent`. | `500` |
|
|
62
|
+
| `--type T …` | Keep only these announcement types (today mode): `new`, `cross`, `replace`, `replace-cross`. | all |
|
|
63
|
+
| `--short` | Trim each abstract to about 300 characters. | off |
|
|
64
|
+
| `--json FILE` | Also save the results as JSON (all fields plus the abs and PDF URLs). | — |
|
|
65
|
+
| `--md FILE` | Also save the results as a Markdown reading list. | — |
|
|
66
|
+
| `--from-file XML` | Parse a saved RSS or API XML file instead of fetching. | — |
|
|
67
|
+
| `--list` | Print every built-in domain and subdomain, then exit. | — |
|
|
68
|
+
| `--no-color` | Plain output, for example when piping to a file. `NO_COLOR` is also respected. | — |
|
|
69
|
+
| `--version` | Show the version. | — |
|
|
70
|
+
|
|
71
|
+
### Which mode should I use?
|
|
72
|
+
|
|
73
|
+
| Mode | Source | Answers | Notes |
|
|
74
|
+
|---|---|---|---|
|
|
75
|
+
| `today` (default) | `rss.arxiv.org/rss/<cats>` | "What did arXiv announce today?" | Matches arXiv's daily "new" listing. Each paper is tagged `new`, `cross` (cross-listed from another category), `replace` or `replace-cross` (an updated version of an older paper). |
|
|
76
|
+
| `recent` | `export.arxiv.org/api/query` | "What was submitted in the last N days?" | Filters on submission date (UTC). Also includes author comments (page counts, venue) and journal refs. Capped by `--max`. |
|
|
77
|
+
|
|
78
|
+
**When is there something new?** arXiv announces Sunday to Thursday at 20:00 US Eastern time, which is about **05:30 IST the next morning**. There are no announcements on Friday or Saturday nights US Eastern, so the Saturday and Sunday (IST) feeds are empty. On those days, use `--mode recent --days 3`. `cs.CV` usually has 150–300 papers per announcement.
|
|
79
|
+
|
|
80
|
+
### Example output
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
arXiv · cs.CV (Computer Vision and Pattern Recognition)
|
|
84
|
+
Announcement: Mon, 28 Sep 2026 00:00:00 -0400
|
|
85
|
+
3 papers new: 1 cross: 1 replace: 1
|
|
86
|
+
|
|
87
|
+
1. 2609.00001v1 [new]
|
|
88
|
+
Sample Paper A: Looped Vision Transformers for Fine-Grained Recognition
|
|
89
|
+
Alice Author, Bob Builder, Chandra Kumar
|
|
90
|
+
cs.CV, cs.LG · 2026-09-28
|
|
91
|
+
We study looped vision transformers and show strong results on fine-grained benchmarks.
|
|
92
|
+
https://arxiv.org/abs/2609.00001 https://arxiv.org/pdf/2609.00001
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Codes
|
|
96
|
+
|
|
97
|
+
Run `arxivscanner --list` to see all built-in codes. They cover cs, eess, stat, math, q-bio, q-fin, econ, astro-ph, cond-mat, physics, nlin, quant-ph, gr-qc, hep-th and hep-ph. Any other valid arXiv code also works, even if it isn't in the list. Some common ones:
|
|
98
|
+
|
|
99
|
+
| Code | Subject |
|
|
100
|
+
|---|---|
|
|
101
|
+
| `cs.CV` | Computer Vision and Pattern Recognition |
|
|
102
|
+
| `cs.LG` | Machine Learning |
|
|
103
|
+
| `cs.CL` | Computation and Language (NLP) |
|
|
104
|
+
| `cs.AI` | Artificial Intelligence |
|
|
105
|
+
| `cs.RO` | Robotics |
|
|
106
|
+
| `eess.IV` | Image and Video Processing |
|
|
107
|
+
| `stat.ML` | Machine Learning (Statistics) |
|
|
108
|
+
|
|
109
|
+
### Rate limiting
|
|
110
|
+
|
|
111
|
+
The arXiv API sometimes rate-limits in bursts, answering `406` or `429` for a few minutes. The tool retries with back-off for up to about 4 minutes per request and waits 3 seconds between pages, as arXiv's terms ask. If a later page still fails, it keeps the papers it already fetched and prints a warning. If the first request fails, wait a few minutes and run the command again. The RSS feed (`today` mode) is rarely affected.
|
|
112
|
+
|
|
113
|
+
## Development
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
python -m unittest discover -s tests # offline tests, using the XML fixtures in tests/fixtures
|
|
117
|
+
python -m build # builds dist/arxivscanner-<version>.tar.gz and .whl (pip install build)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The version lives in `arxivscanner/__init__.py` (`__version__`).
|
|
121
|
+
|
|
122
|
+
## Design
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
arxivscanner ──► cli.py ──► fetchers.py ──► arXiv (RSS / API)
|
|
126
|
+
│ │
|
|
127
|
+
│ └─► models.Paper (one normalised record)
|
|
128
|
+
├─► taxonomy.py (domain → subdomain names, picker)
|
|
129
|
+
└─► display.py (terminal view, JSON / Markdown export)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
| Module | Role |
|
|
133
|
+
|---|---|
|
|
134
|
+
| `taxonomy.py` | Built-in map of domains and subdomains, plus the interactive picker. |
|
|
135
|
+
| `models.py` | `Paper` dataclass: id, version, title, authors, abstract, categories, primary category, announce type, dates, comment, journal ref, DOI, abs and PDF URLs. |
|
|
136
|
+
| `fetchers.py` | Two sources, one output type. `fetch_today()` reads the RSS feed. `fetch_recent()` pages through the API. Also handles retries with back-off, the 3 s delay between API calls, de-duplication and type filtering. |
|
|
137
|
+
| `display.py` | Colour terminal output (works in Windows 10+ consoles too) plus `export_json` and `export_markdown`. |
|
|
138
|
+
| `cli.py` | Flags and the interactive picker. `--from-file` parses a saved XML file offline. |
|
|
139
|
+
|
|
140
|
+
## Roadmap
|
|
141
|
+
|
|
142
|
+
- **0.1** (current) Fetch, display and export by domain and subdomain.
|
|
143
|
+
- **0.2** Remember papers already seen (a local SQLite or JSON file) so each run shows only unseen papers.
|
|
144
|
+
- **0.3** Keyword or interest filtering and ranking (title and abstract match, later embeddings).
|
|
145
|
+
- **0.4** Daily automation (a scheduled task or cron) and a digest by email, Telegram or HTML.
|
|
146
|
+
- **0.5** Dashboard view with bookmarking.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Command-line entry point and interactive picker."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from . import __version__, display, taxonomy
|
|
10
|
+
from .fetchers import ANNOUNCE_TYPES, FetchError, fetch_recent, fetch_today, filter_types, parse_file
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
14
|
+
p = argparse.ArgumentParser(
|
|
15
|
+
prog="arxivscanner",
|
|
16
|
+
description="Fetch and display new arXiv papers for a domain (e.g. cs) or subdomain (e.g. cs.CV).",
|
|
17
|
+
)
|
|
18
|
+
p.add_argument("-c", "--cats", nargs="+", metavar="CODE",
|
|
19
|
+
help="domain(s) or subdomain(s), e.g. cs.CV cs.LG or cs. Omit for the interactive picker.")
|
|
20
|
+
p.add_argument("--mode", choices=("today", "recent"), default="today",
|
|
21
|
+
help="today = today's announcement (RSS, default); recent = submitted in the last N days (API)")
|
|
22
|
+
p.add_argument("--days", type=int, default=3, help="window for --mode recent (default 3)")
|
|
23
|
+
p.add_argument("--max", type=int, default=500, dest="max_results",
|
|
24
|
+
help="cap on papers for --mode recent (default 500)")
|
|
25
|
+
p.add_argument("--type", nargs="+", choices=ANNOUNCE_TYPES, dest="types",
|
|
26
|
+
help="keep only these announce types (today mode), e.g. --type new cross")
|
|
27
|
+
p.add_argument("--short", action="store_true", help="trim abstracts")
|
|
28
|
+
p.add_argument("--json", metavar="FILE", help="also save results as JSON")
|
|
29
|
+
p.add_argument("--md", metavar="FILE", help="also save results as Markdown")
|
|
30
|
+
p.add_argument("--from-file", metavar="XML", help="parse a saved RSS/API XML file instead of fetching")
|
|
31
|
+
p.add_argument("--list", action="store_true", help="show known domains and subdomains, then exit")
|
|
32
|
+
p.add_argument("--no-color", action="store_true", help="disable colour output")
|
|
33
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
34
|
+
return p
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
38
|
+
args = build_parser().parse_args(argv)
|
|
39
|
+
display.setup_output(color=False if args.no_color else None)
|
|
40
|
+
|
|
41
|
+
if args.list:
|
|
42
|
+
print(taxonomy.format_listing())
|
|
43
|
+
return 0
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
if args.from_file:
|
|
47
|
+
papers, meta = parse_file(Path(args.from_file).read_bytes())
|
|
48
|
+
cats = args.cats or sorted({p.primary_category for p in papers if p.primary_category})
|
|
49
|
+
mode = "today" if "pub_date" in meta else "file"
|
|
50
|
+
else:
|
|
51
|
+
cats, mode, days = _resolve_target(args)
|
|
52
|
+
bad = [cat for cat in cats if not taxonomy.is_valid_code(cat)]
|
|
53
|
+
if bad:
|
|
54
|
+
print(f"Not a valid arXiv code: {', '.join(bad)} (try --list)", file=sys.stderr)
|
|
55
|
+
return 2
|
|
56
|
+
if mode == "today":
|
|
57
|
+
print(f"Fetching today's announcement for {' + '.join(cats)} …", file=sys.stderr)
|
|
58
|
+
papers, meta = fetch_today(cats)
|
|
59
|
+
else:
|
|
60
|
+
if args.days < 1:
|
|
61
|
+
print("--days must be at least 1", file=sys.stderr)
|
|
62
|
+
return 2
|
|
63
|
+
print(f"Querying the arXiv API for {' + '.join(cats)}, last {days} day(s) …", file=sys.stderr)
|
|
64
|
+
papers, meta = fetch_recent(cats, days=days, max_results=args.max_results)
|
|
65
|
+
except (KeyboardInterrupt, EOFError):
|
|
66
|
+
print("\nCancelled.", file=sys.stderr)
|
|
67
|
+
return 130
|
|
68
|
+
except (FetchError, OSError) as e:
|
|
69
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
70
|
+
return 1
|
|
71
|
+
|
|
72
|
+
if args.types:
|
|
73
|
+
papers = filter_types(papers, set(args.types))
|
|
74
|
+
|
|
75
|
+
display.print_header(cats, mode, meta, papers)
|
|
76
|
+
if not papers:
|
|
77
|
+
if mode == "today":
|
|
78
|
+
print("No papers in this feed. arXiv does not announce on Friday/Saturday nights (US Eastern),\n"
|
|
79
|
+
"so weekend feeds are empty; try --mode recent --days 3.")
|
|
80
|
+
else:
|
|
81
|
+
print("No papers found.")
|
|
82
|
+
return 0
|
|
83
|
+
display.print_papers(papers, short=args.short)
|
|
84
|
+
|
|
85
|
+
title = f"arXiv {' + '.join(cats)} — {meta.get('pub_date') or meta.get('end', '')[:10] or mode}"
|
|
86
|
+
if args.json:
|
|
87
|
+
display.export_json(papers, args.json, meta={**meta, "categories": cats, "mode": mode})
|
|
88
|
+
print(f"Saved JSON → {args.json}", file=sys.stderr)
|
|
89
|
+
if args.md:
|
|
90
|
+
display.export_markdown(papers, args.md, title=title)
|
|
91
|
+
print(f"Saved Markdown → {args.md}", file=sys.stderr)
|
|
92
|
+
return 0
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _resolve_target(args: argparse.Namespace):
|
|
96
|
+
if args.cats:
|
|
97
|
+
return args.cats, args.mode, args.days
|
|
98
|
+
cats = taxonomy.pick_categories()
|
|
99
|
+
mode, days = taxonomy.pick_mode()
|
|
100
|
+
print()
|
|
101
|
+
return cats, mode, days
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Colour terminal output plus JSON / Markdown export."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import sys
|
|
8
|
+
import textwrap
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import List, Optional, Sequence
|
|
12
|
+
|
|
13
|
+
from .models import Paper
|
|
14
|
+
from .taxonomy import describe
|
|
15
|
+
|
|
16
|
+
_COLORS = {
|
|
17
|
+
"bold": "1", "dim": "2", "red": "31", "green": "32", "yellow": "33",
|
|
18
|
+
"blue": "34", "magenta": "35", "cyan": "36",
|
|
19
|
+
}
|
|
20
|
+
TYPE_COLOR = {"new": "green", "cross": "cyan", "replace": "yellow", "replace-cross": "magenta"}
|
|
21
|
+
|
|
22
|
+
_use_color = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _enable_windows_vt() -> bool:
|
|
26
|
+
"""Turn on ANSI escape handling in Windows 10+ consoles."""
|
|
27
|
+
try:
|
|
28
|
+
import ctypes
|
|
29
|
+
kernel32 = ctypes.windll.kernel32
|
|
30
|
+
handle = kernel32.GetStdHandle(-11) # STD_OUTPUT_HANDLE
|
|
31
|
+
mode = ctypes.c_uint32()
|
|
32
|
+
if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
|
|
33
|
+
return False
|
|
34
|
+
return bool(kernel32.SetConsoleMode(handle, mode.value | 0x0004)) # ENABLE_VIRTUAL_TERMINAL_PROCESSING
|
|
35
|
+
except Exception:
|
|
36
|
+
return False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def setup_output(color: Optional[bool] = None) -> None:
|
|
40
|
+
"""Make stdout UTF-8 safe and decide whether to use colour."""
|
|
41
|
+
global _use_color
|
|
42
|
+
for stream in (sys.stdout, sys.stderr):
|
|
43
|
+
try:
|
|
44
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
45
|
+
except (AttributeError, ValueError):
|
|
46
|
+
pass
|
|
47
|
+
if color is None:
|
|
48
|
+
color = sys.stdout.isatty() and "NO_COLOR" not in os.environ
|
|
49
|
+
if color and os.name == "nt":
|
|
50
|
+
color = _enable_windows_vt()
|
|
51
|
+
_use_color = bool(color)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def c(text: str, *styles: str) -> str:
|
|
55
|
+
if not _use_color or not styles:
|
|
56
|
+
return text
|
|
57
|
+
codes = ";".join(_COLORS[s] for s in styles)
|
|
58
|
+
return f"\033[{codes}m{text}\033[0m"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _width() -> int:
|
|
62
|
+
return max(60, min(shutil.get_terminal_size((100, 24)).columns, 120))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _authors(authors: Sequence[str], limit: int = 6) -> str:
|
|
66
|
+
if len(authors) <= limit:
|
|
67
|
+
return ", ".join(authors)
|
|
68
|
+
return ", ".join(authors[:limit]) + f", … (+{len(authors) - limit})"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def print_header(cats: Sequence[str], mode: str, meta: dict, papers: List[Paper]) -> None:
|
|
72
|
+
names = ", ".join(f"{cat} ({describe(cat)})" if describe(cat) else cat for cat in cats)
|
|
73
|
+
print(c(f"arXiv · {names}", "bold"))
|
|
74
|
+
if mode == "today":
|
|
75
|
+
when = meta.get("pub_date") or "unknown date"
|
|
76
|
+
print(c(f"Announcement: {when}", "dim"))
|
|
77
|
+
elif mode == "recent":
|
|
78
|
+
print(c(f"Submitted {meta.get('start', '')[:10]} → {meta.get('end', '')[:10]} (UTC); "
|
|
79
|
+
f"{meta.get('total', 0)} matched on arXiv", "dim"))
|
|
80
|
+
if meta.get("warning"):
|
|
81
|
+
print(c(f"Warning: {meta['warning']}", "yellow"))
|
|
82
|
+
counts = Counter(p.announce_type for p in papers if p.announce_type)
|
|
83
|
+
summary = " ".join(c(f"{k}: {v}", TYPE_COLOR.get(k, "bold")) for k, v in counts.most_common())
|
|
84
|
+
print(f"{len(papers)} papers {summary}\n")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def print_papers(papers: List[Paper], short: bool = False) -> None:
|
|
88
|
+
w = _width()
|
|
89
|
+
for i, p in enumerate(papers, 1):
|
|
90
|
+
tag = f"[{p.announce_type}]" if p.announce_type else ""
|
|
91
|
+
head = f"{i:>3}. {p.arxiv_id}{p.version} "
|
|
92
|
+
print(c(head, "bold") + c(tag, TYPE_COLOR.get(p.announce_type, "dim")))
|
|
93
|
+
indent = " "
|
|
94
|
+
for line in textwrap.wrap(p.title, w - len(indent)):
|
|
95
|
+
print(indent + c(line, "bold", "blue"))
|
|
96
|
+
if p.authors:
|
|
97
|
+
print(textwrap.fill(_authors(p.authors), w, initial_indent=indent, subsequent_indent=indent))
|
|
98
|
+
meta = [", ".join(p.categories)]
|
|
99
|
+
if p.published:
|
|
100
|
+
meta.append(p.published[:10])
|
|
101
|
+
if p.comment:
|
|
102
|
+
meta.append(p.comment)
|
|
103
|
+
print(indent + c(textwrap.shorten(" · ".join(meta), w * 2 - len(indent), placeholder=" …"), "dim"))
|
|
104
|
+
if p.abstract:
|
|
105
|
+
abstract = textwrap.shorten(p.abstract, 300, placeholder=" …") if short else p.abstract
|
|
106
|
+
print(textwrap.fill(abstract, w, initial_indent=indent, subsequent_indent=indent))
|
|
107
|
+
print(indent + c(p.abs_url, "cyan") + " " + c(p.pdf_url, "dim"))
|
|
108
|
+
print()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def export_json(papers: List[Paper], path: str, meta: Optional[dict] = None) -> None:
|
|
112
|
+
doc = {"meta": meta or {}, "count": len(papers), "papers": [p.to_dict() for p in papers]}
|
|
113
|
+
Path(path).write_text(json.dumps(doc, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def export_markdown(papers: List[Paper], path: str, title: str = "arXiv papers") -> None:
|
|
117
|
+
out = [f"# {title}", "", f"{len(papers)} papers", ""]
|
|
118
|
+
for i, p in enumerate(papers, 1):
|
|
119
|
+
tag = f" `{p.announce_type}`" if p.announce_type else ""
|
|
120
|
+
out.append(f"## {i}. {p.title}")
|
|
121
|
+
out.append("")
|
|
122
|
+
out.append(f"[{p.arxiv_id}{p.version}]({p.abs_url}) · [PDF]({p.pdf_url}){tag} · {', '.join(p.categories)}")
|
|
123
|
+
out.append("")
|
|
124
|
+
if p.authors:
|
|
125
|
+
out.append(f"*{', '.join(p.authors)}*")
|
|
126
|
+
out.append("")
|
|
127
|
+
if p.comment:
|
|
128
|
+
out.append(f"> {p.comment}")
|
|
129
|
+
out.append("")
|
|
130
|
+
if p.abstract:
|
|
131
|
+
out.append(p.abstract)
|
|
132
|
+
out.append("")
|
|
133
|
+
Path(path).write_text("\n".join(out), encoding="utf-8")
|