sci-etl-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sci_etl_cli-0.1.0/LICENSE +21 -0
- sci_etl_cli-0.1.0/PKG-INFO +227 -0
- sci_etl_cli-0.1.0/README.md +189 -0
- sci_etl_cli-0.1.0/pyproject.toml +66 -0
- sci_etl_cli-0.1.0/setup.cfg +4 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/__init__.py +1 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/__main__.py +3 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/app.py +70 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/assembly.py +143 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/__init__.py +0 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/init.py +45 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/parse.py +71 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/run.py +135 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/search.py +80 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/status.py +76 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/commands/validate.py +142 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/errors.py +61 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/options.py +35 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/output.py +57 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/py.typed +0 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/reporting.py +30 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/settings.py +164 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/shutdown.py +36 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/templates/config.yaml +40 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/templates/env.example +1 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/templates/extraction.txt +12 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli/templates/relevance.txt +5 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/PKG-INFO +227 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/SOURCES.txt +43 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/dependency_links.txt +1 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/entry_points.txt +2 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/requires.txt +11 -0
- sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/top_level.txt +1 -0
- sci_etl_cli-0.1.0/tests/test_app.py +50 -0
- sci_etl_cli-0.1.0/tests/test_assembly.py +107 -0
- sci_etl_cli-0.1.0/tests/test_errors.py +41 -0
- sci_etl_cli-0.1.0/tests/test_init.py +36 -0
- sci_etl_cli-0.1.0/tests/test_output.py +30 -0
- sci_etl_cli-0.1.0/tests/test_parse.py +69 -0
- sci_etl_cli-0.1.0/tests/test_run.py +98 -0
- sci_etl_cli-0.1.0/tests/test_search.py +63 -0
- sci_etl_cli-0.1.0/tests/test_settings.py +76 -0
- sci_etl_cli-0.1.0/tests/test_shutdown.py +67 -0
- sci_etl_cli-0.1.0/tests/test_status.py +54 -0
- sci_etl_cli-0.1.0/tests/test_validate.py +85 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 sci-etl-cli contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sci-etl-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Command-line tool for running sci-etl-core extraction pipelines from a YAML file.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/xueromll/sci-etl-cli
|
|
7
|
+
Project-URL: Repository, https://github.com/xueromll/sci-etl-cli
|
|
8
|
+
Project-URL: Issues, https://github.com/xueromll/sci-etl-cli/issues
|
|
9
|
+
Keywords: etl,cli,arxiv,llm,text-mining,scientific-literature
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Classifier: Topic :: Text Processing
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: sci-etl-core[async,llm,pdf]<0.2,>=0.1.2
|
|
28
|
+
Requires-Dist: click>=8.2
|
|
29
|
+
Requires-Dist: rich>=13.7
|
|
30
|
+
Requires-Dist: pydantic>=2
|
|
31
|
+
Requires-Dist: python-dotenv
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-mock; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
37
|
+
Dynamic: license-file
|
|
38
|
+
|
|
39
|
+
# sci-etl-cli
|
|
40
|
+
|
|
41
|
+
`sci-etl` runs [sci-etl-core](https://github.com/xueromll/sci-etl-core)
|
|
42
|
+
extraction pipelines from a single YAML file, with no wiring code. It searches
|
|
43
|
+
arXiv, asks an LLM which papers are relevant, extracts structured entities from
|
|
44
|
+
their full text, and upserts them into a CSV. Runs resume where the last one
|
|
45
|
+
stopped.
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
$ sci-etl init hot-jupiters
|
|
49
|
+
$ sci-etl validate hot-jupiters/config.yaml
|
|
50
|
+
$ sci-etl search hot-jupiters/config.yaml --limit 5
|
|
51
|
+
$ sci-etl run hot-jupiters/config.yaml --limit 20
|
|
52
|
+
$ sci-etl status hot-jupiters/config.yaml
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- [Installation](#installation)
|
|
56
|
+
- [Quick start](#quick-start)
|
|
57
|
+
- [Commands](#commands)
|
|
58
|
+
- [Config file](#config-file)
|
|
59
|
+
- [Exit codes](#exit-codes)
|
|
60
|
+
- [Development](#development)
|
|
61
|
+
- [License](#license)
|
|
62
|
+
|
|
63
|
+
## Installation
|
|
64
|
+
|
|
65
|
+
Python 3.10 or newer is required.
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install sci-etl-cli
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
This installs the `sci-etl` command along with `sci-etl-core[async,llm,pdf]`
|
|
72
|
+
0.1.2 or newer, `click`, and `rich`. `pipx install sci-etl-cli` keeps it in an
|
|
73
|
+
environment of its own.
|
|
74
|
+
|
|
75
|
+
## Quick start
|
|
76
|
+
|
|
77
|
+
1. **Create a project.** `sci-etl init hot-jupiters` writes a working example
|
|
78
|
+
that collects hot Jupiter measurements: `config.yaml`, a relevance prompt, an
|
|
79
|
+
extraction prompt, and `.env.example`.
|
|
80
|
+
2. **Add your API key.** Copy `.env.example` to `.env` beside the config and set
|
|
81
|
+
`LLM_API_KEY`. The key is read from the environment or that file, never
|
|
82
|
+
printed, and never written to the config.
|
|
83
|
+
3. **Check the project.** `sci-etl validate hot-jupiters/config.yaml` checks the
|
|
84
|
+
config, the prompts, the key, and the installed packages. Add `--online` to
|
|
85
|
+
also send one arXiv request and one LLM request.
|
|
86
|
+
4. **Tune the query.** `sci-etl search hot-jupiters/config.yaml` lists a page of
|
|
87
|
+
matching papers without calling the LLM.
|
|
88
|
+
5. **Run it.** `sci-etl run hot-jupiters/config.yaml` exports entities to
|
|
89
|
+
`out/planets.csv` and records progress under `state/`. Run the same command
|
|
90
|
+
again to continue; add `--rescan` to pick up papers submitted since.
|
|
91
|
+
|
|
92
|
+
To use the CLI for another subject, change `pipeline.search_query`, both
|
|
93
|
+
prompts, and the `export` columns. The extraction prompt has to ask for JSON
|
|
94
|
+
containing `result_key`, `key_column`, and every value column; `validate` checks
|
|
95
|
+
this.
|
|
96
|
+
|
|
97
|
+
## Commands
|
|
98
|
+
|
|
99
|
+
| Command | What it does | Network | Writes |
|
|
100
|
+
|---------|--------------|---------|--------|
|
|
101
|
+
| `sci-etl init [DIRECTORY]` | Creates a starter project. Refuses to overwrite existing files unless given `--force`. | none | project files |
|
|
102
|
+
| `sci-etl validate CONFIG` | Checks the config, prompts, API key, and packages. `--online` adds one arXiv and one LLM request once the offline checks pass. | only with `--online` | nothing |
|
|
103
|
+
| `sci-etl search CONFIG` | Shows one page of arXiv results, each marked `new` or `processed`. Options: `--query`, `--limit`, `--start-index`, `--json`. | arXiv | nothing |
|
|
104
|
+
| `sci-etl run CONFIG` | Runs the pipeline, resuming from saved state. Options: `--limit`, `--page-size`, `--workers`, `--rescan` or `--start-index`, `--log-file`. | arXiv and the LLM | export CSV, state, log |
|
|
105
|
+
| `sci-etl status CONFIG` | Shows processed records, the saved offset, the last run time, and export rows. `--json` prints JSON. | none | nothing |
|
|
106
|
+
| `sci-etl parse FILE` | Prints the text a bundled parser extracts from a PDF, arXiv e-print, or HTML file. Options: `--format`, `--trim-references`. | none | nothing |
|
|
107
|
+
|
|
108
|
+
Every command accepts `--help`. Logs and errors go to stderr; `search --json`,
|
|
109
|
+
`status --json`, and `parse` print their results to stdout, so they can be
|
|
110
|
+
piped.
|
|
111
|
+
|
|
112
|
+
**Stopping a run.** Press Ctrl+C once: records in flight are cancelled and left
|
|
113
|
+
unmarked, state is saved, and the command exits with 130. The next run retries
|
|
114
|
+
those records. A second Ctrl+C exits immediately.
|
|
115
|
+
|
|
116
|
+
**Resuming.** A run starts at the listing offset saved by the previous one.
|
|
117
|
+
arXiv lists the newest submissions first, so new papers push older ones to
|
|
118
|
+
higher offsets; `--rescan` starts again at offset 0 and skips processed papers
|
|
119
|
+
by id, which costs listing requests but no LLM calls.
|
|
120
|
+
|
|
121
|
+
## Config file
|
|
122
|
+
|
|
123
|
+
A config holds the library's sections (`llm`, `http`, `pipeline`) and the
|
|
124
|
+
CLI's own (`prompts`, `export`, `state`, `logging`). Unknown keys are rejected,
|
|
125
|
+
so a typo fails validation instead of being ignored. Relative paths resolve from
|
|
126
|
+
the config file's folder, so a project behaves the same whichever directory it
|
|
127
|
+
is run from.
|
|
128
|
+
|
|
129
|
+
```yaml
|
|
130
|
+
llm:
|
|
131
|
+
base_url: https://api.openai.com/v1
|
|
132
|
+
model: gpt-4o-mini
|
|
133
|
+
timeout: 120
|
|
134
|
+
api_key_env: LLM_API_KEY
|
|
135
|
+
|
|
136
|
+
http:
|
|
137
|
+
user_agent: "sci-etl-project/0.1 (mailto:you@example.org)"
|
|
138
|
+
max_retries: 4
|
|
139
|
+
backoff_factor: 5.0
|
|
140
|
+
timeout: 25
|
|
141
|
+
|
|
142
|
+
pipeline:
|
|
143
|
+
search_query: 'cat:astro-ph.EP AND abs:"hot Jupiter"'
|
|
144
|
+
max_records: 20
|
|
145
|
+
page_size: 100
|
|
146
|
+
search_delay: 3.0
|
|
147
|
+
sleep_between: 5.0
|
|
148
|
+
max_workers: 4
|
|
149
|
+
|
|
150
|
+
prompts:
|
|
151
|
+
relevance: prompts/relevance.txt
|
|
152
|
+
extraction: prompts/extraction.txt
|
|
153
|
+
result_key: planets
|
|
154
|
+
|
|
155
|
+
export:
|
|
156
|
+
destination: out/planets.csv
|
|
157
|
+
key_column: planet_name
|
|
158
|
+
value_columns: [orbital_period_days, mass_jupiter, radius_jupiter]
|
|
159
|
+
numeric_clip:
|
|
160
|
+
mass_jupiter: [0.0, 80.0]
|
|
161
|
+
|
|
162
|
+
state:
|
|
163
|
+
backend: file
|
|
164
|
+
processed_ids: state/processed_ids.txt
|
|
165
|
+
metadata: state/metadata.json
|
|
166
|
+
|
|
167
|
+
logging:
|
|
168
|
+
file: logs/run.log
|
|
169
|
+
level: INFO
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
| Key | Default | Meaning |
|
|
173
|
+
|-----|---------|---------|
|
|
174
|
+
| `llm.base_url`, `llm.model` | `https://api.openai.com/v1`, `gpt-4o-mini` | Any OpenAI-compatible chat endpoint that supports JSON mode. |
|
|
175
|
+
| `llm.timeout` | `120` | Seconds allowed for each LLM request. |
|
|
176
|
+
| `llm.api_key_env` | `LLM_API_KEY` | Environment variable holding the API key; `.env` beside the config is loaded first. |
|
|
177
|
+
| `http.user_agent` | `sci-etl-core/0.1` | Sent to arXiv. Include a contact address. |
|
|
178
|
+
| `http.max_retries`, `http.backoff_factor` | `3`, `2.0` | Attempts per arXiv request, waiting `backoff_factor ** attempt` seconds between them. arXiv throttles for longer than the defaults allow, so the example uses `4` and `5.0`. |
|
|
179
|
+
| `pipeline.search_query` | required | An [arXiv API query](https://info.arxiv.org/help/api/user-manual.html#query_details). |
|
|
180
|
+
| `pipeline.max_records` | `100` | Relevant records to process per run. `run --limit` overrides it. |
|
|
181
|
+
| `pipeline.page_size` | `100` | Listing entries per request. |
|
|
182
|
+
| `pipeline.search_delay` | `3.0` | Seconds to wait before each listing request, as arXiv asks. |
|
|
183
|
+
| `pipeline.sleep_between` | `5.0` | Seconds to wait between listing pages. |
|
|
184
|
+
| `pipeline.max_workers` | `6` | Records processed at once. |
|
|
185
|
+
| `prompts.relevance` | `prompts/relevance.txt` | System prompt that must ask for `{"relevant": true}` or `{"relevant": false}` in JSON. |
|
|
186
|
+
| `prompts.extraction` | `prompts/extraction.txt` | System prompt that must ask for a JSON object with a list under `result_key`. |
|
|
187
|
+
| `prompts.result_key` | `items` | Key holding the entity list in the extraction reply. |
|
|
188
|
+
| `export.destination` | `out/results.csv` | CSV the entities are upserted into. |
|
|
189
|
+
| `export.key_column` | `name` | Field that identifies an entity; one row is kept per normalized key. |
|
|
190
|
+
| `export.value_columns` | required | Numeric fields to keep. Later papers only fill empty cells. |
|
|
191
|
+
| `export.numeric_clip` | none | `[low, high]` bounds per value column. |
|
|
192
|
+
| `export.escape_formulas` | `true` | Prefix keys that a spreadsheet would run as formulas with an apostrophe. |
|
|
193
|
+
| `state.backend` | `file` | `file` uses `processed_ids` and `metadata`; `sqlite` uses `database`. |
|
|
194
|
+
| `logging.file`, `logging.level` | `logs/run.log`, `INFO` | Log file for `run` (`null` for none) and the level: `DEBUG`, `INFO`, `WARNING`, or `ERROR`. |
|
|
195
|
+
|
|
196
|
+
## Exit codes
|
|
197
|
+
|
|
198
|
+
| Code | Meaning |
|
|
199
|
+
|------|---------|
|
|
200
|
+
| `0` | Success, including a search with no results. |
|
|
201
|
+
| `1` | A run aborted, a request failed, or a file couldn't be parsed. The message names the cause. |
|
|
202
|
+
| `2` | Invalid command-line usage. |
|
|
203
|
+
| `3` | Configuration problem: missing or invalid config, missing prompt, unset API key, or a failed `validate` check. |
|
|
204
|
+
| `4` | A required Python package is missing; the message says what to install. |
|
|
205
|
+
| `130` | A run was interrupted after saving state. |
|
|
206
|
+
|
|
207
|
+
## Development
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
git clone https://github.com/xueromll/sci-etl-cli.git
|
|
211
|
+
cd sci-etl-cli
|
|
212
|
+
python -m venv .venv
|
|
213
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
214
|
+
pip install -e ".[dev]"
|
|
215
|
+
pytest --cov=sci_etl_cli --cov-report=term-missing
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
To work against a local sci-etl-core checkout, install it first with
|
|
219
|
+
`pip install -e "../sci-etl-core[async,llm,pdf]"`.
|
|
220
|
+
|
|
221
|
+
The suite runs offline: arXiv is served by an `httpx.MockTransport` and the LLM
|
|
222
|
+
by a scripted client, so `run` is exercised end to end against real CSV and
|
|
223
|
+
state files. Coverage must stay at 100%.
|
|
224
|
+
|
|
225
|
+
## License
|
|
226
|
+
|
|
227
|
+
Released under the MIT License. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# sci-etl-cli
|
|
2
|
+
|
|
3
|
+
`sci-etl` runs [sci-etl-core](https://github.com/xueromll/sci-etl-core)
|
|
4
|
+
extraction pipelines from a single YAML file, with no wiring code. It searches
|
|
5
|
+
arXiv, asks an LLM which papers are relevant, extracts structured entities from
|
|
6
|
+
their full text, and upserts them into a CSV. Runs resume where the last one
|
|
7
|
+
stopped.
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
$ sci-etl init hot-jupiters
|
|
11
|
+
$ sci-etl validate hot-jupiters/config.yaml
|
|
12
|
+
$ sci-etl search hot-jupiters/config.yaml --limit 5
|
|
13
|
+
$ sci-etl run hot-jupiters/config.yaml --limit 20
|
|
14
|
+
$ sci-etl status hot-jupiters/config.yaml
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
- [Installation](#installation)
|
|
18
|
+
- [Quick start](#quick-start)
|
|
19
|
+
- [Commands](#commands)
|
|
20
|
+
- [Config file](#config-file)
|
|
21
|
+
- [Exit codes](#exit-codes)
|
|
22
|
+
- [Development](#development)
|
|
23
|
+
- [License](#license)
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
Python 3.10 or newer is required.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install sci-etl-cli
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
This installs the `sci-etl` command along with `sci-etl-core[async,llm,pdf]`
|
|
34
|
+
0.1.2 or newer, `click`, and `rich`. `pipx install sci-etl-cli` keeps it in an
|
|
35
|
+
environment of its own.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
1. **Create a project.** `sci-etl init hot-jupiters` writes a working example
|
|
40
|
+
that collects hot Jupiter measurements: `config.yaml`, a relevance prompt, an
|
|
41
|
+
extraction prompt, and `.env.example`.
|
|
42
|
+
2. **Add your API key.** Copy `.env.example` to `.env` beside the config and set
|
|
43
|
+
`LLM_API_KEY`. The key is read from the environment or that file, never
|
|
44
|
+
printed, and never written to the config.
|
|
45
|
+
3. **Check the project.** `sci-etl validate hot-jupiters/config.yaml` checks the
|
|
46
|
+
config, the prompts, the key, and the installed packages. Add `--online` to
|
|
47
|
+
also send one arXiv request and one LLM request.
|
|
48
|
+
4. **Tune the query.** `sci-etl search hot-jupiters/config.yaml` lists a page of
|
|
49
|
+
matching papers without calling the LLM.
|
|
50
|
+
5. **Run it.** `sci-etl run hot-jupiters/config.yaml` exports entities to
|
|
51
|
+
`out/planets.csv` and records progress under `state/`. Run the same command
|
|
52
|
+
again to continue; add `--rescan` to pick up papers submitted since.
|
|
53
|
+
|
|
54
|
+
To use the CLI for another subject, change `pipeline.search_query`, both
|
|
55
|
+
prompts, and the `export` columns. The extraction prompt has to ask for JSON
|
|
56
|
+
containing `result_key`, `key_column`, and every value column; `validate` checks
|
|
57
|
+
this.
|
|
58
|
+
|
|
59
|
+
## Commands
|
|
60
|
+
|
|
61
|
+
| Command | What it does | Network | Writes |
|
|
62
|
+
|---------|--------------|---------|--------|
|
|
63
|
+
| `sci-etl init [DIRECTORY]` | Creates a starter project. Refuses to overwrite existing files unless given `--force`. | none | project files |
|
|
64
|
+
| `sci-etl validate CONFIG` | Checks the config, prompts, API key, and packages. `--online` adds one arXiv and one LLM request once the offline checks pass. | only with `--online` | nothing |
|
|
65
|
+
| `sci-etl search CONFIG` | Shows one page of arXiv results, each marked `new` or `processed`. Options: `--query`, `--limit`, `--start-index`, `--json`. | arXiv | nothing |
|
|
66
|
+
| `sci-etl run CONFIG` | Runs the pipeline, resuming from saved state. Options: `--limit`, `--page-size`, `--workers`, `--rescan` or `--start-index`, `--log-file`. | arXiv and the LLM | export CSV, state, log |
|
|
67
|
+
| `sci-etl status CONFIG` | Shows processed records, the saved offset, the last run time, and export rows. `--json` prints JSON. | none | nothing |
|
|
68
|
+
| `sci-etl parse FILE` | Prints the text a bundled parser extracts from a PDF, arXiv e-print, or HTML file. Options: `--format`, `--trim-references`. | none | nothing |
|
|
69
|
+
|
|
70
|
+
Every command accepts `--help`. Logs and errors go to stderr; `search --json`,
|
|
71
|
+
`status --json`, and `parse` print their results to stdout, so they can be
|
|
72
|
+
piped.
|
|
73
|
+
|
|
74
|
+
**Stopping a run.** Press Ctrl+C once: records in flight are cancelled and left
|
|
75
|
+
unmarked, state is saved, and the command exits with 130. The next run retries
|
|
76
|
+
those records. A second Ctrl+C exits immediately.
|
|
77
|
+
|
|
78
|
+
**Resuming.** A run starts at the listing offset saved by the previous one.
|
|
79
|
+
arXiv lists the newest submissions first, so new papers push older ones to
|
|
80
|
+
higher offsets; `--rescan` starts again at offset 0 and skips processed papers
|
|
81
|
+
by id, which costs listing requests but no LLM calls.
|
|
82
|
+
|
|
83
|
+
## Config file
|
|
84
|
+
|
|
85
|
+
A config holds the library's sections (`llm`, `http`, `pipeline`) and the
|
|
86
|
+
CLI's own (`prompts`, `export`, `state`, `logging`). Unknown keys are rejected,
|
|
87
|
+
so a typo fails validation instead of being ignored. Relative paths resolve from
|
|
88
|
+
the config file's folder, so a project behaves the same whichever directory it
|
|
89
|
+
is run from.
|
|
90
|
+
|
|
91
|
+
```yaml
|
|
92
|
+
llm:
|
|
93
|
+
base_url: https://api.openai.com/v1
|
|
94
|
+
model: gpt-4o-mini
|
|
95
|
+
timeout: 120
|
|
96
|
+
api_key_env: LLM_API_KEY
|
|
97
|
+
|
|
98
|
+
http:
|
|
99
|
+
user_agent: "sci-etl-project/0.1 (mailto:you@example.org)"
|
|
100
|
+
max_retries: 4
|
|
101
|
+
backoff_factor: 5.0
|
|
102
|
+
timeout: 25
|
|
103
|
+
|
|
104
|
+
pipeline:
|
|
105
|
+
search_query: 'cat:astro-ph.EP AND abs:"hot Jupiter"'
|
|
106
|
+
max_records: 20
|
|
107
|
+
page_size: 100
|
|
108
|
+
search_delay: 3.0
|
|
109
|
+
sleep_between: 5.0
|
|
110
|
+
max_workers: 4
|
|
111
|
+
|
|
112
|
+
prompts:
|
|
113
|
+
relevance: prompts/relevance.txt
|
|
114
|
+
extraction: prompts/extraction.txt
|
|
115
|
+
result_key: planets
|
|
116
|
+
|
|
117
|
+
export:
|
|
118
|
+
destination: out/planets.csv
|
|
119
|
+
key_column: planet_name
|
|
120
|
+
value_columns: [orbital_period_days, mass_jupiter, radius_jupiter]
|
|
121
|
+
numeric_clip:
|
|
122
|
+
mass_jupiter: [0.0, 80.0]
|
|
123
|
+
|
|
124
|
+
state:
|
|
125
|
+
backend: file
|
|
126
|
+
processed_ids: state/processed_ids.txt
|
|
127
|
+
metadata: state/metadata.json
|
|
128
|
+
|
|
129
|
+
logging:
|
|
130
|
+
file: logs/run.log
|
|
131
|
+
level: INFO
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
| Key | Default | Meaning |
|
|
135
|
+
|-----|---------|---------|
|
|
136
|
+
| `llm.base_url`, `llm.model` | `https://api.openai.com/v1`, `gpt-4o-mini` | Any OpenAI-compatible chat endpoint that supports JSON mode. |
|
|
137
|
+
| `llm.timeout` | `120` | Seconds allowed for each LLM request. |
|
|
138
|
+
| `llm.api_key_env` | `LLM_API_KEY` | Environment variable holding the API key; `.env` beside the config is loaded first. |
|
|
139
|
+
| `http.user_agent` | `sci-etl-core/0.1` | Sent to arXiv. Include a contact address. |
|
|
140
|
+
| `http.max_retries`, `http.backoff_factor` | `3`, `2.0` | Attempts per arXiv request, waiting `backoff_factor ** attempt` seconds between them. arXiv throttles for longer than the defaults allow, so the example uses `4` and `5.0`. |
|
|
141
|
+
| `pipeline.search_query` | required | An [arXiv API query](https://info.arxiv.org/help/api/user-manual.html#query_details). |
|
|
142
|
+
| `pipeline.max_records` | `100` | Relevant records to process per run. `run --limit` overrides it. |
|
|
143
|
+
| `pipeline.page_size` | `100` | Listing entries per request. |
|
|
144
|
+
| `pipeline.search_delay` | `3.0` | Seconds to wait before each listing request, as arXiv asks. |
|
|
145
|
+
| `pipeline.sleep_between` | `5.0` | Seconds to wait between listing pages. |
|
|
146
|
+
| `pipeline.max_workers` | `6` | Records processed at once. |
|
|
147
|
+
| `prompts.relevance` | `prompts/relevance.txt` | System prompt that must ask for `{"relevant": true}` or `{"relevant": false}` in JSON. |
|
|
148
|
+
| `prompts.extraction` | `prompts/extraction.txt` | System prompt that must ask for a JSON object with a list under `result_key`. |
|
|
149
|
+
| `prompts.result_key` | `items` | Key holding the entity list in the extraction reply. |
|
|
150
|
+
| `export.destination` | `out/results.csv` | CSV the entities are upserted into. |
|
|
151
|
+
| `export.key_column` | `name` | Field that identifies an entity; one row is kept per normalized key. |
|
|
152
|
+
| `export.value_columns` | required | Numeric fields to keep. Later papers only fill empty cells. |
|
|
153
|
+
| `export.numeric_clip` | none | `[low, high]` bounds per value column. |
|
|
154
|
+
| `export.escape_formulas` | `true` | Prefix keys that a spreadsheet would run as formulas with an apostrophe. |
|
|
155
|
+
| `state.backend` | `file` | `file` uses `processed_ids` and `metadata`; `sqlite` uses `database`. |
|
|
156
|
+
| `logging.file`, `logging.level` | `logs/run.log`, `INFO` | Log file for `run` (`null` for none) and the level: `DEBUG`, `INFO`, `WARNING`, or `ERROR`. |
|
|
157
|
+
|
|
158
|
+
## Exit codes
|
|
159
|
+
|
|
160
|
+
| Code | Meaning |
|
|
161
|
+
|------|---------|
|
|
162
|
+
| `0` | Success, including a search with no results. |
|
|
163
|
+
| `1` | A run aborted, a request failed, or a file couldn't be parsed. The message names the cause. |
|
|
164
|
+
| `2` | Invalid command-line usage. |
|
|
165
|
+
| `3` | Configuration problem: missing or invalid config, missing prompt, unset API key, or a failed `validate` check. |
|
|
166
|
+
| `4` | A required Python package is missing; the message says what to install. |
|
|
167
|
+
| `130` | A run was interrupted after saving state. |
|
|
168
|
+
|
|
169
|
+
## Development
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
git clone https://github.com/xueromll/sci-etl-cli.git
|
|
173
|
+
cd sci-etl-cli
|
|
174
|
+
python -m venv .venv
|
|
175
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
176
|
+
pip install -e ".[dev]"
|
|
177
|
+
pytest --cov=sci_etl_cli --cov-report=term-missing
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
To work against a local sci-etl-core checkout, install it first with
|
|
181
|
+
`pip install -e "../sci-etl-core[async,llm,pdf]"`.
|
|
182
|
+
|
|
183
|
+
The suite runs offline: arXiv is served by an `httpx.MockTransport` and the LLM
|
|
184
|
+
by a scripted client, so `run` is exercised end to end against real CSV and
|
|
185
|
+
state files. Coverage must stay at 100%.
|
|
186
|
+
|
|
187
|
+
## License
|
|
188
|
+
|
|
189
|
+
Released under the MIT License. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sci-etl-cli"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Command-line tool for running sci-etl-core extraction pipelines from a YAML file."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
keywords = ["etl", "cli", "arxiv", "llm", "text-mining", "scientific-literature"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Programming Language :: Python :: 3.14",
|
|
26
|
+
"Topic :: Scientific/Engineering",
|
|
27
|
+
"Topic :: Text Processing",
|
|
28
|
+
"Typing :: Typed",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
dependencies = [
|
|
32
|
+
"sci-etl-core[async,llm,pdf]>=0.1.2,<0.2",
|
|
33
|
+
"click>=8.2",
|
|
34
|
+
"rich>=13.7",
|
|
35
|
+
"pydantic>=2",
|
|
36
|
+
"python-dotenv",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
dev = ["pytest", "pytest-asyncio", "pytest-mock", "pytest-cov"]
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
sci-etl = "sci_etl_cli.app:main"
|
|
44
|
+
|
|
45
|
+
[project.urls]
|
|
46
|
+
Homepage = "https://github.com/xueromll/sci-etl-cli"
|
|
47
|
+
Repository = "https://github.com/xueromll/sci-etl-cli"
|
|
48
|
+
Issues = "https://github.com/xueromll/sci-etl-cli/issues"
|
|
49
|
+
|
|
50
|
+
[tool.setuptools]
|
|
51
|
+
package-dir = {"" = "src"}
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
where = ["src"]
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.package-data]
|
|
57
|
+
sci_etl_cli = ["py.typed", "templates/*"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
testpaths = ["tests"]
|
|
61
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
62
|
+
|
|
63
|
+
[tool.coverage.report]
|
|
64
|
+
fail_under = 100
|
|
65
|
+
show_missing = true
|
|
66
|
+
exclude_also = ["if TYPE_CHECKING:"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Command-line tool for running sci-etl-core extraction pipelines from a YAML file."""
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
from sci_etl_core.exceptions import SciEtlError
|
|
8
|
+
|
|
9
|
+
from sci_etl_cli.commands.init import init_command
|
|
10
|
+
from sci_etl_cli.commands.parse import parse_command
|
|
11
|
+
from sci_etl_cli.commands.run import run_command
|
|
12
|
+
from sci_etl_cli.commands.search import search_command
|
|
13
|
+
from sci_etl_cli.commands.status import status_command
|
|
14
|
+
from sci_etl_cli.commands.validate import validate_command
|
|
15
|
+
from sci_etl_cli.errors import to_cli_failure
|
|
16
|
+
|
|
17
|
+
_EXIT_CODES = """\b
|
|
18
|
+
Exit codes:
|
|
19
|
+
0 success
|
|
20
|
+
1 run aborted, or a request or parse failed
|
|
21
|
+
2 invalid command-line usage
|
|
22
|
+
3 configuration problem
|
|
23
|
+
4 missing Python package
|
|
24
|
+
130 interrupted after saving state"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SciEtlGroup(click.Group):
|
|
28
|
+
"""Command group that reports library errors as one line and an exit code."""
|
|
29
|
+
|
|
30
|
+
def invoke(self, ctx: click.Context) -> Any:
|
|
31
|
+
try:
|
|
32
|
+
return super().invoke(ctx)
|
|
33
|
+
except (SciEtlError, ModuleNotFoundError) as error:
|
|
34
|
+
raise to_cli_failure(error) from error
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _print_version(ctx: click.Context, _param: click.Parameter, value: bool) -> None:
|
|
38
|
+
if not value or ctx.resilient_parsing:
|
|
39
|
+
return
|
|
40
|
+
click.echo(f"sci-etl {version('sci-etl-cli')} (sci-etl-core {version('sci-etl-core')})")
|
|
41
|
+
ctx.exit()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@click.group(
|
|
45
|
+
cls=SciEtlGroup,
|
|
46
|
+
context_settings={"help_option_names": ["-h", "--help"]},
|
|
47
|
+
epilog=_EXIT_CODES,
|
|
48
|
+
)
|
|
49
|
+
@click.option(
|
|
50
|
+
"--version",
|
|
51
|
+
is_flag=True,
|
|
52
|
+
expose_value=False,
|
|
53
|
+
is_eager=True,
|
|
54
|
+
callback=_print_version,
|
|
55
|
+
help="Show the CLI and library versions and exit.",
|
|
56
|
+
)
|
|
57
|
+
def cli() -> None:
|
|
58
|
+
"""Run sci-etl-core extraction pipelines from a YAML config file."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
cli.add_command(init_command)
|
|
62
|
+
cli.add_command(validate_command)
|
|
63
|
+
cli.add_command(search_command)
|
|
64
|
+
cli.add_command(run_command)
|
|
65
|
+
cli.add_command(status_command)
|
|
66
|
+
cli.add_command(parse_command)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main() -> None:
|
|
70
|
+
cli(prog_name="sci-etl")
|