sci-etl-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. sci_etl_cli-0.1.0/LICENSE +21 -0
  2. sci_etl_cli-0.1.0/PKG-INFO +227 -0
  3. sci_etl_cli-0.1.0/README.md +189 -0
  4. sci_etl_cli-0.1.0/pyproject.toml +66 -0
  5. sci_etl_cli-0.1.0/setup.cfg +4 -0
  6. sci_etl_cli-0.1.0/src/sci_etl_cli/__init__.py +1 -0
  7. sci_etl_cli-0.1.0/src/sci_etl_cli/__main__.py +3 -0
  8. sci_etl_cli-0.1.0/src/sci_etl_cli/app.py +70 -0
  9. sci_etl_cli-0.1.0/src/sci_etl_cli/assembly.py +143 -0
  10. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/__init__.py +0 -0
  11. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/init.py +45 -0
  12. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/parse.py +71 -0
  13. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/run.py +135 -0
  14. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/search.py +80 -0
  15. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/status.py +76 -0
  16. sci_etl_cli-0.1.0/src/sci_etl_cli/commands/validate.py +142 -0
  17. sci_etl_cli-0.1.0/src/sci_etl_cli/errors.py +61 -0
  18. sci_etl_cli-0.1.0/src/sci_etl_cli/options.py +35 -0
  19. sci_etl_cli-0.1.0/src/sci_etl_cli/output.py +57 -0
  20. sci_etl_cli-0.1.0/src/sci_etl_cli/py.typed +0 -0
  21. sci_etl_cli-0.1.0/src/sci_etl_cli/reporting.py +30 -0
  22. sci_etl_cli-0.1.0/src/sci_etl_cli/settings.py +164 -0
  23. sci_etl_cli-0.1.0/src/sci_etl_cli/shutdown.py +36 -0
  24. sci_etl_cli-0.1.0/src/sci_etl_cli/templates/config.yaml +40 -0
  25. sci_etl_cli-0.1.0/src/sci_etl_cli/templates/env.example +1 -0
  26. sci_etl_cli-0.1.0/src/sci_etl_cli/templates/extraction.txt +12 -0
  27. sci_etl_cli-0.1.0/src/sci_etl_cli/templates/relevance.txt +5 -0
  28. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/PKG-INFO +227 -0
  29. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/SOURCES.txt +43 -0
  30. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/dependency_links.txt +1 -0
  31. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/entry_points.txt +2 -0
  32. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/requires.txt +11 -0
  33. sci_etl_cli-0.1.0/src/sci_etl_cli.egg-info/top_level.txt +1 -0
  34. sci_etl_cli-0.1.0/tests/test_app.py +50 -0
  35. sci_etl_cli-0.1.0/tests/test_assembly.py +107 -0
  36. sci_etl_cli-0.1.0/tests/test_errors.py +41 -0
  37. sci_etl_cli-0.1.0/tests/test_init.py +36 -0
  38. sci_etl_cli-0.1.0/tests/test_output.py +30 -0
  39. sci_etl_cli-0.1.0/tests/test_parse.py +69 -0
  40. sci_etl_cli-0.1.0/tests/test_run.py +98 -0
  41. sci_etl_cli-0.1.0/tests/test_search.py +63 -0
  42. sci_etl_cli-0.1.0/tests/test_settings.py +76 -0
  43. sci_etl_cli-0.1.0/tests/test_shutdown.py +67 -0
  44. sci_etl_cli-0.1.0/tests/test_status.py +54 -0
  45. sci_etl_cli-0.1.0/tests/test_validate.py +85 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 sci-etl-cli contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,227 @@
1
+ Metadata-Version: 2.4
2
+ Name: sci-etl-cli
3
+ Version: 0.1.0
4
+ Summary: Command-line tool for running sci-etl-core extraction pipelines from a YAML file.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/xueromll/sci-etl-cli
7
+ Project-URL: Repository, https://github.com/xueromll/sci-etl-cli
8
+ Project-URL: Issues, https://github.com/xueromll/sci-etl-cli/issues
9
+ Keywords: etl,cli,arxiv,llm,text-mining,scientific-literature
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Scientific/Engineering
22
+ Classifier: Topic :: Text Processing
23
+ Classifier: Typing :: Typed
24
+ Requires-Python: >=3.10
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: sci-etl-core[async,llm,pdf]<0.2,>=0.1.2
28
+ Requires-Dist: click>=8.2
29
+ Requires-Dist: rich>=13.7
30
+ Requires-Dist: pydantic>=2
31
+ Requires-Dist: python-dotenv
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest; extra == "dev"
34
+ Requires-Dist: pytest-asyncio; extra == "dev"
35
+ Requires-Dist: pytest-mock; extra == "dev"
36
+ Requires-Dist: pytest-cov; extra == "dev"
37
+ Dynamic: license-file
38
+
39
+ # sci-etl-cli
40
+
41
+ `sci-etl` runs [sci-etl-core](https://github.com/xueromll/sci-etl-core)
42
+ extraction pipelines from a single YAML file, with no wiring code. It searches
43
+ arXiv, asks an LLM which papers are relevant, extracts structured entities from
44
+ their full text, and upserts them into a CSV. Runs resume where the last one
45
+ stopped.
46
+
47
+ ```text
48
+ $ sci-etl init hot-jupiters
49
+ $ sci-etl validate hot-jupiters/config.yaml
50
+ $ sci-etl search hot-jupiters/config.yaml --limit 5
51
+ $ sci-etl run hot-jupiters/config.yaml --limit 20
52
+ $ sci-etl status hot-jupiters/config.yaml
53
+ ```
54
+
55
+ - [Installation](#installation)
56
+ - [Quick start](#quick-start)
57
+ - [Commands](#commands)
58
+ - [Config file](#config-file)
59
+ - [Exit codes](#exit-codes)
60
+ - [Development](#development)
61
+ - [License](#license)
62
+
63
+ ## Installation
64
+
65
+ Python 3.10 or newer is required.
66
+
67
+ ```bash
68
+ pip install sci-etl-cli
69
+ ```
70
+
71
+ This installs the `sci-etl` command along with `sci-etl-core[async,llm,pdf]`
72
+ 0.1.2 or newer, `click`, and `rich`. `pipx install sci-etl-cli` keeps it in an
73
+ environment of its own.
74
+
75
+ ## Quick start
76
+
77
+ 1. **Create a project.** `sci-etl init hot-jupiters` writes a working example
78
+ that collects hot Jupiter measurements: `config.yaml`, a relevance prompt, an
79
+ extraction prompt, and `.env.example`.
80
+ 2. **Add your API key.** Copy `.env.example` to `.env` beside the config and set
81
+ `LLM_API_KEY`. The key is read from the environment or that file, never
82
+ printed, and never written to the config.
83
+ 3. **Check the project.** `sci-etl validate hot-jupiters/config.yaml` checks the
84
+ config, the prompts, the key, and the installed packages. Add `--online` to
85
+ also send one arXiv request and one LLM request.
86
+ 4. **Tune the query.** `sci-etl search hot-jupiters/config.yaml` lists a page of
87
+ matching papers without calling the LLM.
88
+ 5. **Run it.** `sci-etl run hot-jupiters/config.yaml` exports entities to
89
+ `out/planets.csv` and records progress under `state/`. Run the same command
90
+ again to continue; add `--rescan` to pick up papers submitted since.
91
+
92
+ To use the CLI for another subject, change `pipeline.search_query`, both
93
+ prompts, and the `export` columns. The extraction prompt has to ask for JSON
94
+ containing `result_key`, `key_column`, and every value column; `validate` checks
95
+ this.
96
+
97
+ ## Commands
98
+
99
+ | Command | What it does | Network | Writes |
100
+ |---------|--------------|---------|--------|
101
+ | `sci-etl init [DIRECTORY]` | Creates a starter project. Refuses to overwrite existing files unless given `--force`. | none | project files |
102
+ | `sci-etl validate CONFIG` | Checks the config, prompts, API key, and packages. `--online` adds one arXiv and one LLM request once the offline checks pass. | only with `--online` | nothing |
103
+ | `sci-etl search CONFIG` | Shows one page of arXiv results, each marked `new` or `processed`. Options: `--query`, `--limit`, `--start-index`, `--json`. | arXiv | nothing |
104
+ | `sci-etl run CONFIG` | Runs the pipeline, resuming from saved state. Options: `--limit`, `--page-size`, `--workers`, `--rescan` or `--start-index`, `--log-file`. | arXiv and the LLM | export CSV, state, log |
105
+ | `sci-etl status CONFIG` | Shows processed records, the saved offset, the last run time, and export rows. `--json` prints JSON. | none | nothing |
106
+ | `sci-etl parse FILE` | Prints the text a bundled parser extracts from a PDF, arXiv e-print, or HTML file. Options: `--format`, `--trim-references`. | none | nothing |
107
+
108
+ Every command accepts `--help`. Logs and errors go to stderr; `search --json`,
109
+ `status --json`, and `parse` print their results to stdout, so they can be
110
+ piped.
111
+
112
+ **Stopping a run.** Press Ctrl+C once: records in flight are cancelled and left
113
+ unmarked, state is saved, and the command exits with 130. The next run retries
114
+ those records. A second Ctrl+C exits immediately.
115
+
116
+ **Resuming.** A run starts at the listing offset saved by the previous one.
117
+ arXiv lists the newest submissions first, so new papers push older ones to
118
+ higher offsets; `--rescan` starts again at offset 0 and skips processed papers
119
+ by id, which costs listing requests but no LLM calls.
120
+
121
+ ## Config file
122
+
123
+ A config holds the library's sections (`llm`, `http`, `pipeline`) and the
124
+ CLI's own (`prompts`, `export`, `state`, `logging`). Unknown keys are rejected,
125
+ so a typo fails validation instead of being ignored. Relative paths resolve from
126
+ the config file's folder, so a project behaves the same whichever directory it
127
+ is run from.
128
+
129
+ ```yaml
130
+ llm:
131
+ base_url: https://api.openai.com/v1
132
+ model: gpt-4o-mini
133
+ timeout: 120
134
+ api_key_env: LLM_API_KEY
135
+
136
+ http:
137
+ user_agent: "sci-etl-project/0.1 (mailto:you@example.org)"
138
+ max_retries: 4
139
+ backoff_factor: 5.0
140
+ timeout: 25
141
+
142
+ pipeline:
143
+ search_query: 'cat:astro-ph.EP AND abs:"hot Jupiter"'
144
+ max_records: 20
145
+ page_size: 100
146
+ search_delay: 3.0
147
+ sleep_between: 5.0
148
+ max_workers: 4
149
+
150
+ prompts:
151
+ relevance: prompts/relevance.txt
152
+ extraction: prompts/extraction.txt
153
+ result_key: planets
154
+
155
+ export:
156
+ destination: out/planets.csv
157
+ key_column: planet_name
158
+ value_columns: [orbital_period_days, mass_jupiter, radius_jupiter]
159
+ numeric_clip:
160
+ mass_jupiter: [0.0, 80.0]
161
+
162
+ state:
163
+ backend: file
164
+ processed_ids: state/processed_ids.txt
165
+ metadata: state/metadata.json
166
+
167
+ logging:
168
+ file: logs/run.log
169
+ level: INFO
170
+ ```
171
+
172
+ | Key | Default | Meaning |
173
+ |-----|---------|---------|
174
+ | `llm.base_url`, `llm.model` | `https://api.openai.com/v1`, `gpt-4o-mini` | Any OpenAI-compatible chat endpoint that supports JSON mode. |
175
+ | `llm.timeout` | `120` | Seconds allowed for each LLM request. |
176
+ | `llm.api_key_env` | `LLM_API_KEY` | Environment variable holding the API key; `.env` beside the config is loaded first. |
177
+ | `http.user_agent` | `sci-etl-core/0.1` | Sent to arXiv. Include a contact address. |
178
+ | `http.max_retries`, `http.backoff_factor` | `3`, `2.0` | Attempts per arXiv request, waiting `backoff_factor ** attempt` seconds between them. arXiv throttles for longer than the defaults allow, so the example uses `4` and `5.0`. |
179
+ | `pipeline.search_query` | required | An [arXiv API query](https://info.arxiv.org/help/api/user-manual.html#query_details). |
180
+ | `pipeline.max_records` | `100` | Relevant records to process per run. `run --limit` overrides it. |
181
+ | `pipeline.page_size` | `100` | Listing entries per request. |
182
+ | `pipeline.search_delay` | `3.0` | Seconds to wait before each listing request, as arXiv asks. |
183
+ | `pipeline.sleep_between` | `5.0` | Seconds to wait between listing pages. |
184
+ | `pipeline.max_workers` | `6` | Records processed at once. |
185
+ | `prompts.relevance` | `prompts/relevance.txt` | System prompt that must ask for `{"relevant": true}` or `{"relevant": false}` in JSON. |
186
+ | `prompts.extraction` | `prompts/extraction.txt` | System prompt that must ask for a JSON object with a list under `result_key`. |
187
+ | `prompts.result_key` | `items` | Key holding the entity list in the extraction reply. |
188
+ | `export.destination` | `out/results.csv` | CSV the entities are upserted into. |
189
+ | `export.key_column` | `name` | Field that identifies an entity; one row is kept per normalized key. |
190
+ | `export.value_columns` | required | Numeric fields to keep. Later papers only fill empty cells. |
191
+ | `export.numeric_clip` | none | `[low, high]` bounds per value column. |
192
+ | `export.escape_formulas` | `true` | Prefix keys that a spreadsheet would run as formulas with an apostrophe. |
193
+ | `state.backend` | `file` | `file` uses `processed_ids` and `metadata`; `sqlite` uses `database`. |
194
+ | `logging.file`, `logging.level` | `logs/run.log`, `INFO` | Log file for `run` (`null` for none) and the level: `DEBUG`, `INFO`, `WARNING`, or `ERROR`. |
195
+
196
+ ## Exit codes
197
+
198
+ | Code | Meaning |
199
+ |------|---------|
200
+ | `0` | Success, including a search with no results. |
201
+ | `1` | A run aborted, a request failed, or a file couldn't be parsed. The message names the cause. |
202
+ | `2` | Invalid command-line usage. |
203
+ | `3` | Configuration problem: missing or invalid config, missing prompt, unset API key, or a failed `validate` check. |
204
+ | `4` | A required Python package is missing; the message says what to install. |
205
+ | `130` | A run was interrupted after saving state. |
206
+
207
+ ## Development
208
+
209
+ ```bash
210
+ git clone https://github.com/xueromll/sci-etl-cli.git
211
+ cd sci-etl-cli
212
+ python -m venv .venv
213
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
214
+ pip install -e ".[dev]"
215
+ pytest --cov=sci_etl_cli --cov-report=term-missing
216
+ ```
217
+
218
+ To work against a local sci-etl-core checkout, install it first with
219
+ `pip install -e "../sci-etl-core[async,llm,pdf]"`.
220
+
221
+ The suite runs offline: arXiv is served by an `httpx.MockTransport` and the LLM
222
+ by a scripted client, so `run` is exercised end to end against real CSV and
223
+ state files. Coverage must stay at 100%.
224
+
225
+ ## License
226
+
227
+ Released under the MIT License. See [LICENSE](LICENSE).
@@ -0,0 +1,189 @@
1
+ # sci-etl-cli
2
+
3
+ `sci-etl` runs [sci-etl-core](https://github.com/xueromll/sci-etl-core)
4
+ extraction pipelines from a single YAML file, with no wiring code. It searches
5
+ arXiv, asks an LLM which papers are relevant, extracts structured entities from
6
+ their full text, and upserts them into a CSV. Runs resume where the last one
7
+ stopped.
8
+
9
+ ```text
10
+ $ sci-etl init hot-jupiters
11
+ $ sci-etl validate hot-jupiters/config.yaml
12
+ $ sci-etl search hot-jupiters/config.yaml --limit 5
13
+ $ sci-etl run hot-jupiters/config.yaml --limit 20
14
+ $ sci-etl status hot-jupiters/config.yaml
15
+ ```
16
+
17
+ - [Installation](#installation)
18
+ - [Quick start](#quick-start)
19
+ - [Commands](#commands)
20
+ - [Config file](#config-file)
21
+ - [Exit codes](#exit-codes)
22
+ - [Development](#development)
23
+ - [License](#license)
24
+
25
+ ## Installation
26
+
27
+ Python 3.10 or newer is required.
28
+
29
+ ```bash
30
+ pip install sci-etl-cli
31
+ ```
32
+
33
+ This installs the `sci-etl` command along with `sci-etl-core[async,llm,pdf]`
34
+ 0.1.2 or newer, `click`, and `rich`. `pipx install sci-etl-cli` keeps it in an
35
+ environment of its own.
36
+
37
+ ## Quick start
38
+
39
+ 1. **Create a project.** `sci-etl init hot-jupiters` writes a working example
40
+ that collects hot Jupiter measurements: `config.yaml`, a relevance prompt, an
41
+ extraction prompt, and `.env.example`.
42
+ 2. **Add your API key.** Copy `.env.example` to `.env` beside the config and set
43
+ `LLM_API_KEY`. The key is read from the environment or that file, never
44
+ printed, and never written to the config.
45
+ 3. **Check the project.** `sci-etl validate hot-jupiters/config.yaml` checks the
46
+ config, the prompts, the key, and the installed packages. Add `--online` to
47
+ also send one arXiv request and one LLM request.
48
+ 4. **Tune the query.** `sci-etl search hot-jupiters/config.yaml` lists a page of
49
+ matching papers without calling the LLM.
50
+ 5. **Run it.** `sci-etl run hot-jupiters/config.yaml` exports entities to
51
+ `out/planets.csv` and records progress under `state/`. Run the same command
52
+ again to continue; add `--rescan` to pick up papers submitted since.
53
+
54
+ To use the CLI for another subject, change `pipeline.search_query`, both
55
+ prompts, and the `export` columns. The extraction prompt has to ask for JSON
56
+ containing `result_key`, `key_column`, and every value column; `validate` checks
57
+ this.
58
+
59
+ ## Commands
60
+
61
+ | Command | What it does | Network | Writes |
62
+ |---------|--------------|---------|--------|
63
+ | `sci-etl init [DIRECTORY]` | Creates a starter project. Refuses to overwrite existing files unless given `--force`. | none | project files |
64
+ | `sci-etl validate CONFIG` | Checks the config, prompts, API key, and packages. `--online` adds one arXiv and one LLM request once the offline checks pass. | only with `--online` | nothing |
65
+ | `sci-etl search CONFIG` | Shows one page of arXiv results, each marked `new` or `processed`. Options: `--query`, `--limit`, `--start-index`, `--json`. | arXiv | nothing |
66
+ | `sci-etl run CONFIG` | Runs the pipeline, resuming from saved state. Options: `--limit`, `--page-size`, `--workers`, `--rescan` or `--start-index`, `--log-file`. | arXiv and the LLM | export CSV, state, log |
67
+ | `sci-etl status CONFIG` | Shows processed records, the saved offset, the last run time, and export rows. `--json` prints JSON. | none | nothing |
68
+ | `sci-etl parse FILE` | Prints the text a bundled parser extracts from a PDF, arXiv e-print, or HTML file. Options: `--format`, `--trim-references`. | none | nothing |
69
+
70
+ Every command accepts `--help`. Logs and errors go to stderr; `search --json`,
71
+ `status --json`, and `parse` print their results to stdout, so they can be
72
+ piped.
73
+
74
+ **Stopping a run.** Press Ctrl+C once: records in flight are cancelled and left
75
+ unmarked, state is saved, and the command exits with 130. The next run retries
76
+ those records. A second Ctrl+C exits immediately.
77
+
78
+ **Resuming.** A run starts at the listing offset saved by the previous one.
79
+ arXiv lists the newest submissions first, so new papers push older ones to
80
+ higher offsets; `--rescan` starts again at offset 0 and skips processed papers
81
+ by id, which costs listing requests but no LLM calls.
82
+
83
+ ## Config file
84
+
85
+ A config holds the library's sections (`llm`, `http`, `pipeline`) and the
86
+ CLI's own (`prompts`, `export`, `state`, `logging`). Unknown keys are rejected,
87
+ so a typo fails validation instead of being ignored. Relative paths resolve from
88
+ the config file's folder, so a project behaves the same whichever directory it
89
+ is run from.
90
+
91
+ ```yaml
92
+ llm:
93
+ base_url: https://api.openai.com/v1
94
+ model: gpt-4o-mini
95
+ timeout: 120
96
+ api_key_env: LLM_API_KEY
97
+
98
+ http:
99
+ user_agent: "sci-etl-project/0.1 (mailto:you@example.org)"
100
+ max_retries: 4
101
+ backoff_factor: 5.0
102
+ timeout: 25
103
+
104
+ pipeline:
105
+ search_query: 'cat:astro-ph.EP AND abs:"hot Jupiter"'
106
+ max_records: 20
107
+ page_size: 100
108
+ search_delay: 3.0
109
+ sleep_between: 5.0
110
+ max_workers: 4
111
+
112
+ prompts:
113
+ relevance: prompts/relevance.txt
114
+ extraction: prompts/extraction.txt
115
+ result_key: planets
116
+
117
+ export:
118
+ destination: out/planets.csv
119
+ key_column: planet_name
120
+ value_columns: [orbital_period_days, mass_jupiter, radius_jupiter]
121
+ numeric_clip:
122
+ mass_jupiter: [0.0, 80.0]
123
+
124
+ state:
125
+ backend: file
126
+ processed_ids: state/processed_ids.txt
127
+ metadata: state/metadata.json
128
+
129
+ logging:
130
+ file: logs/run.log
131
+ level: INFO
132
+ ```
133
+
134
+ | Key | Default | Meaning |
135
+ |-----|---------|---------|
136
+ | `llm.base_url`, `llm.model` | `https://api.openai.com/v1`, `gpt-4o-mini` | Any OpenAI-compatible chat endpoint that supports JSON mode. |
137
+ | `llm.timeout` | `120` | Seconds allowed for each LLM request. |
138
+ | `llm.api_key_env` | `LLM_API_KEY` | Environment variable holding the API key; `.env` beside the config is loaded first. |
139
+ | `http.user_agent` | `sci-etl-core/0.1` | Sent to arXiv. Include a contact address. |
140
+ | `http.max_retries`, `http.backoff_factor` | `3`, `2.0` | Attempts per arXiv request, waiting `backoff_factor ** attempt` seconds between them. arXiv throttles for longer than the defaults allow, so the example uses `4` and `5.0`. |
141
+ | `pipeline.search_query` | required | An [arXiv API query](https://info.arxiv.org/help/api/user-manual.html#query_details). |
142
+ | `pipeline.max_records` | `100` | Relevant records to process per run. `run --limit` overrides it. |
143
+ | `pipeline.page_size` | `100` | Listing entries per request. |
144
+ | `pipeline.search_delay` | `3.0` | Seconds to wait before each listing request, as arXiv asks. |
145
+ | `pipeline.sleep_between` | `5.0` | Seconds to wait between listing pages. |
146
+ | `pipeline.max_workers` | `6` | Records processed at once. |
147
+ | `prompts.relevance` | `prompts/relevance.txt` | System prompt that must ask for `{"relevant": true}` or `{"relevant": false}` in JSON. |
148
+ | `prompts.extraction` | `prompts/extraction.txt` | System prompt that must ask for a JSON object with a list under `result_key`. |
149
+ | `prompts.result_key` | `items` | Key holding the entity list in the extraction reply. |
150
+ | `export.destination` | `out/results.csv` | CSV the entities are upserted into. |
151
+ | `export.key_column` | `name` | Field that identifies an entity; one row is kept per normalized key. |
152
+ | `export.value_columns` | required | Numeric fields to keep. Later papers only fill empty cells. |
153
+ | `export.numeric_clip` | none | `[low, high]` bounds per value column. |
154
+ | `export.escape_formulas` | `true` | Prefix keys that a spreadsheet would run as formulas with an apostrophe. |
155
+ | `state.backend` | `file` | `file` uses `processed_ids` and `metadata`; `sqlite` uses `database`. |
156
+ | `logging.file`, `logging.level` | `logs/run.log`, `INFO` | Log file for `run` (`null` for none) and the level: `DEBUG`, `INFO`, `WARNING`, or `ERROR`. |
157
+
158
+ ## Exit codes
159
+
160
+ | Code | Meaning |
161
+ |------|---------|
162
+ | `0` | Success, including a search with no results. |
163
+ | `1` | A run aborted, a request failed, or a file couldn't be parsed. The message names the cause. |
164
+ | `2` | Invalid command-line usage. |
165
+ | `3` | Configuration problem: missing or invalid config, missing prompt, unset API key, or a failed `validate` check. |
166
+ | `4` | A required Python package is missing; the message says what to install. |
167
+ | `130` | A run was interrupted after saving state. |
168
+
169
+ ## Development
170
+
171
+ ```bash
172
+ git clone https://github.com/xueromll/sci-etl-cli.git
173
+ cd sci-etl-cli
174
+ python -m venv .venv
175
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
176
+ pip install -e ".[dev]"
177
+ pytest --cov=sci_etl_cli --cov-report=term-missing
178
+ ```
179
+
180
+ To work against a local sci-etl-core checkout, install it first with
181
+ `pip install -e "../sci-etl-core[async,llm,pdf]"`.
182
+
183
+ The suite runs offline: arXiv is served by an `httpx.MockTransport` and the LLM
184
+ by a scripted client, so `run` is exercised end to end against real CSV and
185
+ state files. Coverage must stay at 100%.
186
+
187
+ ## License
188
+
189
+ Released under the MIT License. See [LICENSE](LICENSE).
@@ -0,0 +1,66 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "sci-etl-cli"
7
+ version = "0.1.0"
8
+ description = "Command-line tool for running sci-etl-core extraction pipelines from a YAML file."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.10"
13
+ keywords = ["etl", "cli", "arxiv", "llm", "text-mining", "scientific-literature"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Environment :: Console",
17
+ "Intended Audience :: Science/Research",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3 :: Only",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Programming Language :: Python :: 3.13",
25
+ "Programming Language :: Python :: 3.14",
26
+ "Topic :: Scientific/Engineering",
27
+ "Topic :: Text Processing",
28
+ "Typing :: Typed",
29
+ ]
30
+
31
+ dependencies = [
32
+ "sci-etl-core[async,llm,pdf]>=0.1.2,<0.2",
33
+ "click>=8.2",
34
+ "rich>=13.7",
35
+ "pydantic>=2",
36
+ "python-dotenv",
37
+ ]
38
+
39
+ [project.optional-dependencies]
40
+ dev = ["pytest", "pytest-asyncio", "pytest-mock", "pytest-cov"]
41
+
42
+ [project.scripts]
43
+ sci-etl = "sci_etl_cli.app:main"
44
+
45
+ [project.urls]
46
+ Homepage = "https://github.com/xueromll/sci-etl-cli"
47
+ Repository = "https://github.com/xueromll/sci-etl-cli"
48
+ Issues = "https://github.com/xueromll/sci-etl-cli/issues"
49
+
50
+ [tool.setuptools]
51
+ package-dir = {"" = "src"}
52
+
53
+ [tool.setuptools.packages.find]
54
+ where = ["src"]
55
+
56
+ [tool.setuptools.package-data]
57
+ sci_etl_cli = ["py.typed", "templates/*"]
58
+
59
+ [tool.pytest.ini_options]
60
+ testpaths = ["tests"]
61
+ asyncio_default_fixture_loop_scope = "function"
62
+
63
+ [tool.coverage.report]
64
+ fail_under = 100
65
+ show_missing = true
66
+ exclude_also = ["if TYPE_CHECKING:"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ """Command-line tool for running sci-etl-core extraction pipelines from a YAML file."""
@@ -0,0 +1,3 @@
1
+ from sci_etl_cli.app import main
2
+
3
+ main()
@@ -0,0 +1,70 @@
1
+ from __future__ import annotations
2
+
3
+ from importlib.metadata import version
4
+ from typing import Any
5
+
6
+ import click
7
+ from sci_etl_core.exceptions import SciEtlError
8
+
9
+ from sci_etl_cli.commands.init import init_command
10
+ from sci_etl_cli.commands.parse import parse_command
11
+ from sci_etl_cli.commands.run import run_command
12
+ from sci_etl_cli.commands.search import search_command
13
+ from sci_etl_cli.commands.status import status_command
14
+ from sci_etl_cli.commands.validate import validate_command
15
+ from sci_etl_cli.errors import to_cli_failure
16
+
17
+ _EXIT_CODES = """\b
18
+ Exit codes:
19
+ 0 success
20
+ 1 run aborted, or a request or parse failed
21
+ 2 invalid command-line usage
22
+ 3 configuration problem
23
+ 4 missing Python package
24
+ 130 interrupted after saving state"""
25
+
26
+
27
+ class SciEtlGroup(click.Group):
28
+ """Command group that reports library errors as one line and an exit code."""
29
+
30
+ def invoke(self, ctx: click.Context) -> Any:
31
+ try:
32
+ return super().invoke(ctx)
33
+ except (SciEtlError, ModuleNotFoundError) as error:
34
+ raise to_cli_failure(error) from error
35
+
36
+
37
+ def _print_version(ctx: click.Context, _param: click.Parameter, value: bool) -> None:
38
+ if not value or ctx.resilient_parsing:
39
+ return
40
+ click.echo(f"sci-etl {version('sci-etl-cli')} (sci-etl-core {version('sci-etl-core')})")
41
+ ctx.exit()
42
+
43
+
44
+ @click.group(
45
+ cls=SciEtlGroup,
46
+ context_settings={"help_option_names": ["-h", "--help"]},
47
+ epilog=_EXIT_CODES,
48
+ )
49
+ @click.option(
50
+ "--version",
51
+ is_flag=True,
52
+ expose_value=False,
53
+ is_eager=True,
54
+ callback=_print_version,
55
+ help="Show the CLI and library versions and exit.",
56
+ )
57
+ def cli() -> None:
58
+ """Run sci-etl-core extraction pipelines from a YAML config file."""
59
+
60
+
61
+ cli.add_command(init_command)
62
+ cli.add_command(validate_command)
63
+ cli.add_command(search_command)
64
+ cli.add_command(run_command)
65
+ cli.add_command(status_command)
66
+ cli.add_command(parse_command)
67
+
68
+
69
+ def main() -> None:
70
+ cli(prog_name="sci-etl")