indagar 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {indagar-0.2.0 → indagar-0.3.0}/.env.example +7 -0
- {indagar-0.2.0 → indagar-0.3.0}/.github/workflows/ci.yml +2 -0
- {indagar-0.2.0 → indagar-0.3.0}/AGENTS.md +7 -0
- {indagar-0.2.0 → indagar-0.3.0}/CHANGELOG.md +25 -0
- {indagar-0.2.0 → indagar-0.3.0}/PKG-INFO +55 -7
- {indagar-0.2.0 → indagar-0.3.0}/README.md +50 -6
- {indagar-0.2.0 → indagar-0.3.0}/SPEC.md +57 -18
- {indagar-0.2.0 → indagar-0.3.0}/pyproject.toml +6 -1
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/__init__.py +1 -1
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/cli.py +76 -2
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/config.py +30 -0
- indagar-0.3.0/src/indagar/estimate.py +112 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/llm/anthropic.py +10 -1
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/llm/factory.py +17 -3
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/llm/openai_compatible.py +10 -1
- indagar-0.3.0/src/indagar/progress.py +27 -0
- indagar-0.3.0/src/indagar/rate_limit.py +99 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/deep.py +140 -16
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/loop.py +29 -0
- indagar-0.3.0/src/indagar/web/__init__.py +5 -0
- indagar-0.3.0/src/indagar/web/app.py +194 -0
- indagar-0.3.0/src/indagar/web/jobs.py +260 -0
- indagar-0.3.0/src/indagar/web/static/app.js +732 -0
- indagar-0.3.0/src/indagar/web/static/index.html +120 -0
- indagar-0.3.0/src/indagar/web/static/styles.css +589 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_config.py +16 -0
- indagar-0.3.0/tests/test_estimate.py +69 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_llm.py +65 -1
- indagar-0.3.0/tests/test_rate_limit.py +123 -0
- indagar-0.3.0/tests/test_web.py +207 -0
- {indagar-0.2.0 → indagar-0.3.0}/.gitignore +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/LICENSE +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/__main__.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/citations.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/llm/__init__.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/llm/base.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/models.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/__init__.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/extract.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/gaps.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/json_utils.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/planner.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/reader.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/research/synthesize.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/search/__init__.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/search/base.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/search/chain.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/search/providers.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/store.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/src/indagar/usage.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_chain.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_citations.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_deep.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_extract.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_gaps.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_loop.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_planner.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_reader.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_synthesize.py +0 -0
- {indagar-0.2.0 → indagar-0.3.0}/tests/test_usage.py +0 -0
|
@@ -24,6 +24,13 @@ INDAGAR_WRITER_MODEL=
|
|
|
24
24
|
INDAGAR_LLM_MAX_TOKENS=1024
|
|
25
25
|
INDAGAR_LLM_TEMPERATURE=0.2
|
|
26
26
|
INDAGAR_LLM_TIMEOUT=60
|
|
27
|
+
INDAGAR_LLM_MAX_RETRIES=2
|
|
28
|
+
INDAGAR_RETRY_BASE_SECONDS=0.5
|
|
29
|
+
INDAGAR_LLM_MIN_INTERVAL=0
|
|
30
|
+
|
|
31
|
+
# Optional user-supplied rates for estimates, USD per million tokens.
|
|
32
|
+
# INDAGAR_INPUT_COST_PER_MTOK=
|
|
33
|
+
# INDAGAR_OUTPUT_COST_PER_MTOK=
|
|
27
34
|
|
|
28
35
|
ANTHROPIC_API_KEY=
|
|
29
36
|
# ANTHROPIC_BASE_URL=https://api.anthropic.com
|
|
@@ -12,6 +12,9 @@ pytest -q
|
|
|
12
12
|
indagar doctor
|
|
13
13
|
indagar llm "Say hello" --role writer
|
|
14
14
|
indagar research "What is X?" --llm --depth 0 --max-sources 3
|
|
15
|
+
indagar estimate "What is X?" --max-sources 3 --depth 0
|
|
16
|
+
indagar research "What is X?" --llm --dry-run
|
|
17
|
+
indagar serve --host 127.0.0.1 --port 8765
|
|
15
18
|
```
|
|
16
19
|
|
|
17
20
|
## Invariants
|
|
@@ -26,6 +29,10 @@ indagar research "What is X?" --llm --depth 0 --max-sources 3
|
|
|
26
29
|
- Synthesis may cite only ledger IDs. Unknown IDs are a hard error and trigger
|
|
27
30
|
the evidence-table fallback.
|
|
28
31
|
- Do not store raw LLM prompts or provider payloads in run records. Store the final report, citations, source-read status, and usage.
|
|
32
|
+
- Retry only transient failures (`408`, `425`, `429`, `5xx`, transport errors) and honour `Retry-After`.
|
|
33
|
+
- Estimates are rough upper bounds. Cost estimates use user-supplied per-million-token rates, not built-in pricing.
|
|
29
34
|
- Web and media sources are not peer-reviewed and must not be labelled as such.
|
|
30
35
|
- Keep the CLI flow: `doctor` is offline, `search` is retrieval-only, `research`
|
|
31
36
|
writes a run record and a report or dossier.
|
|
37
|
+
- The web server binds `127.0.0.1` by default and has no auth. Do not expose it
|
|
38
|
+
to untrusted networks without warning the user.
|
|
@@ -3,6 +3,31 @@
|
|
|
3
3
|
All notable changes to Indagar are documented here. The format follows Keep a
|
|
4
4
|
Changelog, and the project uses semantic versioning.
|
|
5
5
|
|
|
6
|
+
## [0.3.0] - 2026-11-21
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- Local FastAPI service with background research jobs and SSE progress.
|
|
11
|
+
- Web UI for question/budget input, provider status, live timeline, history,
|
|
12
|
+
stop, report, source inspector, citations, run details, and usage.
|
|
13
|
+
- Markdown and JSON downloads from the web UI.
|
|
14
|
+
- `indagar serve` command and optional `web` extra.
|
|
15
|
+
- API and job-manager tests with fake runners and a TestClient.
|
|
16
|
+
|
|
17
|
+
## [0.2.1] - 2026-11-21
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- `indagar estimate` and `research --dry-run` for offline pre-run budgets.
|
|
22
|
+
- Optional input/output cost rates for dollar estimates.
|
|
23
|
+
- LLM retry/backoff on `429`, `5xx`, and transport errors, with `Retry-After`.
|
|
24
|
+
- Shared minimum interval between LLM calls.
|
|
25
|
+
- Tests for retries, rate limiting, estimates, and retry/cost settings.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
|
|
29
|
+
- `indagar doctor` reports retry count and minimum request interval.
|
|
30
|
+
|
|
6
31
|
## [0.2.0] - 2026-11-21
|
|
7
32
|
|
|
8
33
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: indagar
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
|
|
5
5
|
Author: Michael Borck
|
|
6
6
|
License: MIT
|
|
@@ -18,8 +18,12 @@ Requires-Dist: python-dotenv>=1.0
|
|
|
18
18
|
Requires-Dist: rich>=13.7
|
|
19
19
|
Requires-Dist: typer>=0.12
|
|
20
20
|
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: fastapi>=0.115; extra == 'dev'
|
|
21
22
|
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
23
|
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
24
|
+
Provides-Extra: web
|
|
25
|
+
Requires-Dist: fastapi>=0.115; extra == 'web'
|
|
26
|
+
Requires-Dist: uvicorn[standard]>=0.30; extra == 'web'
|
|
23
27
|
Description-Content-Type: text/markdown
|
|
24
28
|
|
|
25
29
|
# Indagar
|
|
@@ -34,6 +38,13 @@ Phase 1 core loop is implemented. `research` is retrieval-only by default;
|
|
|
34
38
|
adding `--llm` turns on LLM planning, source reading, grounded quote
|
|
35
39
|
extraction, a bounded gap check, and citation-checked synthesis.
|
|
36
40
|
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
uv pip install indagar
|
|
45
|
+
# or: pip install 'indagar[web]' to include the local web UI
|
|
46
|
+
```
|
|
47
|
+
|
|
37
48
|
## Quick start
|
|
38
49
|
|
|
39
50
|
```bash
|
|
@@ -51,6 +62,12 @@ indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
|
51
62
|
# Full Phase 1 loop; requires an LLM provider and model.
|
|
52
63
|
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
53
64
|
--llm --max-sources 10 --depth 1
|
|
65
|
+
|
|
66
|
+
# See the rough budget before running.
|
|
67
|
+
indagar estimate "Does retrieval augmented generation reduce hallucination?" \
|
|
68
|
+
--max-sources 10 --depth 1
|
|
69
|
+
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
70
|
+
--llm --dry-run
|
|
54
71
|
```
|
|
55
72
|
|
|
56
73
|
If the checkout is on an exFAT drive, create the virtual environment on a
|
|
@@ -98,6 +115,22 @@ writer roles. If synthesis uses an unknown citation ID, Indagar rejects it and
|
|
|
98
115
|
falls back to the grounded evidence table rather than keeping an unverifiable
|
|
99
116
|
claim.
|
|
100
117
|
|
|
118
|
+
## Web UI
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
uv pip install 'indagar[web]'
|
|
122
|
+
indagar serve
|
|
123
|
+
# open http://127.0.0.1:8765
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The local UI starts Phase 0 or `--llm` runs, streams progress over SSE, shows
|
|
127
|
+
sources and citations, and exports Markdown or JSON. A run can be stopped from
|
|
128
|
+
the page. History and the source inspector are included.
|
|
129
|
+
|
|
130
|
+
The server binds `127.0.0.1` by default and has no authentication. Use
|
|
131
|
+
`--host`/`--port` to change the bind address; do not expose it to an untrusted
|
|
132
|
+
network.
|
|
133
|
+
|
|
101
134
|
## Configuration
|
|
102
135
|
|
|
103
136
|
Copy `.env.example` to `.env` and set the providers you have.
|
|
@@ -122,6 +155,11 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
122
155
|
| `INDAGAR_LLM_MAX_TOKENS` | Tokens per call. Default `1024` |
|
|
123
156
|
| `INDAGAR_LLM_TEMPERATURE` | Sampling temperature. Default `0.2` |
|
|
124
157
|
| `INDAGAR_LLM_TIMEOUT` | LLM request timeout in seconds. Default `60` |
|
|
158
|
+
| `INDAGAR_LLM_MAX_RETRIES` | Retries on 429/5xx/transport errors. Default `2` |
|
|
159
|
+
| `INDAGAR_RETRY_BASE_SECONDS` | Exponential backoff base. Default `0.5` |
|
|
160
|
+
| `INDAGAR_LLM_MIN_INTERVAL` | Minimum seconds between LLM calls. Default `0` |
|
|
161
|
+
| `INDAGAR_INPUT_COST_PER_MTOK` | Optional input USD per million tokens |
|
|
162
|
+
| `INDAGAR_OUTPUT_COST_PER_MTOK` | Optional output USD per million tokens |
|
|
125
163
|
| `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
|
|
126
164
|
| `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
|
|
127
165
|
| `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
|
|
@@ -132,11 +170,18 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
132
170
|
| `INDAGAR_SYNTHESIS_MAX_TOKENS` | Synthesis output cap. Default `2048` |
|
|
133
171
|
| `INDAGAR_READER_TIMEOUT` | Page read timeout in seconds. Default `45` |
|
|
134
172
|
|
|
135
|
-
`indagar doctor` shows what is configured.
|
|
173
|
+
`indagar doctor` shows what is configured. `indagar estimate` is offline and
|
|
174
|
+
prints a rough upper bound for searches, sources, LLM calls, and tokens. Set
|
|
175
|
+
the two cost variables above to add a dollar estimate; rates are user-supplied
|
|
176
|
+
and model-specific.
|
|
177
|
+
|
|
178
|
+
LLM requests retry on `429`, `5xx`, and transport failures with exponential
|
|
179
|
+
backoff and `Retry-After` support. `INDAGAR_LLM_MIN_INTERVAL` spaces calls
|
|
180
|
+
across the planner, reader, and writer roles.
|
|
136
181
|
|
|
137
182
|
## Status
|
|
138
183
|
|
|
139
|
-
- [x] CLI with `doctor`, `llm`, `search`, and `research`
|
|
184
|
+
- [x] CLI with `doctor`, `llm`, `estimate`, `search`, and `research`
|
|
140
185
|
- [x] Firecrawl, Brave, Serper, and SearXNG adapters
|
|
141
186
|
- [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
|
|
142
187
|
- [x] Deterministic research plan and cited source dossier
|
|
@@ -144,12 +189,15 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
144
189
|
- [x] LLM planner, source reader, verbatim quote extraction, gap loop, and synthesis
|
|
145
190
|
- [x] Citation ledger with dedupe, source grouping, and Markdown rendering
|
|
146
191
|
- [x] Per-run source-read and model-usage tables
|
|
147
|
-
- [
|
|
148
|
-
- [
|
|
192
|
+
- [x] Pre-run token/cost estimate, LLM retry/backoff, and request spacing
|
|
193
|
+
- [x] Local FastAPI service, job history, live SSE progress, and stop
|
|
194
|
+
- [x] Web report, source inspector, citations, usage, and Markdown/JSON export
|
|
195
|
+
- [ ] Resume, DOCX/PDF, source confidence labels, and concurrency
|
|
196
|
+
- [ ] Desktop shell
|
|
149
197
|
- [ ] Academic, podcast, and video sources
|
|
150
198
|
|
|
151
|
-
PyPI has `0.1
|
|
152
|
-
|
|
199
|
+
PyPI has `0.2.1`; the Phase 2 web UI is version `0.3.0` on `main` and is next
|
|
200
|
+
to publish.
|
|
153
201
|
|
|
154
202
|
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
155
203
|
|
|
@@ -10,6 +10,13 @@ Phase 1 core loop is implemented. `research` is retrieval-only by default;
|
|
|
10
10
|
adding `--llm` turns on LLM planning, source reading, grounded quote
|
|
11
11
|
extraction, a bounded gap check, and citation-checked synthesis.
|
|
12
12
|
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
uv pip install indagar
|
|
17
|
+
# or: pip install 'indagar[web]' to include the local web UI
|
|
18
|
+
```
|
|
19
|
+
|
|
13
20
|
## Quick start
|
|
14
21
|
|
|
15
22
|
```bash
|
|
@@ -27,6 +34,12 @@ indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
|
27
34
|
# Full Phase 1 loop; requires an LLM provider and model.
|
|
28
35
|
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
29
36
|
--llm --max-sources 10 --depth 1
|
|
37
|
+
|
|
38
|
+
# See the rough budget before running.
|
|
39
|
+
indagar estimate "Does retrieval augmented generation reduce hallucination?" \
|
|
40
|
+
--max-sources 10 --depth 1
|
|
41
|
+
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
42
|
+
--llm --dry-run
|
|
30
43
|
```
|
|
31
44
|
|
|
32
45
|
If the checkout is on an exFAT drive, create the virtual environment on a
|
|
@@ -74,6 +87,22 @@ writer roles. If synthesis uses an unknown citation ID, Indagar rejects it and
|
|
|
74
87
|
falls back to the grounded evidence table rather than keeping an unverifiable
|
|
75
88
|
claim.
|
|
76
89
|
|
|
90
|
+
## Web UI
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
uv pip install 'indagar[web]'
|
|
94
|
+
indagar serve
|
|
95
|
+
# open http://127.0.0.1:8765
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The local UI starts Phase 0 or `--llm` runs, streams progress over SSE, shows
|
|
99
|
+
sources and citations, and exports Markdown or JSON. A run can be stopped from
|
|
100
|
+
the page. History and the source inspector are included.
|
|
101
|
+
|
|
102
|
+
The server binds `127.0.0.1` by default and has no authentication. Use
|
|
103
|
+
`--host`/`--port` to change the bind address; do not expose it to an untrusted
|
|
104
|
+
network.
|
|
105
|
+
|
|
77
106
|
## Configuration
|
|
78
107
|
|
|
79
108
|
Copy `.env.example` to `.env` and set the providers you have.
|
|
@@ -98,6 +127,11 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
98
127
|
| `INDAGAR_LLM_MAX_TOKENS` | Tokens per call. Default `1024` |
|
|
99
128
|
| `INDAGAR_LLM_TEMPERATURE` | Sampling temperature. Default `0.2` |
|
|
100
129
|
| `INDAGAR_LLM_TIMEOUT` | LLM request timeout in seconds. Default `60` |
|
|
130
|
+
| `INDAGAR_LLM_MAX_RETRIES` | Retries on 429/5xx/transport errors. Default `2` |
|
|
131
|
+
| `INDAGAR_RETRY_BASE_SECONDS` | Exponential backoff base. Default `0.5` |
|
|
132
|
+
| `INDAGAR_LLM_MIN_INTERVAL` | Minimum seconds between LLM calls. Default `0` |
|
|
133
|
+
| `INDAGAR_INPUT_COST_PER_MTOK` | Optional input USD per million tokens |
|
|
134
|
+
| `INDAGAR_OUTPUT_COST_PER_MTOK` | Optional output USD per million tokens |
|
|
101
135
|
| `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
|
|
102
136
|
| `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
|
|
103
137
|
| `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
|
|
@@ -108,11 +142,18 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
108
142
|
| `INDAGAR_SYNTHESIS_MAX_TOKENS` | Synthesis output cap. Default `2048` |
|
|
109
143
|
| `INDAGAR_READER_TIMEOUT` | Page read timeout in seconds. Default `45` |
|
|
110
144
|
|
|
111
|
-
`indagar doctor` shows what is configured.
|
|
145
|
+
`indagar doctor` shows what is configured. `indagar estimate` is offline and
|
|
146
|
+
prints a rough upper bound for searches, sources, LLM calls, and tokens. Set
|
|
147
|
+
the two cost variables above to add a dollar estimate; rates are user-supplied
|
|
148
|
+
and model-specific.
|
|
149
|
+
|
|
150
|
+
LLM requests retry on `429`, `5xx`, and transport failures with exponential
|
|
151
|
+
backoff and `Retry-After` support. `INDAGAR_LLM_MIN_INTERVAL` spaces calls
|
|
152
|
+
across the planner, reader, and writer roles.
|
|
112
153
|
|
|
113
154
|
## Status
|
|
114
155
|
|
|
115
|
-
- [x] CLI with `doctor`, `llm`, `search`, and `research`
|
|
156
|
+
- [x] CLI with `doctor`, `llm`, `estimate`, `search`, and `research`
|
|
116
157
|
- [x] Firecrawl, Brave, Serper, and SearXNG adapters
|
|
117
158
|
- [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
|
|
118
159
|
- [x] Deterministic research plan and cited source dossier
|
|
@@ -120,12 +161,15 @@ Copy `.env.example` to `.env` and set the providers you have.
|
|
|
120
161
|
- [x] LLM planner, source reader, verbatim quote extraction, gap loop, and synthesis
|
|
121
162
|
- [x] Citation ledger with dedupe, source grouping, and Markdown rendering
|
|
122
163
|
- [x] Per-run source-read and model-usage tables
|
|
123
|
-
- [
|
|
124
|
-
- [
|
|
164
|
+
- [x] Pre-run token/cost estimate, LLM retry/backoff, and request spacing
|
|
165
|
+
- [x] Local FastAPI service, job history, live SSE progress, and stop
|
|
166
|
+
- [x] Web report, source inspector, citations, usage, and Markdown/JSON export
|
|
167
|
+
- [ ] Resume, DOCX/PDF, source confidence labels, and concurrency
|
|
168
|
+
- [ ] Desktop shell
|
|
125
169
|
- [ ] Academic, podcast, and video sources
|
|
126
170
|
|
|
127
|
-
PyPI has `0.1
|
|
128
|
-
|
|
171
|
+
PyPI has `0.2.1`; the Phase 2 web UI is version `0.3.0` on `main` and is next
|
|
172
|
+
to publish.
|
|
129
173
|
|
|
130
174
|
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
131
175
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Indagar — design specification
|
|
2
2
|
|
|
3
|
-
**Status:** Phase
|
|
3
|
+
**Status:** Phase 2 in progress: local web UI
|
|
4
4
|
**Name:** Indagar (Spanish: to investigate, inquire, probe)
|
|
5
5
|
**Licence:** MIT
|
|
6
6
|
|
|
@@ -130,12 +130,18 @@ Every run has explicit caps:
|
|
|
130
130
|
|
|
131
131
|
Controls:
|
|
132
132
|
|
|
133
|
-
-
|
|
134
|
-
|
|
133
|
+
- LLM requests retry on `429`, `5xx`, and transport failures with exponential
|
|
134
|
+
backoff, jitter, and `Retry-After` support.
|
|
135
|
+
- A shared minimum interval spaces calls across the planner, reader, and writer
|
|
136
|
+
roles (`INDAGAR_LLM_MIN_INTERVAL`).
|
|
137
|
+
- Search results and identical queries are cached; page-level caching is still
|
|
138
|
+
to come.
|
|
135
139
|
- Small/local model for extraction and summarisation.
|
|
136
|
-
- Stop and resume: a run is a saved state, not a single function call
|
|
137
|
-
|
|
138
|
-
|
|
140
|
+
- Stop and resume: a run is a saved state, not a single function call; resume
|
|
141
|
+
is Phase 2.
|
|
142
|
+
- A pre-run estimate reports upper-bound searches, sources, calls, tokens, and
|
|
143
|
+
an optional dollar cost from user-supplied per-million-token rates.
|
|
144
|
+
- Actual token usage is recorded per provider/model after the run.
|
|
139
145
|
|
|
140
146
|
Deep research is mostly a scheduling and grounding problem. Treat every network
|
|
141
147
|
call as budgeted.
|
|
@@ -144,13 +150,15 @@ call as budgeted.
|
|
|
144
150
|
|
|
145
151
|
```
|
|
146
152
|
src/indagar/
|
|
147
|
-
cli.py Typer commands: doctor, llm, search, research
|
|
153
|
+
cli.py Typer commands: doctor, llm, estimate, search, research, serve
|
|
148
154
|
config.py Environment settings and provider status
|
|
149
155
|
models.py Source, Citation, SubQuestion, ResearchPlan, ReadAttempt,
|
|
150
156
|
ModelUsage, SearchOutcome, Run
|
|
151
157
|
citations.py Citation ledger, dedupe, Markdown rendering
|
|
152
158
|
store.py Run JSON + Markdown storage
|
|
153
159
|
usage.py Per-model call and token accounting
|
|
160
|
+
estimate.py Pre-run tokens, calls, and optional cost estimate
|
|
161
|
+
rate_limit.py LLM retry/backoff and request spacing
|
|
154
162
|
llm/
|
|
155
163
|
base.py LLMProvider protocol, response, errors
|
|
156
164
|
anthropic.py Anthropic Messages adapter
|
|
@@ -160,6 +168,10 @@ src/indagar/
|
|
|
160
168
|
base.py SearchProvider protocol and SearchError
|
|
161
169
|
providers.py Firecrawl, Brave, Serper, SearXNG
|
|
162
170
|
chain.py Ordered fallback, dedupe, cache
|
|
171
|
+
web/
|
|
172
|
+
app.py FastAPI routes, SSE, static UI
|
|
173
|
+
jobs.py Background jobs, events, history, cancel
|
|
174
|
+
static/ index.html, app.js, styles.css
|
|
163
175
|
research/
|
|
164
176
|
planner.py Deterministic and LLM planners
|
|
165
177
|
reader.py Firecrawl, simple HTTP, and fallback readers
|
|
@@ -171,8 +183,8 @@ src/indagar/
|
|
|
171
183
|
json_utils.py Tolerant JSON object parsing
|
|
172
184
|
```
|
|
173
185
|
|
|
174
|
-
|
|
175
|
-
|
|
186
|
+
Phase 1 is implemented. Phase 2 ships the local FastAPI service and web UI;
|
|
187
|
+
resume, DOCX/PDF export, and source confidence labels remain.
|
|
176
188
|
|
|
177
189
|
## 9. Interfaces
|
|
178
190
|
|
|
@@ -181,19 +193,30 @@ limiting, resume, and the web UI remain.
|
|
|
181
193
|
```bash
|
|
182
194
|
indagar doctor
|
|
183
195
|
indagar llm "prompt" --role writer
|
|
196
|
+
indagar estimate "question" --max-sources 12 --depth 1
|
|
184
197
|
indagar search "question" --limit 8
|
|
185
198
|
indagar research "question" --max-sources 12 --out report.md
|
|
186
199
|
indagar research "question" --llm --depth 1 --max-sources 12
|
|
200
|
+
indagar research "question" --llm --dry-run
|
|
201
|
+
indagar serve --host 127.0.0.1 --port 8765
|
|
187
202
|
```
|
|
188
203
|
|
|
189
|
-
### Web UI (Phase 2)
|
|
204
|
+
### Web UI (Phase 2, in progress)
|
|
205
|
+
|
|
206
|
+
Implemented:
|
|
190
207
|
|
|
191
208
|
- Question, budget, provider status.
|
|
192
|
-
- Live progress: plan, searches,
|
|
193
|
-
-
|
|
194
|
-
-
|
|
195
|
-
-
|
|
196
|
-
|
|
209
|
+
- Live progress over SSE: plan, searches, reads, gaps, synthesis.
|
|
210
|
+
- Report view, source inspector, citations, run details, and usage.
|
|
211
|
+
- Markdown and JSON export.
|
|
212
|
+
- Job history and stop.
|
|
213
|
+
|
|
214
|
+
Remaining:
|
|
215
|
+
|
|
216
|
+
- Resume interrupted runs.
|
|
217
|
+
- DOCX and PDF export.
|
|
218
|
+
- Source type and confidence labels.
|
|
219
|
+
- Concurrency controls.
|
|
197
220
|
|
|
198
221
|
### Desktop (Phase 3)
|
|
199
222
|
|
|
@@ -219,7 +242,7 @@ Do not fork these projects. Borrow patterns and keep Indagar thin.
|
|
|
219
242
|
|---|---|
|
|
220
243
|
| 0 | Skeleton: CLI, search chain, deterministic plan, source dossier, tests, CI |
|
|
221
244
|
| 1 | LLM planner, reader/extractor, gap loop, synthesis, citation ledger, usage panel |
|
|
222
|
-
| 2 | Web UI,
|
|
245
|
+
| 2 | Web UI, job history, live progress, source inspector, Markdown/JSON exports, resume, DOCX/PDF |
|
|
223
246
|
| 3 | Desktop shell, academic APIs, podcasts/videos, provider-native search as an option |
|
|
224
247
|
|
|
225
248
|
## 12. Phase 0 acceptance
|
|
@@ -239,7 +262,23 @@ Do not fork these projects. Borrow patterns and keep Indagar thin.
|
|
|
239
262
|
- Synthesis rejects unknown citation IDs and falls back to the evidence table.
|
|
240
263
|
- The citation ledger rejects empty quotes and dedupes identical evidence.
|
|
241
264
|
- Reports render sources, citations, source-read status, and per-model usage.
|
|
265
|
+
- `indagar estimate` and `research --dry-run` report an upper-bound budget
|
|
266
|
+
without calling a provider.
|
|
267
|
+
- LLM calls retry transient failures and can be spaced by a shared interval.
|
|
242
268
|
- Raw prompts, provider payloads, and API keys never enter run records.
|
|
243
|
-
- Tests cover request shaping, responses, errors,
|
|
244
|
-
synthesis, citation validation, and the
|
|
269
|
+
- Tests cover request shaping, responses, errors, retries, rate limiting,
|
|
270
|
+
estimates, readers, extraction, gaps, synthesis, citation validation, and the
|
|
271
|
+
full loop with fakes.
|
|
245
272
|
- `ruff check`, `ruff format --check`, and `pytest` pass.
|
|
273
|
+
|
|
274
|
+
## 14. Phase 2 acceptance (initial)
|
|
275
|
+
|
|
276
|
+
- `indagar serve` starts the local FastAPI service and static UI.
|
|
277
|
+
- The UI starts Phase 0 or `--llm` runs and shows provider status.
|
|
278
|
+
- Progress streams over SSE; a running job can be stopped.
|
|
279
|
+
- Completed runs are saved, listed in history, and reloadable.
|
|
280
|
+
- The UI renders the report, sources, citations, run details, and usage.
|
|
281
|
+
- Markdown and JSON exports download from the run.
|
|
282
|
+
- API tests cover status, estimates, job lifecycle, SSE, cancellation, exports,
|
|
283
|
+
and missing runs.
|
|
284
|
+
- `ruff check`, `ruff format --check`, `pytest`, and `node --check` pass.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "indagar"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Deep research from the command line: plan, search, read, and synthesize with citations."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -23,9 +23,14 @@ dependencies = [
|
|
|
23
23
|
]
|
|
24
24
|
|
|
25
25
|
[project.optional-dependencies]
|
|
26
|
+
web = [
|
|
27
|
+
"fastapi>=0.115",
|
|
28
|
+
"uvicorn[standard]>=0.30",
|
|
29
|
+
]
|
|
26
30
|
dev = [
|
|
27
31
|
"pytest>=8",
|
|
28
32
|
"ruff>=0.6",
|
|
33
|
+
"fastapi>=0.115",
|
|
29
34
|
]
|
|
30
35
|
|
|
31
36
|
[project.scripts]
|
|
@@ -12,6 +12,7 @@ from rich.table import Table
|
|
|
12
12
|
|
|
13
13
|
from . import __version__
|
|
14
14
|
from .config import Settings
|
|
15
|
+
from .estimate import cost_for_tokens, estimate_run
|
|
15
16
|
from .llm.base import LLM_ROLES, LLMError
|
|
16
17
|
from .llm.factory import build_llm, build_llms
|
|
17
18
|
from .research.deep import run_deep_research
|
|
@@ -74,7 +75,9 @@ def doctor() -> None:
|
|
|
74
75
|
console.print(
|
|
75
76
|
f"Budgets: sources={settings.max_sources}, "
|
|
76
77
|
f"seconds={settings.max_seconds}, search_limit={settings.search_limit}, "
|
|
77
|
-
f"depth={settings.max_depth},
|
|
78
|
+
f"depth={settings.max_depth}, retries={settings.llm_max_retries}, "
|
|
79
|
+
f"min_interval={settings.llm_min_interval}s, "
|
|
80
|
+
f"llm_max_tokens={settings.llm_max_tokens}"
|
|
78
81
|
)
|
|
79
82
|
if not configured:
|
|
80
83
|
console.print(
|
|
@@ -132,6 +135,63 @@ def llm_prompt(
|
|
|
132
135
|
console.print(detail)
|
|
133
136
|
|
|
134
137
|
|
|
138
|
+
@app.command()
|
|
139
|
+
def estimate(
|
|
140
|
+
question: str = typer.Argument(..., help="Question or topic to estimate"),
|
|
141
|
+
llm: bool = typer.Option(True, "--llm/--no-llm", help="Include LLM calls"),
|
|
142
|
+
depth: int | None = typer.Option(None, "--depth", min=0, max=2),
|
|
143
|
+
max_sources: int = typer.Option(0, "--max-sources", min=0, help="0 uses INDAGAR_MAX_SOURCES"),
|
|
144
|
+
) -> None:
|
|
145
|
+
"""Print a rough pre-run budget estimate without calling any provider."""
|
|
146
|
+
settings = _settings_or_exit()
|
|
147
|
+
try:
|
|
148
|
+
result = estimate_run(
|
|
149
|
+
question,
|
|
150
|
+
settings,
|
|
151
|
+
max_sources=max_sources or None,
|
|
152
|
+
depth=depth,
|
|
153
|
+
llm=llm,
|
|
154
|
+
)
|
|
155
|
+
except ValueError as exc:
|
|
156
|
+
console.print(f"[red]{exc}[/red]")
|
|
157
|
+
raise typer.Exit(1) from exc
|
|
158
|
+
|
|
159
|
+
table = Table(title="Run estimate")
|
|
160
|
+
table.add_column("Item")
|
|
161
|
+
table.add_column("Value", justify="right")
|
|
162
|
+
table.add_row("Searches", str(result.searches))
|
|
163
|
+
table.add_row("Sources", str(result.sources))
|
|
164
|
+
table.add_row("LLM calls", str(result.llm_calls))
|
|
165
|
+
table.add_row("Input tokens", f"~{result.input_tokens:,}")
|
|
166
|
+
table.add_row("Output tokens", f"~{result.output_tokens:,}")
|
|
167
|
+
if result.cost_usd is not None:
|
|
168
|
+
table.add_row("Estimated cost", f"~${result.cost_usd:.4f}")
|
|
169
|
+
console.print(table)
|
|
170
|
+
if result.llm_calls and result.cost_usd is None:
|
|
171
|
+
console.print(
|
|
172
|
+
"[dim]Set INDAGAR_INPUT_COST_PER_MTOK and "
|
|
173
|
+
"INDAGAR_OUTPUT_COST_PER_MTOK for a dollar estimate.[/dim]"
|
|
174
|
+
)
|
|
175
|
+
console.print("[dim]Rough upper bound; actual usage is recorded in the run.[/dim]")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@app.command()
|
|
179
|
+
def serve(
|
|
180
|
+
host: str = typer.Option("127.0.0.1", "--host", help="Bind address"),
|
|
181
|
+
port: int = typer.Option(8765, "--port", min=1, max=65535),
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Serve the local web UI and API."""
|
|
184
|
+
try:
|
|
185
|
+
import uvicorn
|
|
186
|
+
except ImportError as exc:
|
|
187
|
+
console.print("[red]Install the web extra: pip install 'indagar[web]'[/red]")
|
|
188
|
+
raise typer.Exit(1) from exc
|
|
189
|
+
from .web.app import create_app
|
|
190
|
+
|
|
191
|
+
console.print(f"Indagar web UI: http://{host}:{port}")
|
|
192
|
+
uvicorn.run(create_app(), host=host, port=port)
|
|
193
|
+
|
|
194
|
+
|
|
135
195
|
@app.command()
|
|
136
196
|
def search(
|
|
137
197
|
query: str = typer.Argument(..., help="Search query"),
|
|
@@ -186,12 +246,23 @@ def research(
|
|
|
186
246
|
),
|
|
187
247
|
max_sources: int = typer.Option(0, "--max-sources", min=0, help="0 uses INDAGAR_MAX_SOURCES"),
|
|
188
248
|
out: Path | None = typer.Option(None, "--out", "-o", help="Markdown output path"),
|
|
249
|
+
dry_run: bool = typer.Option(False, "--dry-run", help="Print the estimate and exit"),
|
|
189
250
|
) -> None:
|
|
190
251
|
"""Plan sub-questions, search, and write a source dossier or cited report."""
|
|
191
252
|
settings = _settings_or_exit()
|
|
192
253
|
if max_sources:
|
|
193
254
|
settings = replace(settings, max_sources=max_sources)
|
|
194
255
|
|
|
256
|
+
try:
|
|
257
|
+
estimate = estimate_run(question, settings, depth=depth, llm=llm)
|
|
258
|
+
except ValueError as exc:
|
|
259
|
+
console.print(f"[red]{exc}[/red]")
|
|
260
|
+
raise typer.Exit(1) from exc
|
|
261
|
+
if llm or dry_run:
|
|
262
|
+
console.print(estimate.line())
|
|
263
|
+
if dry_run:
|
|
264
|
+
return
|
|
265
|
+
|
|
195
266
|
try:
|
|
196
267
|
if llm:
|
|
197
268
|
llms = build_llms(settings)
|
|
@@ -231,10 +302,13 @@ def research(
|
|
|
231
302
|
input_tokens = sum(item.input_tokens for item in run.usage)
|
|
232
303
|
output_tokens = sum(item.output_tokens for item in run.usage)
|
|
233
304
|
console.print(f"LLM usage: {calls} calls · {input_tokens} in / {output_tokens} out")
|
|
305
|
+
cost = cost_for_tokens(input_tokens, output_tokens, settings)
|
|
306
|
+
if cost is not None:
|
|
307
|
+
console.print(f"Token cost: ${cost:.4f}")
|
|
234
308
|
|
|
235
309
|
json_path, markdown_path = save_run(run, out=out)
|
|
236
310
|
console.print(f"Run record: {json_path}")
|
|
237
|
-
console.print(f"
|
|
311
|
+
console.print(f"Markdown: {markdown_path}")
|
|
238
312
|
|
|
239
313
|
|
|
240
314
|
def main() -> None:
|
|
@@ -36,6 +36,16 @@ def _env_float(
|
|
|
36
36
|
return default
|
|
37
37
|
|
|
38
38
|
|
|
39
|
+
def _env_optional_float(name: str) -> float | None:
|
|
40
|
+
raw = (os.getenv(name) or "").strip()
|
|
41
|
+
if not raw:
|
|
42
|
+
return None
|
|
43
|
+
try:
|
|
44
|
+
return max(0.0, float(raw))
|
|
45
|
+
except ValueError:
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
39
49
|
def _normalize_llm_provider(raw: str | None) -> str | None:
|
|
40
50
|
value = (raw or "").strip().lower().replace("_", "-")
|
|
41
51
|
if not value:
|
|
@@ -73,6 +83,11 @@ class Settings:
|
|
|
73
83
|
llm_max_tokens: int = 1024
|
|
74
84
|
llm_temperature: float = 0.2
|
|
75
85
|
llm_timeout: float = 60.0
|
|
86
|
+
llm_max_retries: int = 2
|
|
87
|
+
retry_base_seconds: float = 0.5
|
|
88
|
+
llm_min_interval: float = 0.0
|
|
89
|
+
input_cost_per_mtok: float | None = None
|
|
90
|
+
output_cost_per_mtok: float | None = None
|
|
76
91
|
max_sources: int = 12
|
|
77
92
|
max_seconds: int = 300
|
|
78
93
|
search_limit: int = 8
|
|
@@ -112,6 +127,21 @@ class Settings:
|
|
|
112
127
|
minimum=1.0,
|
|
113
128
|
maximum=3600.0,
|
|
114
129
|
),
|
|
130
|
+
llm_max_retries=_env_int("INDAGAR_LLM_MAX_RETRIES", 2, minimum=0),
|
|
131
|
+
retry_base_seconds=_env_float(
|
|
132
|
+
"INDAGAR_RETRY_BASE_SECONDS",
|
|
133
|
+
0.5,
|
|
134
|
+
minimum=0.0,
|
|
135
|
+
maximum=60.0,
|
|
136
|
+
),
|
|
137
|
+
llm_min_interval=_env_float(
|
|
138
|
+
"INDAGAR_LLM_MIN_INTERVAL",
|
|
139
|
+
0.0,
|
|
140
|
+
minimum=0.0,
|
|
141
|
+
maximum=60.0,
|
|
142
|
+
),
|
|
143
|
+
input_cost_per_mtok=_env_optional_float("INDAGAR_INPUT_COST_PER_MTOK"),
|
|
144
|
+
output_cost_per_mtok=_env_optional_float("INDAGAR_OUTPUT_COST_PER_MTOK"),
|
|
115
145
|
max_sources=_env_int("INDAGAR_MAX_SOURCES", 12),
|
|
116
146
|
max_seconds=_env_int("INDAGAR_MAX_SECONDS", 300),
|
|
117
147
|
search_limit=_env_int("INDAGAR_SEARCH_LIMIT", 8),
|