indagar 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {indagar-0.2.1 → indagar-0.3.0}/.github/workflows/ci.yml +2 -0
- {indagar-0.2.1 → indagar-0.3.0}/AGENTS.md +3 -0
- {indagar-0.2.1 → indagar-0.3.0}/CHANGELOG.md +11 -0
- {indagar-0.2.1 → indagar-0.3.0}/PKG-INFO +27 -4
- {indagar-0.2.1 → indagar-0.3.0}/README.md +22 -3
- {indagar-0.2.1 → indagar-0.3.0}/SPEC.md +36 -11
- {indagar-0.2.1 → indagar-0.3.0}/pyproject.toml +6 -1
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/__init__.py +1 -1
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/cli.py +17 -0
- indagar-0.3.0/src/indagar/progress.py +27 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/deep.py +140 -16
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/loop.py +29 -0
- indagar-0.3.0/src/indagar/web/__init__.py +5 -0
- indagar-0.3.0/src/indagar/web/app.py +194 -0
- indagar-0.3.0/src/indagar/web/jobs.py +260 -0
- indagar-0.3.0/src/indagar/web/static/app.js +732 -0
- indagar-0.3.0/src/indagar/web/static/index.html +120 -0
- indagar-0.3.0/src/indagar/web/static/styles.css +589 -0
- indagar-0.3.0/tests/test_web.py +207 -0
- {indagar-0.2.1 → indagar-0.3.0}/.env.example +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/.gitignore +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/LICENSE +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/__main__.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/citations.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/config.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/estimate.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/llm/__init__.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/llm/anthropic.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/llm/base.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/llm/factory.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/llm/openai_compatible.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/models.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/rate_limit.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/__init__.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/extract.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/gaps.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/json_utils.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/planner.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/reader.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/research/synthesize.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/search/__init__.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/search/base.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/search/chain.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/search/providers.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/store.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/src/indagar/usage.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_chain.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_citations.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_config.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_deep.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_estimate.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_extract.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_gaps.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_llm.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_loop.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_planner.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_rate_limit.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_reader.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_synthesize.py +0 -0
- {indagar-0.2.1 → indagar-0.3.0}/tests/test_usage.py +0 -0
|
@@ -14,6 +14,7 @@ indagar llm "Say hello" --role writer
|
|
|
14
14
|
indagar research "What is X?" --llm --depth 0 --max-sources 3
|
|
15
15
|
indagar estimate "What is X?" --max-sources 3 --depth 0
|
|
16
16
|
indagar research "What is X?" --llm --dry-run
|
|
17
|
+
indagar serve --host 127.0.0.1 --port 8765
|
|
17
18
|
```
|
|
18
19
|
|
|
19
20
|
## Invariants
|
|
@@ -33,3 +34,5 @@ indagar research "What is X?" --llm --dry-run
|
|
|
33
34
|
- Web and media sources are not peer-reviewed and must not be labelled as such.
|
|
34
35
|
- Keep the CLI flow: `doctor` is offline, `search` is retrieval-only, `research`
|
|
35
36
|
writes a run record and a report or dossier.
|
|
37
|
+
- The web server binds `127.0.0.1` by default and has no auth. Do not expose it
|
|
38
|
+
to untrusted networks without warning the user.
|
|
@@ -3,6 +3,17 @@
|
|
|
3
3
|
All notable changes to Indagar are documented here. The format follows Keep a
|
|
4
4
|
Changelog, and the project uses semantic versioning.
|
|
5
5
|
|
|
6
|
+
## [0.3.0] - 2026-11-21
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- Local FastAPI service with background research jobs and SSE progress.
|
|
11
|
+
- Web UI for question/budget input, provider status, live timeline, history,
|
|
12
|
+
stop, report, source inspector, citations, run details, and usage.
|
|
13
|
+
- Markdown and JSON downloads from the web UI.
|
|
14
|
+
- `indagar serve` command and optional `web` extra.
|
|
15
|
+
- API and job-manager tests with fake runners and a TestClient.
|
|
16
|
+
|
|
6
17
|
## [0.2.1] - 2026-11-21
|
|
7
18
|
|
|
8
19
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: indagar
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
|
|
5
5
|
Author: Michael Borck
|
|
6
6
|
License: MIT
|
|
@@ -18,8 +18,12 @@ Requires-Dist: python-dotenv>=1.0
|
|
|
18
18
|
Requires-Dist: rich>=13.7
|
|
19
19
|
Requires-Dist: typer>=0.12
|
|
20
20
|
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: fastapi>=0.115; extra == 'dev'
|
|
21
22
|
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
23
|
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
24
|
+
Provides-Extra: web
|
|
25
|
+
Requires-Dist: fastapi>=0.115; extra == 'web'
|
|
26
|
+
Requires-Dist: uvicorn[standard]>=0.30; extra == 'web'
|
|
23
27
|
Description-Content-Type: text/markdown
|
|
24
28
|
|
|
25
29
|
# Indagar
|
|
@@ -38,7 +42,7 @@ extraction, a bounded gap check, and citation-checked synthesis.
|
|
|
38
42
|
|
|
39
43
|
```bash
|
|
40
44
|
uv pip install indagar
|
|
41
|
-
# or: pip install indagar
|
|
45
|
+
# or: pip install 'indagar[web]' to include the local web UI
|
|
42
46
|
```
|
|
43
47
|
|
|
44
48
|
## Quick start
|
|
@@ -111,6 +115,22 @@ writer roles. If synthesis uses an unknown citation ID, Indagar rejects it and
|
|
|
111
115
|
falls back to the grounded evidence table rather than keeping an unverifiable
|
|
112
116
|
claim.
|
|
113
117
|
|
|
118
|
+
## Web UI
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
uv pip install 'indagar[web]'
|
|
122
|
+
indagar serve
|
|
123
|
+
# open http://127.0.0.1:8765
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The local UI starts Phase 0 or `--llm` runs, streams progress over SSE, shows
|
|
127
|
+
sources and citations, and exports Markdown or JSON. A run can be stopped from
|
|
128
|
+
the page. History and the source inspector are included.
|
|
129
|
+
|
|
130
|
+
The server binds `127.0.0.1` by default and has no authentication. Use
|
|
131
|
+
`--host`/`--port` to change the bind address; do not expose it to an untrusted
|
|
132
|
+
network.
|
|
133
|
+
|
|
114
134
|
## Configuration
|
|
115
135
|
|
|
116
136
|
Copy `.env.example` to `.env` and set the providers you have.
|
|
@@ -170,11 +190,14 @@ across the planner, reader, and writer roles.
|
|
|
170
190
|
- [x] Citation ledger with dedupe, source grouping, and Markdown rendering
|
|
171
191
|
- [x] Per-run source-read and model-usage tables
|
|
172
192
|
- [x] Pre-run token/cost estimate, LLM retry/backoff, and request spacing
|
|
173
|
-
- [
|
|
193
|
+
- [x] Local FastAPI service, job history, live SSE progress, and stop
|
|
194
|
+
- [x] Web report, source inspector, citations, usage, and Markdown/JSON export
|
|
195
|
+
- [ ] Resume, DOCX/PDF, source confidence labels, and concurrency
|
|
174
196
|
- [ ] Desktop shell
|
|
175
197
|
- [ ] Academic, podcast, and video sources
|
|
176
198
|
|
|
177
|
-
PyPI has `0.2.
|
|
199
|
+
PyPI has `0.2.1`; the Phase 2 web UI is version `0.3.0` on `main` and is next
|
|
200
|
+
to publish.
|
|
178
201
|
|
|
179
202
|
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
180
203
|
|
|
@@ -14,7 +14,7 @@ extraction, a bounded gap check, and citation-checked synthesis.
|
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
16
|
uv pip install indagar
|
|
17
|
-
# or: pip install indagar
|
|
17
|
+
# or: pip install 'indagar[web]' to include the local web UI
|
|
18
18
|
```
|
|
19
19
|
|
|
20
20
|
## Quick start
|
|
@@ -87,6 +87,22 @@ writer roles. If synthesis uses an unknown citation ID, Indagar rejects it and
|
|
|
87
87
|
falls back to the grounded evidence table rather than keeping an unverifiable
|
|
88
88
|
claim.
|
|
89
89
|
|
|
90
|
+
## Web UI
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
uv pip install 'indagar[web]'
|
|
94
|
+
indagar serve
|
|
95
|
+
# open http://127.0.0.1:8765
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The local UI starts Phase 0 or `--llm` runs, streams progress over SSE, shows
|
|
99
|
+
sources and citations, and exports Markdown or JSON. A run can be stopped from
|
|
100
|
+
the page. History and the source inspector are included.
|
|
101
|
+
|
|
102
|
+
The server binds `127.0.0.1` by default and has no authentication. Use
|
|
103
|
+
`--host`/`--port` to change the bind address; do not expose it to an untrusted
|
|
104
|
+
network.
|
|
105
|
+
|
|
90
106
|
## Configuration
|
|
91
107
|
|
|
92
108
|
Copy `.env.example` to `.env` and set the providers you have.
|
|
@@ -146,11 +162,14 @@ across the planner, reader, and writer roles.
|
|
|
146
162
|
- [x] Citation ledger with dedupe, source grouping, and Markdown rendering
|
|
147
163
|
- [x] Per-run source-read and model-usage tables
|
|
148
164
|
- [x] Pre-run token/cost estimate, LLM retry/backoff, and request spacing
|
|
149
|
-
- [
|
|
165
|
+
- [x] Local FastAPI service, job history, live SSE progress, and stop
|
|
166
|
+
- [x] Web report, source inspector, citations, usage, and Markdown/JSON export
|
|
167
|
+
- [ ] Resume, DOCX/PDF, source confidence labels, and concurrency
|
|
150
168
|
- [ ] Desktop shell
|
|
151
169
|
- [ ] Academic, podcast, and video sources
|
|
152
170
|
|
|
153
|
-
PyPI has `0.2.
|
|
171
|
+
PyPI has `0.2.1`; the Phase 2 web UI is version `0.3.0` on `main` and is next
|
|
172
|
+
to publish.
|
|
154
173
|
|
|
155
174
|
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
156
175
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Indagar — design specification
|
|
2
2
|
|
|
3
|
-
**Status:** Phase
|
|
3
|
+
**Status:** Phase 2 in progress: local web UI
|
|
4
4
|
**Name:** Indagar (Spanish: to investigate, inquire, probe)
|
|
5
5
|
**Licence:** MIT
|
|
6
6
|
|
|
@@ -150,7 +150,7 @@ call as budgeted.
|
|
|
150
150
|
|
|
151
151
|
```
|
|
152
152
|
src/indagar/
|
|
153
|
-
cli.py Typer commands: doctor, llm, search, research
|
|
153
|
+
cli.py Typer commands: doctor, llm, estimate, search, research, serve
|
|
154
154
|
config.py Environment settings and provider status
|
|
155
155
|
models.py Source, Citation, SubQuestion, ResearchPlan, ReadAttempt,
|
|
156
156
|
ModelUsage, SearchOutcome, Run
|
|
@@ -168,6 +168,10 @@ src/indagar/
|
|
|
168
168
|
base.py SearchProvider protocol and SearchError
|
|
169
169
|
providers.py Firecrawl, Brave, Serper, SearXNG
|
|
170
170
|
chain.py Ordered fallback, dedupe, cache
|
|
171
|
+
web/
|
|
172
|
+
app.py FastAPI routes, SSE, static UI
|
|
173
|
+
jobs.py Background jobs, events, history, cancel
|
|
174
|
+
static/ index.html, app.js, styles.css
|
|
171
175
|
research/
|
|
172
176
|
planner.py Deterministic and LLM planners
|
|
173
177
|
reader.py Firecrawl, simple HTTP, and fallback readers
|
|
@@ -179,8 +183,8 @@ src/indagar/
|
|
|
179
183
|
json_utils.py Tolerant JSON object parsing
|
|
180
184
|
```
|
|
181
185
|
|
|
182
|
-
|
|
183
|
-
|
|
186
|
+
Phase 1 is implemented. Phase 2 ships the local FastAPI service and web UI;
|
|
187
|
+
resume, DOCX/PDF export, and source confidence labels remain.
|
|
184
188
|
|
|
185
189
|
## 9. Interfaces
|
|
186
190
|
|
|
@@ -194,16 +198,25 @@ indagar search "question" --limit 8
|
|
|
194
198
|
indagar research "question" --max-sources 12 --out report.md
|
|
195
199
|
indagar research "question" --llm --depth 1 --max-sources 12
|
|
196
200
|
indagar research "question" --llm --dry-run
|
|
201
|
+
indagar serve --host 127.0.0.1 --port 8765
|
|
197
202
|
```
|
|
198
203
|
|
|
199
|
-
### Web UI (Phase 2)
|
|
204
|
+
### Web UI (Phase 2, in progress)
|
|
205
|
+
|
|
206
|
+
Implemented:
|
|
200
207
|
|
|
201
208
|
- Question, budget, provider status.
|
|
202
|
-
- Live progress: plan, searches,
|
|
203
|
-
-
|
|
204
|
-
-
|
|
205
|
-
-
|
|
206
|
-
|
|
209
|
+
- Live progress over SSE: plan, searches, reads, gaps, synthesis.
|
|
210
|
+
- Report view, source inspector, citations, run details, and usage.
|
|
211
|
+
- Markdown and JSON export.
|
|
212
|
+
- Job history and stop.
|
|
213
|
+
|
|
214
|
+
Remaining:
|
|
215
|
+
|
|
216
|
+
- Resume interrupted runs.
|
|
217
|
+
- DOCX and PDF export.
|
|
218
|
+
- Source type and confidence labels.
|
|
219
|
+
- Concurrency controls.
|
|
207
220
|
|
|
208
221
|
### Desktop (Phase 3)
|
|
209
222
|
|
|
@@ -229,7 +242,7 @@ Do not fork these projects. Borrow patterns and keep Indagar thin.
|
|
|
229
242
|
|---|---|
|
|
230
243
|
| 0 | Skeleton: CLI, search chain, deterministic plan, source dossier, tests, CI |
|
|
231
244
|
| 1 | LLM planner, reader/extractor, gap loop, synthesis, citation ledger, usage panel |
|
|
232
|
-
| 2 | Web UI,
|
|
245
|
+
| 2 | Web UI, job history, live progress, source inspector, Markdown/JSON exports, resume, DOCX/PDF |
|
|
233
246
|
| 3 | Desktop shell, academic APIs, podcasts/videos, provider-native search as an option |
|
|
234
247
|
|
|
235
248
|
## 12. Phase 0 acceptance
|
|
@@ -257,3 +270,15 @@ Do not fork these projects. Borrow patterns and keep Indagar thin.
|
|
|
257
270
|
estimates, readers, extraction, gaps, synthesis, citation validation, and the
|
|
258
271
|
full loop with fakes.
|
|
259
272
|
- `ruff check`, `ruff format --check`, and `pytest` pass.
|
|
273
|
+
|
|
274
|
+
## 14. Phase 2 acceptance (initial)
|
|
275
|
+
|
|
276
|
+
- `indagar serve` starts the local FastAPI service and static UI.
|
|
277
|
+
- The UI starts Phase 0 or `--llm` runs and shows provider status.
|
|
278
|
+
- Progress streams over SSE; a running job can be stopped.
|
|
279
|
+
- Completed runs are saved, listed in history, and reloadable.
|
|
280
|
+
- The UI renders the report, sources, citations, run details, and usage.
|
|
281
|
+
- Markdown and JSON exports download from the run.
|
|
282
|
+
- API tests cover status, estimates, job lifecycle, SSE, cancellation, exports,
|
|
283
|
+
and missing runs.
|
|
284
|
+
- `ruff check`, `ruff format --check`, `pytest`, and `node --check` pass.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "indagar"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Deep research from the command line: plan, search, read, and synthesize with citations."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -23,9 +23,14 @@ dependencies = [
|
|
|
23
23
|
]
|
|
24
24
|
|
|
25
25
|
[project.optional-dependencies]
|
|
26
|
+
web = [
|
|
27
|
+
"fastapi>=0.115",
|
|
28
|
+
"uvicorn[standard]>=0.30",
|
|
29
|
+
]
|
|
26
30
|
dev = [
|
|
27
31
|
"pytest>=8",
|
|
28
32
|
"ruff>=0.6",
|
|
33
|
+
"fastapi>=0.115",
|
|
29
34
|
]
|
|
30
35
|
|
|
31
36
|
[project.scripts]
|
|
@@ -175,6 +175,23 @@ def estimate(
|
|
|
175
175
|
console.print("[dim]Rough upper bound; actual usage is recorded in the run.[/dim]")
|
|
176
176
|
|
|
177
177
|
|
|
178
|
+
@app.command()
|
|
179
|
+
def serve(
|
|
180
|
+
host: str = typer.Option("127.0.0.1", "--host", help="Bind address"),
|
|
181
|
+
port: int = typer.Option(8765, "--port", min=1, max=65535),
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Serve the local web UI and API."""
|
|
184
|
+
try:
|
|
185
|
+
import uvicorn
|
|
186
|
+
except ImportError as exc:
|
|
187
|
+
console.print("[red]Install the web extra: pip install 'indagar[web]'[/red]")
|
|
188
|
+
raise typer.Exit(1) from exc
|
|
189
|
+
from .web.app import create_app
|
|
190
|
+
|
|
191
|
+
console.print(f"Indagar web UI: http://{host}:{port}")
|
|
192
|
+
uvicorn.run(create_app(), host=host, port=port)
|
|
193
|
+
|
|
194
|
+
|
|
178
195
|
@app.command()
|
|
179
196
|
def search(
|
|
180
197
|
query: str = typer.Argument(..., help="Search query"),
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class ProgressEvent:
|
|
9
|
+
type: str
|
|
10
|
+
message: str
|
|
11
|
+
data: dict[str, object] = field(default_factory=dict)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
ProgressCallback = Callable[[ProgressEvent], None]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def emit_progress(
|
|
18
|
+
callback: ProgressCallback | None,
|
|
19
|
+
event_type: str,
|
|
20
|
+
message: str,
|
|
21
|
+
**data: object,
|
|
22
|
+
) -> None:
|
|
23
|
+
if callback is not None:
|
|
24
|
+
callback(ProgressEvent(event_type, message, data))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
__all__ = ["ProgressCallback", "ProgressEvent", "emit_progress"]
|
|
@@ -18,6 +18,7 @@ from ..models import (
|
|
|
18
18
|
SubQuestion,
|
|
19
19
|
utc_now,
|
|
20
20
|
)
|
|
21
|
+
from ..progress import ProgressCallback, emit_progress
|
|
21
22
|
from ..search.chain import SearchChain, build_chain, normalize_url
|
|
22
23
|
from ..usage import TrackedLLM, UsageTracker
|
|
23
24
|
from .extract import extract_claims
|
|
@@ -69,25 +70,49 @@ async def _read_source(
|
|
|
69
70
|
reads: list[ReadAttempt],
|
|
70
71
|
*,
|
|
71
72
|
max_claims: int,
|
|
73
|
+
progress: ProgressCallback | None = None,
|
|
72
74
|
) -> None:
|
|
73
75
|
started = time.perf_counter()
|
|
76
|
+
emit_progress(
|
|
77
|
+
progress,
|
|
78
|
+
"read",
|
|
79
|
+
f"Reading {source.title}",
|
|
80
|
+
url=source.url,
|
|
81
|
+
title=source.title,
|
|
82
|
+
domain=source.domain,
|
|
83
|
+
)
|
|
74
84
|
try:
|
|
75
85
|
content = await reader.read(source)
|
|
76
86
|
except Exception as exc:
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
87
|
+
latency = _ms(started)
|
|
88
|
+
error = _error_text(exc)
|
|
89
|
+
reads.append(ReadAttempt(source.url, "error", 0, 0, latency, error))
|
|
90
|
+
emit_progress(
|
|
91
|
+
progress,
|
|
92
|
+
"read_done",
|
|
93
|
+
f"Read failed: {source.title}",
|
|
94
|
+
url=source.url,
|
|
95
|
+
status="error",
|
|
96
|
+
chars=0,
|
|
97
|
+
claims=0,
|
|
98
|
+
latency_ms=latency,
|
|
99
|
+
error=error,
|
|
86
100
|
)
|
|
87
101
|
return
|
|
88
102
|
|
|
89
103
|
if len(content.text.strip()) < 200:
|
|
90
|
-
|
|
104
|
+
latency = _ms(started)
|
|
105
|
+
reads.append(ReadAttempt(source.url, "short", len(content.text), 0, latency))
|
|
106
|
+
emit_progress(
|
|
107
|
+
progress,
|
|
108
|
+
"read_done",
|
|
109
|
+
f"Too short to extract: {source.title}",
|
|
110
|
+
url=source.url,
|
|
111
|
+
status="short",
|
|
112
|
+
chars=len(content.text),
|
|
113
|
+
claims=0,
|
|
114
|
+
latency_ms=latency,
|
|
115
|
+
)
|
|
91
116
|
return
|
|
92
117
|
|
|
93
118
|
try:
|
|
@@ -99,16 +124,29 @@ async def _read_source(
|
|
|
99
124
|
max_claims=max_claims,
|
|
100
125
|
)
|
|
101
126
|
except (LLMError, ValueError) as exc:
|
|
127
|
+
latency = _ms(started)
|
|
128
|
+
error = _error_text(exc)
|
|
102
129
|
reads.append(
|
|
103
130
|
ReadAttempt(
|
|
104
131
|
source.url,
|
|
105
132
|
"error",
|
|
106
133
|
len(content.text),
|
|
107
134
|
0,
|
|
108
|
-
|
|
109
|
-
|
|
135
|
+
latency,
|
|
136
|
+
error,
|
|
110
137
|
)
|
|
111
138
|
)
|
|
139
|
+
emit_progress(
|
|
140
|
+
progress,
|
|
141
|
+
"read_done",
|
|
142
|
+
f"Extraction failed: {source.title}",
|
|
143
|
+
url=source.url,
|
|
144
|
+
status="error",
|
|
145
|
+
chars=len(content.text),
|
|
146
|
+
claims=0,
|
|
147
|
+
latency_ms=latency,
|
|
148
|
+
error=error,
|
|
149
|
+
)
|
|
112
150
|
return
|
|
113
151
|
|
|
114
152
|
for candidate in candidates:
|
|
@@ -118,15 +156,28 @@ async def _read_source(
|
|
|
118
156
|
locator=candidate.locator,
|
|
119
157
|
note=candidate.note,
|
|
120
158
|
)
|
|
159
|
+
status = "ok" if candidates else "no_evidence"
|
|
160
|
+
latency = _ms(started)
|
|
121
161
|
reads.append(
|
|
122
162
|
ReadAttempt(
|
|
123
163
|
source.url,
|
|
124
|
-
|
|
164
|
+
status,
|
|
125
165
|
len(content.text),
|
|
126
166
|
len(candidates),
|
|
127
|
-
|
|
167
|
+
latency,
|
|
128
168
|
)
|
|
129
169
|
)
|
|
170
|
+
claim_word = "claim" if len(candidates) == 1 else "claims"
|
|
171
|
+
emit_progress(
|
|
172
|
+
progress,
|
|
173
|
+
"read_done",
|
|
174
|
+
f"{len(candidates)} {claim_word} from {source.title}",
|
|
175
|
+
url=source.url,
|
|
176
|
+
status=status,
|
|
177
|
+
chars=len(content.text),
|
|
178
|
+
claims=len(candidates),
|
|
179
|
+
latency_ms=latency,
|
|
180
|
+
)
|
|
130
181
|
|
|
131
182
|
|
|
132
183
|
async def run_deep_research(
|
|
@@ -139,6 +190,7 @@ async def run_deep_research(
|
|
|
139
190
|
max_sources: int | None = None,
|
|
140
191
|
depth: int | None = None,
|
|
141
192
|
started: float | None = None,
|
|
193
|
+
progress: ProgressCallback | None = None,
|
|
142
194
|
) -> Run:
|
|
143
195
|
"""Plan, search, read, extract, gap-check, and synthesize one run."""
|
|
144
196
|
normalized = " ".join(question.split())
|
|
@@ -173,6 +225,21 @@ async def run_deep_research(
|
|
|
173
225
|
plan_question(normalized, max_sub_questions=4),
|
|
174
226
|
planner="deterministic-fallback",
|
|
175
227
|
)
|
|
228
|
+
emit_progress(
|
|
229
|
+
progress,
|
|
230
|
+
"plan_fallback",
|
|
231
|
+
"Planner unavailable; using the deterministic plan",
|
|
232
|
+
planner=plan.planner,
|
|
233
|
+
sub_questions=[item.question for item in plan.sub_questions],
|
|
234
|
+
)
|
|
235
|
+
else:
|
|
236
|
+
emit_progress(
|
|
237
|
+
progress,
|
|
238
|
+
"plan",
|
|
239
|
+
f"Planned {len(plan.sub_questions)} sub-questions",
|
|
240
|
+
planner=plan.planner,
|
|
241
|
+
sub_questions=[item.question for item in plan.sub_questions],
|
|
242
|
+
)
|
|
176
243
|
|
|
177
244
|
buckets: dict[str, list[Source]] = {}
|
|
178
245
|
seen: set[str] = set()
|
|
@@ -184,11 +251,25 @@ async def run_deep_research(
|
|
|
184
251
|
async def search_question(sub_question: SubQuestion) -> None:
|
|
185
252
|
if timed_out():
|
|
186
253
|
return
|
|
254
|
+
emit_progress(
|
|
255
|
+
progress,
|
|
256
|
+
"search",
|
|
257
|
+
f"Searching: {sub_question.question}",
|
|
258
|
+
question=sub_question.question,
|
|
259
|
+
)
|
|
187
260
|
outcome = await chain.search(
|
|
188
261
|
sub_question.question,
|
|
189
262
|
limit=settings.search_limit,
|
|
190
263
|
)
|
|
191
264
|
attempts.extend(outcome.attempts)
|
|
265
|
+
result_word = "result" if len(outcome.sources) == 1 else "results"
|
|
266
|
+
emit_progress(
|
|
267
|
+
progress,
|
|
268
|
+
"search_done",
|
|
269
|
+
f"{len(outcome.sources)} {result_word}",
|
|
270
|
+
question=sub_question.question,
|
|
271
|
+
count=len(outcome.sources),
|
|
272
|
+
)
|
|
192
273
|
bucket = buckets.setdefault(sub_question.question, [])
|
|
193
274
|
for source in outcome.sources:
|
|
194
275
|
key = normalize_url(source.url)
|
|
@@ -216,6 +297,7 @@ async def run_deep_research(
|
|
|
216
297
|
ledger,
|
|
217
298
|
reads,
|
|
218
299
|
max_claims=settings.max_claims,
|
|
300
|
+
progress=progress,
|
|
219
301
|
)
|
|
220
302
|
if not added:
|
|
221
303
|
break
|
|
@@ -226,9 +308,15 @@ async def run_deep_research(
|
|
|
226
308
|
await search_question(sub_question)
|
|
227
309
|
await read_round_robin()
|
|
228
310
|
|
|
229
|
-
for
|
|
311
|
+
for round_index in range(rounds):
|
|
230
312
|
if timed_out() or len(sources) >= limit:
|
|
231
313
|
break
|
|
314
|
+
emit_progress(
|
|
315
|
+
progress,
|
|
316
|
+
"gap",
|
|
317
|
+
f"Checking coverage (round {round_index + 1})",
|
|
318
|
+
round=round_index + 1,
|
|
319
|
+
)
|
|
232
320
|
gap = await check_gaps(
|
|
233
321
|
planner_llm,
|
|
234
322
|
normalized,
|
|
@@ -237,6 +325,13 @@ async def run_deep_research(
|
|
|
237
325
|
tuple(sources),
|
|
238
326
|
max_follow_ups=settings.max_follow_ups,
|
|
239
327
|
)
|
|
328
|
+
emit_progress(
|
|
329
|
+
progress,
|
|
330
|
+
"gap_done",
|
|
331
|
+
"Coverage sufficient" if gap.done else f"{len(gap.follow_up)} follow-up questions",
|
|
332
|
+
done=gap.done,
|
|
333
|
+
follow_up=[item.question for item in gap.follow_up],
|
|
334
|
+
)
|
|
240
335
|
if gap.done or not gap.follow_up:
|
|
241
336
|
break
|
|
242
337
|
plan = replace(plan, sub_questions=plan.sub_questions + gap.follow_up)
|
|
@@ -246,6 +341,13 @@ async def run_deep_research(
|
|
|
246
341
|
await search_question(follow_up)
|
|
247
342
|
await read_round_robin()
|
|
248
343
|
|
|
344
|
+
emit_progress(
|
|
345
|
+
progress,
|
|
346
|
+
"synthesis",
|
|
347
|
+
"Writing the cited report",
|
|
348
|
+
citations=len(ledger.citations),
|
|
349
|
+
sources=len(sources),
|
|
350
|
+
)
|
|
249
351
|
try:
|
|
250
352
|
narrative = await synthesize_report(
|
|
251
353
|
writer_llm,
|
|
@@ -259,14 +361,36 @@ async def run_deep_research(
|
|
|
259
361
|
"Synthesized with inline citations. Verify important claims against "
|
|
260
362
|
"the source quotes below."
|
|
261
363
|
)
|
|
364
|
+
emit_progress(
|
|
365
|
+
progress,
|
|
366
|
+
"synthesis_done",
|
|
367
|
+
"Report synthesized",
|
|
368
|
+
citations=len(ledger.citations),
|
|
369
|
+
)
|
|
262
370
|
except (LLMError, SynthesisError) as exc:
|
|
371
|
+
error = _error_text(exc)
|
|
263
372
|
narrative = (
|
|
264
373
|
"No synthesized narrative was produced. The grounded evidence table "
|
|
265
374
|
"below lists the extracted claims."
|
|
266
375
|
)
|
|
267
|
-
notice = f"Synthesis unavailable: {
|
|
376
|
+
notice = f"Synthesis unavailable: {error}"
|
|
377
|
+
emit_progress(
|
|
378
|
+
progress,
|
|
379
|
+
"synthesis_fallback",
|
|
380
|
+
"Synthesis unavailable; using the evidence table",
|
|
381
|
+
error=error,
|
|
382
|
+
)
|
|
268
383
|
|
|
269
384
|
usage = tracker.to_models()
|
|
385
|
+
emit_progress(
|
|
386
|
+
progress,
|
|
387
|
+
"done",
|
|
388
|
+
f"Research complete: {len(ledger.citations)} citations",
|
|
389
|
+
citations=len(ledger.citations),
|
|
390
|
+
sources=len(sources),
|
|
391
|
+
reads=len(reads),
|
|
392
|
+
calls=sum(item.calls for item in usage),
|
|
393
|
+
)
|
|
270
394
|
report = _compose_report(
|
|
271
395
|
normalized,
|
|
272
396
|
plan,
|
|
@@ -17,6 +17,7 @@ from ..models import (
|
|
|
17
17
|
Source,
|
|
18
18
|
utc_now,
|
|
19
19
|
)
|
|
20
|
+
from ..progress import ProgressCallback, emit_progress
|
|
20
21
|
from ..search.chain import SearchChain, build_chain, normalize_url
|
|
21
22
|
from .planner import plan_question
|
|
22
23
|
|
|
@@ -146,8 +147,16 @@ async def run_research(
|
|
|
146
147
|
chain: SearchChain | None = None,
|
|
147
148
|
plan: ResearchPlan | None = None,
|
|
148
149
|
max_sources: int | None = None,
|
|
150
|
+
progress: ProgressCallback | None = None,
|
|
149
151
|
) -> Run:
|
|
150
152
|
plan = plan or plan_question(question, max_sub_questions=4)
|
|
153
|
+
emit_progress(
|
|
154
|
+
progress,
|
|
155
|
+
"plan",
|
|
156
|
+
f"Planned {len(plan.sub_questions)} sub-questions",
|
|
157
|
+
planner=plan.planner,
|
|
158
|
+
sub_questions=[item.question for item in plan.sub_questions],
|
|
159
|
+
)
|
|
151
160
|
chain = chain or build_chain(settings)
|
|
152
161
|
if not chain.providers:
|
|
153
162
|
raise RuntimeError(
|
|
@@ -163,9 +172,23 @@ async def run_research(
|
|
|
163
172
|
attempts: list[SearchAttempt] = []
|
|
164
173
|
|
|
165
174
|
for sub_question in plan.sub_questions:
|
|
175
|
+
emit_progress(
|
|
176
|
+
progress,
|
|
177
|
+
"search",
|
|
178
|
+
f"Searching: {sub_question.question}",
|
|
179
|
+
question=sub_question.question,
|
|
180
|
+
)
|
|
166
181
|
outcome: SearchOutcome = await chain.search(
|
|
167
182
|
sub_question.question, limit=settings.search_limit
|
|
168
183
|
)
|
|
184
|
+
result_word = "result" if len(outcome.sources) == 1 else "results"
|
|
185
|
+
emit_progress(
|
|
186
|
+
progress,
|
|
187
|
+
"search_done",
|
|
188
|
+
f"{len(outcome.sources)} {result_word}",
|
|
189
|
+
question=sub_question.question,
|
|
190
|
+
count=len(outcome.sources),
|
|
191
|
+
)
|
|
169
192
|
attempts.extend(outcome.attempts)
|
|
170
193
|
for source in outcome.sources:
|
|
171
194
|
key = normalize_url(source.url)
|
|
@@ -191,6 +214,12 @@ async def run_research(
|
|
|
191
214
|
break
|
|
192
215
|
|
|
193
216
|
selected = tuple(selected)
|
|
217
|
+
emit_progress(
|
|
218
|
+
progress,
|
|
219
|
+
"done",
|
|
220
|
+
f"Collected {len(selected)} sources",
|
|
221
|
+
sources=len(selected),
|
|
222
|
+
)
|
|
194
223
|
report = render_dossier(plan, selected, tuple(attempts))
|
|
195
224
|
return Run(
|
|
196
225
|
id=run_id(plan.question),
|