indagar 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indagar-0.1.0/.env.example +25 -0
- indagar-0.1.0/.github/workflows/ci.yml +24 -0
- indagar-0.1.0/.gitignore +15 -0
- indagar-0.1.0/AGENTS.md +23 -0
- indagar-0.1.0/CHANGELOG.md +16 -0
- indagar-0.1.0/LICENSE +21 -0
- indagar-0.1.0/PKG-INFO +102 -0
- indagar-0.1.0/README.md +78 -0
- indagar-0.1.0/SPEC.md +197 -0
- indagar-0.1.0/pyproject.toml +51 -0
- indagar-0.1.0/src/indagar/__init__.py +3 -0
- indagar-0.1.0/src/indagar/__main__.py +4 -0
- indagar-0.1.0/src/indagar/cli.py +140 -0
- indagar-0.1.0/src/indagar/config.py +82 -0
- indagar-0.1.0/src/indagar/models.py +90 -0
- indagar-0.1.0/src/indagar/research/__init__.py +4 -0
- indagar-0.1.0/src/indagar/research/loop.py +147 -0
- indagar-0.1.0/src/indagar/research/planner.py +25 -0
- indagar-0.1.0/src/indagar/search/__init__.py +4 -0
- indagar-0.1.0/src/indagar/search/base.py +22 -0
- indagar-0.1.0/src/indagar/search/chain.py +164 -0
- indagar-0.1.0/src/indagar/search/providers.py +216 -0
- indagar-0.1.0/src/indagar/store.py +28 -0
- indagar-0.1.0/tests/test_chain.py +92 -0
- indagar-0.1.0/tests/test_config.py +25 -0
- indagar-0.1.0/tests/test_loop.py +46 -0
- indagar-0.1.0/tests/test_planner.py +22 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Copy to .env and fill in the providers you have.
|
|
2
|
+
|
|
3
|
+
# Search: first configured provider in this order wins.
|
|
4
|
+
INDAGAR_SEARCH_ORDER=firecrawl,brave,serper,searxng
|
|
5
|
+
|
|
6
|
+
# Firecrawl: self-hosted is fine and can be keyless.
|
|
7
|
+
FIRECRAWL_URL=http://localhost:3002
|
|
8
|
+
# FIRECRAWL_API_KEY=
|
|
9
|
+
|
|
10
|
+
# Keyed search providers.
|
|
11
|
+
BRAVE_API_KEY=
|
|
12
|
+
SERPER_API_KEY=
|
|
13
|
+
|
|
14
|
+
# Self-hosted SearXNG.
|
|
15
|
+
# SEARXNG_URL=http://127.0.0.1:8888
|
|
16
|
+
|
|
17
|
+
# LLM providers for the next phase.
|
|
18
|
+
ANTHROPIC_API_KEY=
|
|
19
|
+
OPENAI_API_KEY=
|
|
20
|
+
OPENAI_BASE_URL=https://api.openai.com/v1
|
|
21
|
+
|
|
22
|
+
# Budgets.
|
|
23
|
+
INDAGAR_MAX_SOURCES=12
|
|
24
|
+
INDAGAR_MAX_SECONDS=300
|
|
25
|
+
INDAGAR_SEARCH_LIMIT=8
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- name: Install
|
|
16
|
+
run: |
|
|
17
|
+
python -m pip install --upgrade pip
|
|
18
|
+
pip install -e '.[dev]'
|
|
19
|
+
- name: Lint
|
|
20
|
+
run: ruff check .
|
|
21
|
+
- name: Format
|
|
22
|
+
run: ruff format --check .
|
|
23
|
+
- name: Test
|
|
24
|
+
run: pytest
|
indagar-0.1.0/.gitignore
ADDED
indagar-0.1.0/AGENTS.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Indagar — agent notes
|
|
2
|
+
|
|
3
|
+
## Commands
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
uv venv
|
|
7
|
+
source .venv/bin/activate
|
|
8
|
+
uv pip install -e '.[dev]'
|
|
9
|
+
ruff check .
|
|
10
|
+
ruff format --check .
|
|
11
|
+
pytest -q
|
|
12
|
+
indagar doctor
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Invariants
|
|
16
|
+
|
|
17
|
+
- Search providers are ordered. One provider failure never fails the chain.
|
|
18
|
+
- Every run records provider attempts, source URLs, and sub-questions.
|
|
19
|
+
- No API keys, tokens, or secrets are written to run records or reports.
|
|
20
|
+
- Phase 0 does not call an LLM. `research` collects and dedupes sources.
|
|
21
|
+
- Web and media sources are not peer-reviewed and must not be labelled as such.
|
|
22
|
+
- Keep the CLI flow: `doctor` is offline, `search` is retrieval-only, `research`
|
|
23
|
+
writes a run record and dossier.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Indagar are documented here. The format follows Keep a
|
|
4
|
+
Changelog, and the project uses semantic versioning.
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- Phase 0 CLI skeleton: `doctor`, `search`, and `research`.
|
|
11
|
+
- Firecrawl, Brave, Serper, and SearXNG search adapters.
|
|
12
|
+
- Ordered fallback, over-fetch, URL normalisation, dedupe, and a 10-minute
|
|
13
|
+
query cache.
|
|
14
|
+
- Deterministic sub-question planner and a cited source dossier.
|
|
15
|
+
- Run JSON and Markdown storage.
|
|
16
|
+
- Tests for the search chain, planner, and research loop.
|
indagar-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Michael Borck
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
indagar-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: indagar
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
|
|
5
|
+
Author: Michael Borck
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: citations,cli,deep-research,research,web-search
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Requires-Dist: httpx>=0.27
|
|
17
|
+
Requires-Dist: python-dotenv>=1.0
|
|
18
|
+
Requires-Dist: rich>=13.7
|
|
19
|
+
Requires-Dist: typer>=0.12
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# Indagar
|
|
26
|
+
|
|
27
|
+
**Deep research from the command line.**
|
|
28
|
+
|
|
29
|
+
Indagar asks a question, plans sub-questions, searches multiple sources, reads
|
|
30
|
+
what it finds, and produces a cited research dossier. The name is Spanish for
|
|
31
|
+
*to investigate, inquire, or probe*.
|
|
32
|
+
|
|
33
|
+
This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
|
|
34
|
+
chain, a deterministic research plan, and a source dossier. LLM planning,
|
|
35
|
+
extraction, and synthesis are the next phase.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
git clone https://github.com/michael-borck/indagar.git
|
|
41
|
+
cd indagar
|
|
42
|
+
uv venv
|
|
43
|
+
source .venv/bin/activate
|
|
44
|
+
uv pip install -e '.[dev]'
|
|
45
|
+
|
|
46
|
+
indagar doctor
|
|
47
|
+
indagar search "retrieval augmented generation" --limit 5
|
|
48
|
+
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
49
|
+
--max-sources 10
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
If the checkout is on an exFAT drive, create the virtual environment on a
|
|
53
|
+
local disk instead: `uv venv ~/.venvs/indagar`.
|
|
54
|
+
|
|
55
|
+
The `research` command writes a Markdown dossier and a JSON run record under
|
|
56
|
+
`runs/`. Every source keeps its URL, provider, and sub-question.
|
|
57
|
+
|
|
58
|
+
## Configuration
|
|
59
|
+
|
|
60
|
+
Copy `.env.example` to `.env` and set the providers you have.
|
|
61
|
+
|
|
62
|
+
| Variable | Purpose |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
|
|
65
|
+
| `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
|
|
66
|
+
| `BRAVE_API_KEY` | Brave Search API key |
|
|
67
|
+
| `SERPER_API_KEY` | Serper API key |
|
|
68
|
+
| `SEARXNG_URL` | Self-hosted SearXNG base URL |
|
|
69
|
+
| `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
|
|
70
|
+
| `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
|
|
71
|
+
| `OPENAI_API_KEY` | OpenAI-compatible key |
|
|
72
|
+
| `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
|
|
73
|
+
| `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
|
|
74
|
+
| `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
|
|
75
|
+
| `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
|
|
76
|
+
|
|
77
|
+
`indagar doctor` shows what is configured.
|
|
78
|
+
|
|
79
|
+
## Status
|
|
80
|
+
|
|
81
|
+
- [x] CLI with `doctor`, `search`, and `research`
|
|
82
|
+
- [x] Firecrawl, Brave, Serper, and SearXNG adapters
|
|
83
|
+
- [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
|
|
84
|
+
- [x] Deterministic research plan and cited source dossier
|
|
85
|
+
- [ ] LLM planner, claim extraction, and synthesis
|
|
86
|
+
- [ ] Web UI and desktop shell
|
|
87
|
+
- [ ] Academic, podcast, and video sources
|
|
88
|
+
|
|
89
|
+
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
uv pip install -e '.[dev]'
|
|
95
|
+
ruff check .
|
|
96
|
+
ruff format --check .
|
|
97
|
+
pytest -q
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Licence
|
|
101
|
+
|
|
102
|
+
MIT. See [LICENSE](LICENSE).
|
indagar-0.1.0/README.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Indagar
|
|
2
|
+
|
|
3
|
+
**Deep research from the command line.**
|
|
4
|
+
|
|
5
|
+
Indagar asks a question, plans sub-questions, searches multiple sources, reads
|
|
6
|
+
what it finds, and produces a cited research dossier. The name is Spanish for
|
|
7
|
+
*to investigate, inquire, or probe*.
|
|
8
|
+
|
|
9
|
+
This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
|
|
10
|
+
chain, a deterministic research plan, and a source dossier. LLM planning,
|
|
11
|
+
extraction, and synthesis are the next phase.
|
|
12
|
+
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
git clone https://github.com/michael-borck/indagar.git
|
|
17
|
+
cd indagar
|
|
18
|
+
uv venv
|
|
19
|
+
source .venv/bin/activate
|
|
20
|
+
uv pip install -e '.[dev]'
|
|
21
|
+
|
|
22
|
+
indagar doctor
|
|
23
|
+
indagar search "retrieval augmented generation" --limit 5
|
|
24
|
+
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
25
|
+
--max-sources 10
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
If the checkout is on an exFAT drive, create the virtual environment on a
|
|
29
|
+
local disk instead: `uv venv ~/.venvs/indagar`.
|
|
30
|
+
|
|
31
|
+
The `research` command writes a Markdown dossier and a JSON run record under
|
|
32
|
+
`runs/`. Every source keeps its URL, provider, and sub-question.
|
|
33
|
+
|
|
34
|
+
## Configuration
|
|
35
|
+
|
|
36
|
+
Copy `.env.example` to `.env` and set the providers you have.
|
|
37
|
+
|
|
38
|
+
| Variable | Purpose |
|
|
39
|
+
|---|---|
|
|
40
|
+
| `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
|
|
41
|
+
| `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
|
|
42
|
+
| `BRAVE_API_KEY` | Brave Search API key |
|
|
43
|
+
| `SERPER_API_KEY` | Serper API key |
|
|
44
|
+
| `SEARXNG_URL` | Self-hosted SearXNG base URL |
|
|
45
|
+
| `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
|
|
46
|
+
| `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
|
|
47
|
+
| `OPENAI_API_KEY` | OpenAI-compatible key |
|
|
48
|
+
| `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
|
|
49
|
+
| `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
|
|
50
|
+
| `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
|
|
51
|
+
| `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
|
|
52
|
+
|
|
53
|
+
`indagar doctor` shows what is configured.
|
|
54
|
+
|
|
55
|
+
## Status
|
|
56
|
+
|
|
57
|
+
- [x] CLI with `doctor`, `search`, and `research`
|
|
58
|
+
- [x] Firecrawl, Brave, Serper, and SearXNG adapters
|
|
59
|
+
- [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
|
|
60
|
+
- [x] Deterministic research plan and cited source dossier
|
|
61
|
+
- [ ] LLM planner, claim extraction, and synthesis
|
|
62
|
+
- [ ] Web UI and desktop shell
|
|
63
|
+
- [ ] Academic, podcast, and video sources
|
|
64
|
+
|
|
65
|
+
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
66
|
+
|
|
67
|
+
## Development
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
uv pip install -e '.[dev]'
|
|
71
|
+
ruff check .
|
|
72
|
+
ruff format --check .
|
|
73
|
+
pytest -q
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Licence
|
|
77
|
+
|
|
78
|
+
MIT. See [LICENSE](LICENSE).
|
indagar-0.1.0/SPEC.md
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# Indagar — design specification
|
|
2
|
+
|
|
3
|
+
**Status:** Phase 0 skeleton
|
|
4
|
+
**Name:** Indagar (Spanish: to investigate, inquire, probe)
|
|
5
|
+
**Licence:** MIT
|
|
6
|
+
|
|
7
|
+
## 1. Vision
|
|
8
|
+
|
|
9
|
+
Turn a question into a researched report with a transparent trail:
|
|
10
|
+
|
|
11
|
+
1. Plan the question into sub-questions.
|
|
12
|
+
2. Search multiple providers.
|
|
13
|
+
3. Read the sources.
|
|
14
|
+
4. Extract notes and claims with source spans.
|
|
15
|
+
5. Find gaps and run focused follow-up rounds.
|
|
16
|
+
6. Synthesize a cited report.
|
|
17
|
+
7. Save the run so it can be inspected, resumed, or replayed.
|
|
18
|
+
|
|
19
|
+
The user should always be able to see what was searched, what was read, what
|
|
20
|
+
was cited, and what it cost.
|
|
21
|
+
|
|
22
|
+
## 2. Non-goals
|
|
23
|
+
|
|
24
|
+
- A systematic review engine. Winnow owns that.
|
|
25
|
+
- A NotebookLM clone over user-supplied sources. Open Notebook owns that.
|
|
26
|
+
- Provider-native deep research lock-in. The loop is ours so citations, budgets,
|
|
27
|
+
and source policy stay portable.
|
|
28
|
+
- A general chat client.
|
|
29
|
+
- Scraping behind logins, paywalls, or CAPTCHAs.
|
|
30
|
+
|
|
31
|
+
## 3. Source model
|
|
32
|
+
|
|
33
|
+
Sources are not equal. Every source carries a type and a confidence label.
|
|
34
|
+
|
|
35
|
+
| Type | Examples | Default confidence |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| `academic` | OpenAlex, Crossref, Semantic Scholar, arXiv | High |
|
|
38
|
+
| `institution` | Government, university, standards body | High |
|
|
39
|
+
| `news` | Reputable outlet with a date and byline | Medium |
|
|
40
|
+
| `practitioner` | Practitioner blog, vendor report, consultancy | Low to medium |
|
|
41
|
+
| `media` | Podcast, video, interview | Low to medium, transcript required |
|
|
42
|
+
| `web` | General page | Low until checked |
|
|
43
|
+
|
|
44
|
+
Academic records are verified separately. Web and media sources are never
|
|
45
|
+
presented as peer-reviewed.
|
|
46
|
+
|
|
47
|
+
## 4. Research loop
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
question
|
|
51
|
+
-> plan sub-questions
|
|
52
|
+
-> for each sub-question
|
|
53
|
+
-> search chain
|
|
54
|
+
-> read / extract
|
|
55
|
+
-> notes + claims + source spans
|
|
56
|
+
-> gap check
|
|
57
|
+
-> 0-2 follow-up rounds
|
|
58
|
+
-> synthesize report
|
|
59
|
+
-> save run
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Phase 0 uses a deterministic planner and stops after source collection. LLM
|
|
63
|
+
planning, extraction, and synthesis are Phase 1.
|
|
64
|
+
|
|
65
|
+
## 5. Search providers
|
|
66
|
+
|
|
67
|
+
Ordered fallback:
|
|
68
|
+
|
|
69
|
+
1. Firecrawl (`/v2/search`, self-hosted or cloud)
|
|
70
|
+
2. Brave Search
|
|
71
|
+
3. Serper
|
|
72
|
+
4. SearXNG
|
|
73
|
+
|
|
74
|
+
Rules:
|
|
75
|
+
|
|
76
|
+
- Over-fetch `limit * 2`, filter invalid URLs, dedupe by normalised URL, then
|
|
77
|
+
slice.
|
|
78
|
+
- Stop at the first provider with usable results.
|
|
79
|
+
- Record every attempt: provider, status, count, latency, error.
|
|
80
|
+
- Cache identical queries for 10 minutes.
|
|
81
|
+
- Never let one provider failure fail the run.
|
|
82
|
+
- A keyless self-hosted Firecrawl or SearXNG is a valid only provider.
|
|
83
|
+
|
|
84
|
+
## 6. LLM providers
|
|
85
|
+
|
|
86
|
+
Two adapters cover the practical surface:
|
|
87
|
+
|
|
88
|
+
- **Anthropic native** (`/v1/messages`).
|
|
89
|
+
- **OpenAI-compatible** (`/v1/chat/completions`), covering OpenAI, Groq,
|
|
90
|
+
Together, OpenRouter, Ollama, LM Studio, vLLM, DeepSeek, Mistral, and
|
|
91
|
+
Gemini's OpenAI-compatible endpoint.
|
|
92
|
+
|
|
93
|
+
A Google Gemini native adapter is optional and only worth adding for native
|
|
94
|
+
grounding or citations.
|
|
95
|
+
|
|
96
|
+
Model roles:
|
|
97
|
+
|
|
98
|
+
| Role | Job | Default shape |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| Planner | Sub-questions, search terms, gap checks | Cheap, structured output |
|
|
101
|
+
| Reader/extractor | Notes and claims from one source | Cheap/local, chunked |
|
|
102
|
+
| Synthesizer | Final report with citations | Best affordable model |
|
|
103
|
+
|
|
104
|
+
## 7. Budgets and rate limits
|
|
105
|
+
|
|
106
|
+
Every run has explicit caps:
|
|
107
|
+
|
|
108
|
+
- `max_depth` follow-up rounds
|
|
109
|
+
- `max_sources` total sources
|
|
110
|
+
- `max_seconds` wall clock
|
|
111
|
+
- `max_tokens` per model call and per run
|
|
112
|
+
- `concurrency` 2-4 by default
|
|
113
|
+
|
|
114
|
+
Controls:
|
|
115
|
+
|
|
116
|
+
- Per-provider token-bucket rate limiter and exponential backoff.
|
|
117
|
+
- Search and page caches.
|
|
118
|
+
- Small/local model for extraction and summarisation.
|
|
119
|
+
- Stop and resume: a run is a saved state, not a single function call.
|
|
120
|
+
- Visible estimate before a run and actual usage after it.
|
|
121
|
+
|
|
122
|
+
Deep research is mostly a scheduling and grounding problem. Treat every network
|
|
123
|
+
call as budgeted.
|
|
124
|
+
|
|
125
|
+
## 8. Architecture
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
src/indagar/
|
|
129
|
+
cli.py Typer commands: doctor, search, research
|
|
130
|
+
config.py Environment settings and provider status
|
|
131
|
+
models.py Source, SubQuestion, ResearchPlan, SearchOutcome, Run
|
|
132
|
+
store.py Run JSON + Markdown storage
|
|
133
|
+
search/
|
|
134
|
+
base.py SearchProvider protocol and SearchError
|
|
135
|
+
providers.py Firecrawl, Brave, Serper, SearXNG
|
|
136
|
+
chain.py Ordered fallback, dedupe, cache
|
|
137
|
+
research/
|
|
138
|
+
planner.py Deterministic Phase 0 planner
|
|
139
|
+
loop.py Search, collect, dedupe, dossier
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Phase 1 adds `providers/` for LLMs, `research/extract.py`,
|
|
143
|
+
`research/gaps.py`, `research/synthesize.py`, and a citation ledger.
|
|
144
|
+
|
|
145
|
+
## 9. Interfaces
|
|
146
|
+
|
|
147
|
+
### CLI (Phase 0)
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
indagar doctor
|
|
151
|
+
indagar search "question" --limit 8
|
|
152
|
+
indagar research "question" --max-sources 12 --out report.md
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Web UI (Phase 2)
|
|
156
|
+
|
|
157
|
+
- Question, budget, provider status.
|
|
158
|
+
- Live progress: plan, searches, sources read, gaps, synthesis.
|
|
159
|
+
- Source inspector with snippet, type, confidence, and citation.
|
|
160
|
+
- Report view and Markdown/DOCX/PDF export.
|
|
161
|
+
- Pause, resume, stop.
|
|
162
|
+
- Cost and rate-limit panel.
|
|
163
|
+
|
|
164
|
+
### Desktop (Phase 3)
|
|
165
|
+
|
|
166
|
+
Electron or Tauri shell around the local FastAPI server, with the same web UI.
|
|
167
|
+
Reuse the Winnow/HoneyGuide/Curriculum Curator shell pattern.
|
|
168
|
+
|
|
169
|
+
## 10. Reuse
|
|
170
|
+
|
|
171
|
+
| Source | Borrow |
|
|
172
|
+
|---|---|
|
|
173
|
+
| Firecrawl | Search plus scrape, PDF parsing, crawl/map, caching. Deep-research loop: generate queries, search+scrape, analyse, recurse, stop on depth/URLs/time. |
|
|
174
|
+
| Minions | Local/cloud split and map-reduce: local chunks and summaries, remote synthesis. |
|
|
175
|
+
| Open Notebook | Source processing engines (Firecrawl, Jina, Crawl4AI, simple), Ask strategy -> searches -> partial answers -> final synthesis, citations, notes, duplicate detection. |
|
|
176
|
+
| Winnow | CLI, local server, desktop shell, provider abstraction, run database, citation discipline. |
|
|
177
|
+
| HoneyGuide | Multi-source retrieval, diversity, source-type labels, verification. |
|
|
178
|
+
| Curriculum Curator | Python search router, fallback, cache, key hashing, quota. |
|
|
179
|
+
|
|
180
|
+
Do not fork these projects. Borrow patterns and keep Indagar thin.
|
|
181
|
+
|
|
182
|
+
## 11. Phases
|
|
183
|
+
|
|
184
|
+
| Phase | Scope |
|
|
185
|
+
|---|---|
|
|
186
|
+
| 0 | Skeleton: CLI, search chain, deterministic plan, source dossier, tests, CI |
|
|
187
|
+
| 1 | LLM planner, reader/extractor, gap loop, synthesis, citation ledger, cost panel |
|
|
188
|
+
| 2 | Web UI, resume, source inspector, exports |
|
|
189
|
+
| 3 | Desktop shell, academic APIs, podcasts/videos, provider-native search as an option |
|
|
190
|
+
|
|
191
|
+
## 12. Phase 0 acceptance
|
|
192
|
+
|
|
193
|
+
- `indagar doctor` reports configured search and LLM providers.
|
|
194
|
+
- `indagar search` uses the ordered chain and reports attempts.
|
|
195
|
+
- `indagar research` writes a Markdown dossier and JSON run record.
|
|
196
|
+
- No provider key is required for tests.
|
|
197
|
+
- `ruff check`, `ruff format --check`, and `pytest` pass.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "indagar"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Deep research from the command line: plan, search, read, and synthesize with citations."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [{ name = "Michael Borck" }]
|
|
9
|
+
keywords = ["research", "deep-research", "web-search", "citations", "cli"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"License :: OSI Approved :: MIT License",
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"Programming Language :: Python :: 3.11",
|
|
14
|
+
"Programming Language :: Python :: 3.12",
|
|
15
|
+
"Programming Language :: Python :: 3.13",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
]
|
|
18
|
+
dependencies = [
|
|
19
|
+
"httpx>=0.27",
|
|
20
|
+
"python-dotenv>=1.0",
|
|
21
|
+
"rich>=13.7",
|
|
22
|
+
"typer>=0.12",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = [
|
|
27
|
+
"pytest>=8",
|
|
28
|
+
"ruff>=0.6",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.scripts]
|
|
32
|
+
indagar = "indagar.cli:main"
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["hatchling"]
|
|
36
|
+
build-backend = "hatchling.build"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/indagar"]
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 100
|
|
43
|
+
target-version = "py311"
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint]
|
|
46
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
47
|
+
ignore = ["B008"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
51
|
+
addopts = "-q"
|