indagar 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ # Copy to .env and fill in the providers you have.
2
+
3
+ # Search: first configured provider in this order wins.
4
+ INDAGAR_SEARCH_ORDER=firecrawl,brave,serper,searxng
5
+
6
+ # Firecrawl: self-hosted is fine and can be keyless.
7
+ FIRECRAWL_URL=http://localhost:3002
8
+ # FIRECRAWL_API_KEY=
9
+
10
+ # Keyed search providers.
11
+ BRAVE_API_KEY=
12
+ SERPER_API_KEY=
13
+
14
+ # Self-hosted SearXNG.
15
+ # SEARXNG_URL=http://127.0.0.1:8888
16
+
17
+ # LLM providers for the next phase.
18
+ ANTHROPIC_API_KEY=
19
+ OPENAI_API_KEY=
20
+ OPENAI_BASE_URL=https://api.openai.com/v1
21
+
22
+ # Budgets.
23
+ INDAGAR_MAX_SOURCES=12
24
+ INDAGAR_MAX_SECONDS=300
25
+ INDAGAR_SEARCH_LIMIT=8
@@ -0,0 +1,24 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-python@v5
13
+ with:
14
+ python-version: "3.12"
15
+ - name: Install
16
+ run: |
17
+ python -m pip install --upgrade pip
18
+ pip install -e '.[dev]'
19
+ - name: Lint
20
+ run: ruff check .
21
+ - name: Format
22
+ run: ruff format --check .
23
+ - name: Test
24
+ run: pytest
@@ -0,0 +1,15 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ .dist/
8
+ build/
9
+ dist/
10
+ runs/
11
+ .env
12
+ .env.*
13
+ !.env.example
14
+ .DS_Store
15
+ ._*
@@ -0,0 +1,23 @@
1
+ # Indagar — agent notes
2
+
3
+ ## Commands
4
+
5
+ ```bash
6
+ uv venv
7
+ source .venv/bin/activate
8
+ uv pip install -e '.[dev]'
9
+ ruff check .
10
+ ruff format --check .
11
+ pytest -q
12
+ indagar doctor
13
+ ```
14
+
15
+ ## Invariants
16
+
17
+ - Search providers are ordered. One provider failure never fails the chain.
18
+ - Every run records provider attempts, source URLs, and sub-questions.
19
+ - No API keys, tokens, or secrets are written to run records or reports.
20
+ - Phase 0 does not call an LLM. `research` collects and dedupes sources.
21
+ - Web and media sources are not peer-reviewed and must not be labelled as such.
22
+ - Keep the CLI flow: `doctor` is offline, `search` is retrieval-only, `research`
23
+ writes a run record and dossier.
@@ -0,0 +1,16 @@
1
+ # Changelog
2
+
3
+ All notable changes to Indagar are documented here. The format follows Keep a
4
+ Changelog, and the project uses semantic versioning.
5
+
6
+ ## [Unreleased]
7
+
8
+ ### Added
9
+
10
+ - Phase 0 CLI skeleton: `doctor`, `search`, and `research`.
11
+ - Firecrawl, Brave, Serper, and SearXNG search adapters.
12
+ - Ordered fallback, over-fetch, URL normalisation, dedupe, and a 10-minute
13
+ query cache.
14
+ - Deterministic sub-question planner and a cited source dossier.
15
+ - Run JSON and Markdown storage.
16
+ - Tests for the search chain, planner, and research loop.
indagar-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Michael Borck
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
indagar-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.5
2
+ Name: indagar
3
+ Version: 0.1.0
4
+ Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
5
+ Author: Michael Borck
6
+ License: MIT
7
+ License-File: LICENSE
8
+ Keywords: citations,cli,deep-research,research,web-search
9
+ Classifier: Environment :: Console
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Requires-Python: >=3.11
16
+ Requires-Dist: httpx>=0.27
17
+ Requires-Dist: python-dotenv>=1.0
18
+ Requires-Dist: rich>=13.7
19
+ Requires-Dist: typer>=0.12
20
+ Provides-Extra: dev
21
+ Requires-Dist: pytest>=8; extra == 'dev'
22
+ Requires-Dist: ruff>=0.6; extra == 'dev'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # Indagar
26
+
27
+ **Deep research from the command line.**
28
+
29
+ Indagar asks a question, plans sub-questions, searches multiple sources, reads
30
+ what it finds, and produces a cited research dossier. The name is Spanish for
31
+ *to investigate, inquire, or probe*.
32
+
33
+ This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
34
+ chain, a deterministic research plan, and a source dossier. LLM planning,
35
+ extraction, and synthesis are the next phase.
36
+
37
+ ## Quick start
38
+
39
+ ```bash
40
+ git clone https://github.com/michael-borck/indagar.git
41
+ cd indagar
42
+ uv venv
43
+ source .venv/bin/activate
44
+ uv pip install -e '.[dev]'
45
+
46
+ indagar doctor
47
+ indagar search "retrieval augmented generation" --limit 5
48
+ indagar research "Does retrieval augmented generation reduce hallucination?" \
49
+ --max-sources 10
50
+ ```
51
+
52
+ If the checkout is on an exFAT drive, create the virtual environment on a
53
+ local disk instead: `uv venv ~/.venvs/indagar`.
54
+
55
+ The `research` command writes a Markdown dossier and a JSON run record under
56
+ `runs/`. Every source keeps its URL, provider, and sub-question.
57
+
58
+ ## Configuration
59
+
60
+ Copy `.env.example` to `.env` and set the providers you have.
61
+
62
+ | Variable | Purpose |
63
+ |---|---|
64
+ | `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
65
+ | `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
66
+ | `BRAVE_API_KEY` | Brave Search API key |
67
+ | `SERPER_API_KEY` | Serper API key |
68
+ | `SEARXNG_URL` | Self-hosted SearXNG base URL |
69
+ | `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
70
+ | `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
71
+ | `OPENAI_API_KEY` | OpenAI-compatible key |
72
+ | `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
73
+ | `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
74
+ | `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
75
+ | `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
76
+
77
+ `indagar doctor` shows what is configured.
78
+
79
+ ## Status
80
+
81
+ - [x] CLI with `doctor`, `search`, and `research`
82
+ - [x] Firecrawl, Brave, Serper, and SearXNG adapters
83
+ - [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
84
+ - [x] Deterministic research plan and cited source dossier
85
+ - [ ] LLM planner, claim extraction, and synthesis
86
+ - [ ] Web UI and desktop shell
87
+ - [ ] Academic, podcast, and video sources
88
+
89
+ See [SPEC.md](SPEC.md) for the design and roadmap.
90
+
91
+ ## Development
92
+
93
+ ```bash
94
+ uv pip install -e '.[dev]'
95
+ ruff check .
96
+ ruff format --check .
97
+ pytest -q
98
+ ```
99
+
100
+ ## Licence
101
+
102
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,78 @@
1
+ # Indagar
2
+
3
+ **Deep research from the command line.**
4
+
5
+ Indagar asks a question, plans sub-questions, searches multiple sources, reads
6
+ what it finds, and produces a cited research dossier. The name is Spanish for
7
+ *to investigate, inquire, or probe*.
8
+
9
+ This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
10
+ chain, a deterministic research plan, and a source dossier. LLM planning,
11
+ extraction, and synthesis are the next phase.
12
+
13
+ ## Quick start
14
+
15
+ ```bash
16
+ git clone https://github.com/michael-borck/indagar.git
17
+ cd indagar
18
+ uv venv
19
+ source .venv/bin/activate
20
+ uv pip install -e '.[dev]'
21
+
22
+ indagar doctor
23
+ indagar search "retrieval augmented generation" --limit 5
24
+ indagar research "Does retrieval augmented generation reduce hallucination?" \
25
+ --max-sources 10
26
+ ```
27
+
28
+ If the checkout is on an exFAT drive, create the virtual environment on a
29
+ local disk instead: `uv venv ~/.venvs/indagar`.
30
+
31
+ The `research` command writes a Markdown dossier and a JSON run record under
32
+ `runs/`. Every source keeps its URL, provider, and sub-question.
33
+
34
+ ## Configuration
35
+
36
+ Copy `.env.example` to `.env` and set the providers you have.
37
+
38
+ | Variable | Purpose |
39
+ |---|---|
40
+ | `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
41
+ | `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
42
+ | `BRAVE_API_KEY` | Brave Search API key |
43
+ | `SERPER_API_KEY` | Serper API key |
44
+ | `SEARXNG_URL` | Self-hosted SearXNG base URL |
45
+ | `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
46
+ | `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
47
+ | `OPENAI_API_KEY` | OpenAI-compatible key |
48
+ | `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
49
+ | `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
50
+ | `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
51
+ | `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
52
+
53
+ `indagar doctor` shows what is configured.
54
+
55
+ ## Status
56
+
57
+ - [x] CLI with `doctor`, `search`, and `research`
58
+ - [x] Firecrawl, Brave, Serper, and SearXNG adapters
59
+ - [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
60
+ - [x] Deterministic research plan and cited source dossier
61
+ - [ ] LLM planner, claim extraction, and synthesis
62
+ - [ ] Web UI and desktop shell
63
+ - [ ] Academic, podcast, and video sources
64
+
65
+ See [SPEC.md](SPEC.md) for the design and roadmap.
66
+
67
+ ## Development
68
+
69
+ ```bash
70
+ uv pip install -e '.[dev]'
71
+ ruff check .
72
+ ruff format --check .
73
+ pytest -q
74
+ ```
75
+
76
+ ## Licence
77
+
78
+ MIT. See [LICENSE](LICENSE).
indagar-0.1.0/SPEC.md ADDED
@@ -0,0 +1,197 @@
1
+ # Indagar — design specification
2
+
3
+ **Status:** Phase 0 skeleton
4
+ **Name:** Indagar (Spanish: to investigate, inquire, probe)
5
+ **Licence:** MIT
6
+
7
+ ## 1. Vision
8
+
9
+ Turn a question into a researched report with a transparent trail:
10
+
11
+ 1. Plan the question into sub-questions.
12
+ 2. Search multiple providers.
13
+ 3. Read the sources.
14
+ 4. Extract notes and claims with source spans.
15
+ 5. Find gaps and run focused follow-up rounds.
16
+ 6. Synthesize a cited report.
17
+ 7. Save the run so it can be inspected, resumed, or replayed.
18
+
19
+ The user should always be able to see what was searched, what was read, what
20
+ was cited, and what it cost.
21
+
22
+ ## 2. Non-goals
23
+
24
+ - A systematic review engine. Winnow owns that.
25
+ - A NotebookLM clone over user-supplied sources. Open Notebook owns that.
26
+ - Provider-native deep research lock-in. The loop is ours so citations, budgets,
27
+ and source policy stay portable.
28
+ - A general chat client.
29
+ - Scraping behind logins, paywalls, or CAPTCHAs.
30
+
31
+ ## 3. Source model
32
+
33
+ Sources are not equal. Every source carries a type and a confidence label.
34
+
35
+ | Type | Examples | Default confidence |
36
+ |---|---|---|
37
+ | `academic` | OpenAlex, Crossref, Semantic Scholar, arXiv | High |
38
+ | `institution` | Government, university, standards body | High |
39
+ | `news` | Reputable outlet with a date and byline | Medium |
40
+ | `practitioner` | Practitioner blog, vendor report, consultancy | Low to medium |
41
+ | `media` | Podcast, video, interview | Low to medium, transcript required |
42
+ | `web` | General page | Low until checked |
43
+
44
+ Academic records are verified separately. Web and media sources are never
45
+ presented as peer-reviewed.
46
+
47
+ ## 4. Research loop
48
+
49
+ ```
50
+ question
51
+ -> plan sub-questions
52
+ -> for each sub-question
53
+ -> search chain
54
+ -> read / extract
55
+ -> notes + claims + source spans
56
+ -> gap check
57
+ -> 0-2 follow-up rounds
58
+ -> synthesize report
59
+ -> save run
60
+ ```
61
+
62
+ Phase 0 uses a deterministic planner and stops after source collection. LLM
63
+ planning, extraction, and synthesis are Phase 1.
64
+
65
+ ## 5. Search providers
66
+
67
+ Ordered fallback:
68
+
69
+ 1. Firecrawl (`/v2/search`, self-hosted or cloud)
70
+ 2. Brave Search
71
+ 3. Serper
72
+ 4. SearXNG
73
+
74
+ Rules:
75
+
76
+ - Over-fetch `limit * 2`, filter invalid URLs, dedupe by normalised URL, then
77
+ slice.
78
+ - Stop at the first provider with usable results.
79
+ - Record every attempt: provider, status, count, latency, error.
80
+ - Cache identical queries for 10 minutes.
81
+ - Never let one provider failure fail the run.
82
+ - A keyless self-hosted Firecrawl or SearXNG is a valid only provider.
83
+
84
+ ## 6. LLM providers
85
+
86
+ Two adapters cover the practical surface:
87
+
88
+ - **Anthropic native** (`/v1/messages`).
89
+ - **OpenAI-compatible** (`/v1/chat/completions`), covering OpenAI, Groq,
90
+ Together, OpenRouter, Ollama, LM Studio, vLLM, DeepSeek, Mistral, and
91
+ Gemini's OpenAI-compatible endpoint.
92
+
93
+ A Google Gemini native adapter is optional and only worth adding for native
94
+ grounding or citations.
95
+
96
+ Model roles:
97
+
98
+ | Role | Job | Default shape |
99
+ |---|---|---|
100
+ | Planner | Sub-questions, search terms, gap checks | Cheap, structured output |
101
+ | Reader/extractor | Notes and claims from one source | Cheap/local, chunked |
102
+ | Synthesizer | Final report with citations | Best affordable model |
103
+
104
+ ## 7. Budgets and rate limits
105
+
106
+ Every run has explicit caps:
107
+
108
+ - `max_depth` follow-up rounds
109
+ - `max_sources` total sources
110
+ - `max_seconds` wall clock
111
+ - `max_tokens` per model call and per run
112
+ - `concurrency` 2-4 by default
113
+
114
+ Controls:
115
+
116
+ - Per-provider token-bucket rate limiter and exponential backoff.
117
+ - Search and page caches.
118
+ - Small/local model for extraction and summarisation.
119
+ - Stop and resume: a run is a saved state, not a single function call.
120
+ - Visible estimate before a run and actual usage after it.
121
+
122
+ Deep research is mostly a scheduling and grounding problem. Treat every network
123
+ call as budgeted.
124
+
125
+ ## 8. Architecture
126
+
127
+ ```
128
+ src/indagar/
129
+ cli.py Typer commands: doctor, search, research
130
+ config.py Environment settings and provider status
131
+ models.py Source, SubQuestion, ResearchPlan, SearchOutcome, Run
132
+ store.py Run JSON + Markdown storage
133
+ search/
134
+ base.py SearchProvider protocol and SearchError
135
+ providers.py Firecrawl, Brave, Serper, SearXNG
136
+ chain.py Ordered fallback, dedupe, cache
137
+ research/
138
+ planner.py Deterministic Phase 0 planner
139
+ loop.py Search, collect, dedupe, dossier
140
+ ```
141
+
142
+ Phase 1 adds `providers/` for LLMs, `research/extract.py`,
143
+ `research/gaps.py`, `research/synthesize.py`, and a citation ledger.
144
+
145
+ ## 9. Interfaces
146
+
147
+ ### CLI (Phase 0)
148
+
149
+ ```bash
150
+ indagar doctor
151
+ indagar search "question" --limit 8
152
+ indagar research "question" --max-sources 12 --out report.md
153
+ ```
154
+
155
+ ### Web UI (Phase 2)
156
+
157
+ - Question, budget, provider status.
158
+ - Live progress: plan, searches, sources read, gaps, synthesis.
159
+ - Source inspector with snippet, type, confidence, and citation.
160
+ - Report view and Markdown/DOCX/PDF export.
161
+ - Pause, resume, stop.
162
+ - Cost and rate-limit panel.
163
+
164
+ ### Desktop (Phase 3)
165
+
166
+ Electron or Tauri shell around the local FastAPI server, with the same web UI.
167
+ Reuse the Winnow/HoneyGuide/Curriculum Curator shell pattern.
168
+
169
+ ## 10. Reuse
170
+
171
+ | Source | Borrow |
172
+ |---|---|
173
+ | Firecrawl | Search plus scrape, PDF parsing, crawl/map, caching. Deep-research loop: generate queries, search+scrape, analyse, recurse, stop on depth/URLs/time. |
174
+ | Minions | Local/cloud split and map-reduce: local chunks and summaries, remote synthesis. |
175
+ | Open Notebook | Source processing engines (Firecrawl, Jina, Crawl4AI, simple), Ask strategy -> searches -> partial answers -> final synthesis, citations, notes, duplicate detection. |
176
+ | Winnow | CLI, local server, desktop shell, provider abstraction, run database, citation discipline. |
177
+ | HoneyGuide | Multi-source retrieval, diversity, source-type labels, verification. |
178
+ | Curriculum Curator | Python search router, fallback, cache, key hashing, quota. |
179
+
180
+ Do not fork these projects. Borrow patterns and keep Indagar thin.
181
+
182
+ ## 11. Phases
183
+
184
+ | Phase | Scope |
185
+ |---|---|
186
+ | 0 | Skeleton: CLI, search chain, deterministic plan, source dossier, tests, CI |
187
+ | 1 | LLM planner, reader/extractor, gap loop, synthesis, citation ledger, cost panel |
188
+ | 2 | Web UI, resume, source inspector, exports |
189
+ | 3 | Desktop shell, academic APIs, podcasts/videos, provider-native search as an option |
190
+
191
+ ## 12. Phase 0 acceptance
192
+
193
+ - `indagar doctor` reports configured search and LLM providers.
194
+ - `indagar search` uses the ordered chain and reports attempts.
195
+ - `indagar research` writes a Markdown dossier and JSON run record.
196
+ - No provider key is required for tests.
197
+ - `ruff check`, `ruff format --check`, and `pytest` pass.
@@ -0,0 +1,51 @@
1
+ [project]
2
+ name = "indagar"
3
+ version = "0.1.0"
4
+ description = "Deep research from the command line: plan, search, read, and synthesize with citations."
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ license = { text = "MIT" }
8
+ authors = [{ name = "Michael Borck" }]
9
+ keywords = ["research", "deep-research", "web-search", "citations", "cli"]
10
+ classifiers = [
11
+ "License :: OSI Approved :: MIT License",
12
+ "Programming Language :: Python :: 3",
13
+ "Programming Language :: Python :: 3.11",
14
+ "Programming Language :: Python :: 3.12",
15
+ "Programming Language :: Python :: 3.13",
16
+ "Environment :: Console",
17
+ ]
18
+ dependencies = [
19
+ "httpx>=0.27",
20
+ "python-dotenv>=1.0",
21
+ "rich>=13.7",
22
+ "typer>=0.12",
23
+ ]
24
+
25
+ [project.optional-dependencies]
26
+ dev = [
27
+ "pytest>=8",
28
+ "ruff>=0.6",
29
+ ]
30
+
31
+ [project.scripts]
32
+ indagar = "indagar.cli:main"
33
+
34
+ [build-system]
35
+ requires = ["hatchling"]
36
+ build-backend = "hatchling.build"
37
+
38
+ [tool.hatch.build.targets.wheel]
39
+ packages = ["src/indagar"]
40
+
41
+ [tool.ruff]
42
+ line-length = 100
43
+ target-version = "py311"
44
+
45
+ [tool.ruff.lint]
46
+ select = ["E", "F", "I", "UP", "B"]
47
+ ignore = ["B008"]
48
+
49
+ [tool.pytest.ini_options]
50
+ testpaths = ["tests"]
51
+ addopts = "-q"
@@ -0,0 +1,3 @@
1
+ """Indagar — deep research from the command line."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()