indagar 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
indagar/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """Indagar — deep research from the command line."""
2
+
3
+ __version__ = "0.1.0"
indagar/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from .cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
indagar/cli.py ADDED
@@ -0,0 +1,140 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ from dataclasses import replace
5
+ from importlib.metadata import PackageNotFoundError
6
+ from importlib.metadata import version as package_version
7
+ from pathlib import Path
8
+
9
+ import typer
10
+ from rich.console import Console
11
+ from rich.table import Table
12
+
13
+ from .config import Settings
14
+ from .research.loop import run_research
15
+ from .search.chain import build_chain
16
+ from .store import save_run
17
+
18
+ app = typer.Typer(
19
+ add_completion=False,
20
+ no_args_is_help=True,
21
+ help="Indagar — deep research from the command line.",
22
+ )
23
+ console = Console()
24
+
25
+
26
+ def _version() -> str:
27
+ try:
28
+ return package_version("indagar")
29
+ except PackageNotFoundError:
30
+ return "0.1.0"
31
+
32
+
33
+ def _settings_or_exit() -> Settings:
34
+ return Settings.from_env()
35
+
36
+
37
+ @app.command()
38
+ def version() -> None:
39
+ """Print the installed version."""
40
+ console.print(_version())
41
+
42
+
43
+ @app.command()
44
+ def doctor() -> None:
45
+ """Show configured search and LLM providers and the current budgets."""
46
+ settings = _settings_or_exit()
47
+ configured = settings.configured_search_providers()
48
+
49
+ console.print("[bold]Indagar doctor[/bold]")
50
+ console.print(f"Search order: {', '.join(settings.search_order)}")
51
+ console.print(f"Configured search: {', '.join(configured) or 'none'}")
52
+ console.print(f"LLM providers: {', '.join(settings.llm_providers()) or 'none'}")
53
+ console.print(
54
+ f"Budgets: sources={settings.max_sources}, "
55
+ f"seconds={settings.max_seconds}, search_limit={settings.search_limit}"
56
+ )
57
+ if not configured:
58
+ console.print(
59
+ "[yellow]No search provider configured. Set FIRECRAWL_URL, "
60
+ "BRAVE_API_KEY, SERPER_API_KEY, or SEARXNG_URL.[/yellow]"
61
+ )
62
+
63
+
64
+ @app.command()
65
+ def search(
66
+ query: str = typer.Argument(..., help="Search query"),
67
+ limit: int = typer.Option(8, "--limit", "-n", min=1, max=20),
68
+ ) -> None:
69
+ """Run the configured search chain and print the results."""
70
+ settings = _settings_or_exit()
71
+ chain = build_chain(settings)
72
+ if not chain.providers:
73
+ console.print("[red]No search provider configured.[/red]")
74
+ raise typer.Exit(1)
75
+
76
+ try:
77
+ outcome = asyncio.run(chain.search(query, limit=limit))
78
+ except Exception as exc: # noqa: BLE001 - CLI boundary
79
+ console.print(f"[red]Search failed: {exc}[/red]")
80
+ raise typer.Exit(1) from exc
81
+
82
+ table = Table(title=f"Search: {query}", show_lines=False)
83
+ table.add_column("#", justify="right", style="dim")
84
+ table.add_column("Title", overflow="fold")
85
+ table.add_column("Domain", style="cyan")
86
+ table.add_column("Provider", style="green")
87
+ for index, source in enumerate(outcome.sources, start=1):
88
+ table.add_row(str(index), source.title, source.domain, source.provider)
89
+ console.print(table)
90
+
91
+ for attempt in outcome.attempts:
92
+ detail = f" — {attempt.error}" if attempt.error else ""
93
+ console.print(
94
+ f"[dim]{attempt.provider}: {attempt.status} "
95
+ f"({attempt.count}, {attempt.latency_ms}ms){detail}[/dim]"
96
+ )
97
+ if not outcome.sources:
98
+ console.print("[yellow]No results.[/yellow]")
99
+
100
+
101
+ @app.command()
102
+ def research(
103
+ question: str = typer.Argument(..., help="Question or topic to research"),
104
+ max_sources: int = typer.Option(0, "--max-sources", min=0, help="0 uses INDAGAR_MAX_SOURCES"),
105
+ out: Path | None = typer.Option(None, "--out", "-o", help="Markdown output path"),
106
+ ) -> None:
107
+ """Plan sub-questions, search, and write a source dossier."""
108
+ settings = _settings_or_exit()
109
+ if max_sources:
110
+ settings = replace(settings, max_sources=max_sources)
111
+
112
+ try:
113
+ run = asyncio.run(run_research(question, settings))
114
+ except (RuntimeError, ValueError) as exc:
115
+ console.print(f"[red]{exc}[/red]")
116
+ raise typer.Exit(1) from exc
117
+
118
+ console.print(f"[bold]Plan[/bold] ({len(run.plan.sub_questions)} sub-questions)")
119
+ for index, sub_question in enumerate(run.plan.sub_questions, start=1):
120
+ console.print(f" {index}. {sub_question.question}")
121
+
122
+ table = Table(title=f"Sources: {len(run.sources)}")
123
+ table.add_column("#", justify="right", style="dim")
124
+ table.add_column("Title", overflow="fold")
125
+ table.add_column("Domain", style="cyan")
126
+ table.add_column("Provider", style="green")
127
+ for index, source in enumerate(run.sources, start=1):
128
+ table.add_row(str(index), source.title, source.domain, source.provider)
129
+ console.print(table)
130
+
131
+ json_path, markdown_path = save_run(run, out=out)
132
+ console.print(f"Run record: {json_path}")
133
+ console.print(f"Dossier: {markdown_path}")
134
+
135
+
136
+ def main() -> None:
137
+ app()
138
+
139
+
140
+ __all__ = ["app", "main"]
indagar/config.py ADDED
@@ -0,0 +1,82 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+
6
+ from dotenv import load_dotenv
7
+
8
+ DEFAULT_SEARCH_ORDER = ("firecrawl", "brave", "serper", "searxng")
9
+ DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
10
+
11
+
12
+ def _env_int(name: str, default: int) -> int:
13
+ raw = os.getenv(name)
14
+ if raw is None:
15
+ return default
16
+ try:
17
+ return max(1, int(raw))
18
+ except ValueError:
19
+ return default
20
+
21
+
22
+ def _search_order() -> tuple[str, ...]:
23
+ raw = os.getenv("INDAGAR_SEARCH_ORDER", "")
24
+ if not raw.strip():
25
+ return DEFAULT_SEARCH_ORDER
26
+ names = tuple(part.strip().lower() for part in raw.split(",") if part.strip())
27
+ return names or DEFAULT_SEARCH_ORDER
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class Settings:
32
+ firecrawl_url: str | None = None
33
+ firecrawl_api_key: str | None = None
34
+ brave_api_key: str | None = None
35
+ serper_api_key: str | None = None
36
+ searxng_url: str | None = None
37
+ search_order: tuple[str, ...] = DEFAULT_SEARCH_ORDER
38
+ anthropic_api_key: str | None = None
39
+ openai_api_key: str | None = None
40
+ openai_base_url: str = DEFAULT_OPENAI_BASE_URL
41
+ max_sources: int = 12
42
+ max_seconds: int = 300
43
+ search_limit: int = 8
44
+
45
+ @classmethod
46
+ def from_env(cls) -> Settings:
47
+ load_dotenv(override=False)
48
+ return cls(
49
+ firecrawl_url=os.getenv("FIRECRAWL_URL", "").strip() or None,
50
+ firecrawl_api_key=os.getenv("FIRECRAWL_API_KEY", "").strip() or None,
51
+ brave_api_key=os.getenv("BRAVE_API_KEY", "").strip() or None,
52
+ serper_api_key=os.getenv("SERPER_API_KEY", "").strip() or None,
53
+ searxng_url=os.getenv("SEARXNG_URL", "").strip() or None,
54
+ search_order=_search_order(),
55
+ anthropic_api_key=os.getenv("ANTHROPIC_API_KEY", "").strip() or None,
56
+ openai_api_key=os.getenv("OPENAI_API_KEY", "").strip() or None,
57
+ openai_base_url=(os.getenv("OPENAI_BASE_URL", "").strip() or DEFAULT_OPENAI_BASE_URL),
58
+ max_sources=_env_int("INDAGAR_MAX_SOURCES", 12),
59
+ max_seconds=_env_int("INDAGAR_MAX_SECONDS", 300),
60
+ search_limit=_env_int("INDAGAR_SEARCH_LIMIT", 8),
61
+ )
62
+
63
+ def configured_search_providers(self) -> tuple[str, ...]:
64
+ configured: list[str] = []
65
+ for name in self.search_order:
66
+ if name == "firecrawl" and self.firecrawl_url:
67
+ configured.append(name)
68
+ elif name == "brave" and self.brave_api_key:
69
+ configured.append(name)
70
+ elif name == "serper" and self.serper_api_key:
71
+ configured.append(name)
72
+ elif name == "searxng" and self.searxng_url:
73
+ configured.append(name)
74
+ return tuple(configured)
75
+
76
+ def llm_providers(self) -> tuple[str, ...]:
77
+ providers: list[str] = []
78
+ if self.anthropic_api_key:
79
+ providers.append("anthropic")
80
+ if self.openai_api_key or self.openai_base_url != DEFAULT_OPENAI_BASE_URL:
81
+ providers.append("openai-compatible")
82
+ return tuple(providers)
indagar/models.py ADDED
@@ -0,0 +1,90 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import asdict, dataclass, field
4
+ from datetime import UTC, datetime
5
+ from urllib.parse import urlparse
6
+
7
+
8
+ def utc_now() -> str:
9
+ return datetime.now(UTC).isoformat()
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class Source:
14
+ title: str
15
+ url: str
16
+ snippet: str = ""
17
+ provider: str = ""
18
+ source_type: str = "web"
19
+ sub_question: str | None = None
20
+
21
+ @property
22
+ def domain(self) -> str:
23
+ try:
24
+ return urlparse(self.url).hostname or ""
25
+ except ValueError:
26
+ return ""
27
+
28
+ def to_dict(self) -> dict[str, object]:
29
+ return asdict(self)
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class SubQuestion:
34
+ question: str
35
+ rationale: str = ""
36
+
37
+ def to_dict(self) -> dict[str, object]:
38
+ return asdict(self)
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class ResearchPlan:
43
+ question: str
44
+ sub_questions: tuple[SubQuestion, ...]
45
+
46
+ def to_dict(self) -> dict[str, object]:
47
+ return {
48
+ "question": self.question,
49
+ "sub_questions": [item.to_dict() for item in self.sub_questions],
50
+ }
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class SearchAttempt:
55
+ provider: str
56
+ status: str
57
+ count: int
58
+ latency_ms: int
59
+ error: str | None = None
60
+
61
+ def to_dict(self) -> dict[str, object]:
62
+ return asdict(self)
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class SearchOutcome:
67
+ sources: tuple[Source, ...]
68
+ attempts: tuple[SearchAttempt, ...]
69
+
70
+
71
+ @dataclass
72
+ class Run:
73
+ id: str
74
+ question: str
75
+ plan: ResearchPlan
76
+ sources: tuple[Source, ...]
77
+ attempts: tuple[SearchAttempt, ...]
78
+ report: str
79
+ created_at: str = field(default_factory=utc_now)
80
+
81
+ def to_dict(self) -> dict[str, object]:
82
+ return {
83
+ "id": self.id,
84
+ "question": self.question,
85
+ "created_at": self.created_at,
86
+ "plan": self.plan.to_dict(),
87
+ "sources": [source.to_dict() for source in self.sources],
88
+ "attempts": [attempt.to_dict() for attempt in self.attempts],
89
+ "report": self.report,
90
+ }
@@ -0,0 +1,4 @@
1
+ from .loop import run_research
2
+ from .planner import plan_question
3
+
4
+ __all__ = ["plan_question", "run_research"]
@@ -0,0 +1,147 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from dataclasses import replace
5
+ from datetime import UTC, datetime
6
+
7
+ from ..config import Settings
8
+ from ..models import ResearchPlan, Run, SearchAttempt, SearchOutcome, Source, utc_now
9
+ from ..search.chain import SearchChain, build_chain, normalize_url
10
+ from .planner import plan_question
11
+
12
+
13
+ def _run_id(question: str) -> str:
14
+ stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
15
+ slug = re.sub(r"[^a-z0-9]+", "-", question.lower()).strip("-")[:48]
16
+ return f"{stamp}-{slug or 'research'}"
17
+
18
+
19
+ def _escape(value: str) -> str:
20
+ return value.replace("|", "\\|").replace("\n", " ").strip()
21
+
22
+
23
+ def render_dossier(
24
+ plan: ResearchPlan,
25
+ sources: tuple[Source, ...],
26
+ attempts: tuple[SearchAttempt, ...],
27
+ ) -> str:
28
+ lines = [
29
+ f"# Research dossier: {plan.question}",
30
+ "",
31
+ f"Generated: {utc_now()}",
32
+ "",
33
+ "> Phase 0: sources were discovered and deduplicated. LLM planning, claim "
34
+ "extraction, gap checks and synthesis are the next phase.",
35
+ "",
36
+ "## Plan",
37
+ "",
38
+ ]
39
+
40
+ for index, sub_question in enumerate(plan.sub_questions, start=1):
41
+ lines.append(f"{index}. **{sub_question.question}** — {sub_question.rationale}")
42
+
43
+ lines.extend(["", "## Sources", ""])
44
+ if not sources:
45
+ lines.append("_No sources were returned._")
46
+ for index, source in enumerate(sources, start=1):
47
+ lines.append(f"{index}. **{source.title}** — {source.url}")
48
+ meta = [f"provider: {source.provider or 'unknown'}", f"domain: {source.domain}"]
49
+ if source.sub_question:
50
+ meta.append(f"sub-question: {source.sub_question}")
51
+ lines.append(f" {' · '.join(meta)}")
52
+ if source.snippet:
53
+ lines.append(f" {source.snippet}")
54
+ lines.append("")
55
+
56
+ lines.extend(
57
+ [
58
+ "## Search attempts",
59
+ "",
60
+ "| Provider | Status | Count | Latency |",
61
+ "|---|---|---|---|",
62
+ ]
63
+ )
64
+ for attempt in attempts:
65
+ detail = _escape(attempt.error) if attempt.error else ""
66
+ status = attempt.status if not detail else f"{attempt.status}: {detail}"
67
+ lines.append(
68
+ f"| {attempt.provider} | {status} | {attempt.count} | {attempt.latency_ms}ms |"
69
+ )
70
+
71
+ lines.extend(
72
+ [
73
+ "",
74
+ "## Next",
75
+ "",
76
+ "- Add an LLM planner to refine sub-questions and search terms.",
77
+ "- Read sources and extract claims with source spans.",
78
+ "- Add a gap check and one or two follow-up rounds.",
79
+ "- Synthesize a report with inline citations and a citation ledger.",
80
+ ]
81
+ )
82
+ return "\n".join(lines).rstrip() + "\n"
83
+
84
+
85
+ async def run_research(
86
+ question: str,
87
+ settings: Settings,
88
+ chain: SearchChain | None = None,
89
+ plan: ResearchPlan | None = None,
90
+ max_sources: int | None = None,
91
+ ) -> Run:
92
+ plan = plan or plan_question(question, max_sub_questions=4)
93
+ chain = chain or build_chain(settings)
94
+ if not chain.providers:
95
+ raise RuntimeError(
96
+ "No search provider configured. Set FIRECRAWL_URL, BRAVE_API_KEY, "
97
+ "SERPER_API_KEY, or SEARXNG_URL."
98
+ )
99
+
100
+ limit = max(1, max_sources or settings.max_sources)
101
+ buckets: dict[str, list[Source]] = {
102
+ sub_question.question: [] for sub_question in plan.sub_questions
103
+ }
104
+ seen: set[str] = set()
105
+ attempts: list[SearchAttempt] = []
106
+
107
+ for sub_question in plan.sub_questions:
108
+ outcome: SearchOutcome = await chain.search(
109
+ sub_question.question, limit=settings.search_limit
110
+ )
111
+ attempts.extend(outcome.attempts)
112
+ for source in outcome.sources:
113
+ key = normalize_url(source.url)
114
+ if key in seen:
115
+ continue
116
+ seen.add(key)
117
+ buckets[sub_question.question].append(
118
+ replace(source, sub_question=sub_question.question)
119
+ )
120
+
121
+ selected: list[Source] = []
122
+ while len(selected) < limit:
123
+ added = False
124
+ for sub_question in plan.sub_questions:
125
+ bucket = buckets[sub_question.question]
126
+ if not bucket:
127
+ continue
128
+ selected.append(bucket.pop(0))
129
+ added = True
130
+ if len(selected) >= limit:
131
+ break
132
+ if not added:
133
+ break
134
+
135
+ selected = tuple(selected)
136
+ report = render_dossier(plan, selected, tuple(attempts))
137
+ return Run(
138
+ id=_run_id(plan.question),
139
+ question=plan.question,
140
+ plan=plan,
141
+ sources=selected,
142
+ attempts=tuple(attempts),
143
+ report=report,
144
+ )
145
+
146
+
147
+ __all__ = ["render_dossier", "run_research"]
@@ -0,0 +1,25 @@
1
+ from __future__ import annotations
2
+
3
+ from ..models import ResearchPlan, SubQuestion
4
+
5
+ FACETS: tuple[tuple[str, str], ...] = (
6
+ ("overview and definitions", "Establish shared terms and background."),
7
+ ("evidence and data", "Find quantitative or empirical evidence."),
8
+ ("critiques and limitations", "Look for counterarguments, failures, and known limits."),
9
+ ("recent developments", "Check what changed recently."),
10
+ )
11
+
12
+
13
+ def plan_question(question: str, max_sub_questions: int = 4) -> ResearchPlan:
14
+ """Deterministic Phase 0 plan: the question plus focused facets."""
15
+ normalized = " ".join(question.split())
16
+ if not normalized:
17
+ raise ValueError("question is required")
18
+ count = min(max(1, max_sub_questions), len(FACETS) + 1)
19
+ sub_questions = [SubQuestion(normalized, "Answer the question directly.")]
20
+ for suffix, rationale in FACETS[: count - 1]:
21
+ sub_questions.append(SubQuestion(f"{normalized} {suffix}", rationale))
22
+ return ResearchPlan(normalized, tuple(sub_questions))
23
+
24
+
25
+ __all__ = ["FACETS", "plan_question"]
@@ -0,0 +1,4 @@
1
+ from .base import SearchError, SearchProvider
2
+ from .chain import SearchChain, build_chain, normalize_url
3
+
4
+ __all__ = ["SearchChain", "SearchError", "SearchProvider", "build_chain", "normalize_url"]
indagar/search/base.py ADDED
@@ -0,0 +1,22 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Protocol
4
+
5
+ from ..models import Source
6
+
7
+
8
+ class SearchError(Exception):
9
+ """A provider failed. The chain records it and moves on."""
10
+
11
+ def __init__(self, provider: str, message: str, status: int | None = None):
12
+ super().__init__(message)
13
+ self.provider = provider
14
+ self.status = status
15
+
16
+
17
+ class SearchProvider(Protocol):
18
+ name: str
19
+
20
+ async def search(self, query: str, limit: int) -> list[Source]:
21
+ """Return up to limit results. Raise SearchError on failure."""
22
+ ...
@@ -0,0 +1,164 @@
1
+ from __future__ import annotations
2
+
3
+ import time
4
+ from collections import OrderedDict
5
+ from dataclasses import replace
6
+ from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
7
+
8
+ from ..config import Settings
9
+ from ..models import SearchAttempt, SearchOutcome, Source
10
+ from .base import SearchProvider
11
+ from .providers import create_provider
12
+
13
+ TRACKING_PARAMS = {
14
+ "utm_source",
15
+ "utm_medium",
16
+ "utm_campaign",
17
+ "utm_term",
18
+ "utm_content",
19
+ "fbclid",
20
+ "gclid",
21
+ "mc_cid",
22
+ "mc_eid",
23
+ }
24
+
25
+
26
+ def normalize_url(raw: str) -> str:
27
+ parsed = urlparse(raw.strip())
28
+ hostname = (parsed.hostname or "").lower()
29
+ if hostname.startswith("www."):
30
+ hostname = hostname[4:]
31
+ query = [
32
+ (key, value)
33
+ for key, value in parse_qsl(parsed.query, keep_blank_values=True)
34
+ if key.lower() not in TRACKING_PARAMS
35
+ ]
36
+ path = parsed.path.rstrip("/") or "/"
37
+ return urlunparse(
38
+ (
39
+ parsed.scheme.lower(),
40
+ hostname,
41
+ path,
42
+ "",
43
+ urlencode(query),
44
+ "",
45
+ )
46
+ )
47
+
48
+
49
+ def _is_http_url(value: str) -> bool:
50
+ try:
51
+ parsed = urlparse(value)
52
+ except ValueError:
53
+ return False
54
+ return parsed.scheme in {"http", "https"} and bool(parsed.hostname)
55
+
56
+
57
+ class SearchChain:
58
+ """Ordered providers with over-fetch, dedupe, and a short cache."""
59
+
60
+ def __init__(
61
+ self,
62
+ providers: list[SearchProvider],
63
+ ttl_seconds: int = 600,
64
+ max_entries: int = 100,
65
+ ):
66
+ self.providers = providers
67
+ self.ttl_seconds = ttl_seconds
68
+ self.max_entries = max_entries
69
+ self._cache: OrderedDict[str, tuple[float, SearchOutcome]] = OrderedDict()
70
+
71
+ @property
72
+ def provider_names(self) -> tuple[str, ...]:
73
+ return tuple(provider.name for provider in self.providers)
74
+
75
+ async def search(self, query: str, limit: int = 8) -> SearchOutcome:
76
+ key = self._cache_key(query, limit)
77
+ cached = self._cache_get(key)
78
+ if cached is not None:
79
+ return cached
80
+
81
+ attempts: list[SearchAttempt] = []
82
+ seen: set[str] = set()
83
+
84
+ for provider in self.providers:
85
+ started = time.perf_counter()
86
+ try:
87
+ raw = await provider.search(query, limit * 2)
88
+ results: list[Source] = []
89
+ for source in raw:
90
+ if not _is_http_url(source.url):
91
+ continue
92
+ normalized = normalize_url(source.url)
93
+ if normalized in seen:
94
+ continue
95
+ seen.add(normalized)
96
+ results.append(
97
+ replace(
98
+ source,
99
+ provider=source.provider or provider.name,
100
+ )
101
+ )
102
+ if len(results) >= limit:
103
+ break
104
+ latency = int((time.perf_counter() - started) * 1000)
105
+ attempts.append(
106
+ SearchAttempt(
107
+ provider=provider.name,
108
+ status="ok" if results else "empty",
109
+ count=len(results),
110
+ latency_ms=latency,
111
+ )
112
+ )
113
+ if results:
114
+ outcome = SearchOutcome(tuple(results), tuple(attempts))
115
+ self._cache_set(key, outcome)
116
+ return outcome
117
+ except Exception as exc: # noqa: BLE001 - one provider must not stop the chain
118
+ latency = int((time.perf_counter() - started) * 1000)
119
+ attempts.append(
120
+ SearchAttempt(
121
+ provider=provider.name,
122
+ status="error",
123
+ count=0,
124
+ latency_ms=latency,
125
+ error=str(exc),
126
+ )
127
+ )
128
+
129
+ return SearchOutcome((), tuple(attempts))
130
+
131
+ def _cache_key(self, query: str, limit: int) -> str:
132
+ normalized_query = " ".join(query.lower().split())
133
+ providers = ",".join(self.provider_names)
134
+ return f"{normalized_query}|{limit}|{providers}"
135
+
136
+ def _cache_get(self, key: str) -> SearchOutcome | None:
137
+ entry = self._cache.get(key)
138
+ if entry is None:
139
+ return None
140
+ stored_at, outcome = entry
141
+ if time.monotonic() - stored_at >= self.ttl_seconds:
142
+ self._cache.pop(key, None)
143
+ return None
144
+ self._cache.move_to_end(key)
145
+ return outcome
146
+
147
+ def _cache_set(self, key: str, outcome: SearchOutcome) -> None:
148
+ if not outcome.sources:
149
+ return
150
+ if len(self._cache) >= self.max_entries:
151
+ self._cache.popitem(last=False)
152
+ self._cache[key] = (time.monotonic(), outcome)
153
+
154
+
155
+ def build_chain(settings: Settings) -> SearchChain:
156
+ providers = [
157
+ provider
158
+ for name in settings.configured_search_providers()
159
+ if (provider := create_provider(name, settings)) is not None
160
+ ]
161
+ return SearchChain(providers)
162
+
163
+
164
+ __all__ = ["SearchChain", "build_chain", "normalize_url"]
@@ -0,0 +1,216 @@
1
+ from __future__ import annotations
2
+
3
+ from urllib.parse import urlparse
4
+
5
+ import httpx
6
+
7
+ from ..config import Settings
8
+ from ..models import Source
9
+ from .base import SearchError
10
+
11
+ USER_AGENT = "Indagar/0.1 (+https://github.com/michael-borck/indagar)"
12
+
13
+
14
+ def _is_http_url(value: str) -> bool:
15
+ try:
16
+ parsed = urlparse(value)
17
+ except ValueError:
18
+ return False
19
+ return parsed.scheme in {"http", "https"} and bool(parsed.hostname)
20
+
21
+
22
+ def _source(
23
+ title: str | None,
24
+ url: str | None,
25
+ snippet: str | None,
26
+ provider: str,
27
+ ) -> Source | None:
28
+ if not title or not url or not _is_http_url(url):
29
+ return None
30
+ return Source(
31
+ title=" ".join(title.split()),
32
+ url=url,
33
+ snippet=" ".join((snippet or "").split())[:600],
34
+ provider=provider,
35
+ )
36
+
37
+
38
+ def _status_error(provider: str, response: httpx.Response) -> SearchError:
39
+ return SearchError(
40
+ provider,
41
+ f"{provider} returned HTTP {response.status_code}",
42
+ status=response.status_code,
43
+ )
44
+
45
+
46
+ class FirecrawlProvider:
47
+ name = "firecrawl"
48
+
49
+ def __init__(self, base_url: str, api_key: str | None = None):
50
+ self.base_url = base_url.rstrip("/")
51
+ self.api_key = api_key
52
+
53
+ async def search(self, query: str, limit: int) -> list[Source]:
54
+ headers = {"content-type": "application/json", "user-agent": USER_AGENT}
55
+ if self.api_key:
56
+ headers["authorization"] = f"Bearer {self.api_key}"
57
+ try:
58
+ async with httpx.AsyncClient(timeout=30.0) as client:
59
+ response = await client.post(
60
+ f"{self.base_url}/v2/search",
61
+ json={"query": query, "limit": max(1, limit), "sources": ["web"]},
62
+ headers=headers,
63
+ )
64
+ except httpx.HTTPError as exc:
65
+ raise SearchError(self.name, f"firecrawl request failed: {exc}") from exc
66
+ if response.status_code >= 400:
67
+ raise _status_error(self.name, response)
68
+ data = response.json()
69
+ web = ((data.get("data") or {}).get("web")) or []
70
+ results: list[Source] = []
71
+ for item in web:
72
+ source = _source(
73
+ item.get("title"),
74
+ item.get("url"),
75
+ item.get("description"),
76
+ self.name,
77
+ )
78
+ if source:
79
+ results.append(source)
80
+ return results
81
+
82
+
83
+ class BraveProvider:
84
+ name = "brave"
85
+
86
+ def __init__(self, api_key: str):
87
+ self.api_key = api_key
88
+
89
+ async def search(self, query: str, limit: int) -> list[Source]:
90
+ params = {
91
+ "q": query,
92
+ "count": min(max(limit, 1), 20),
93
+ "search_lang": "en",
94
+ "country": "US",
95
+ "safesearch": "moderate",
96
+ }
97
+ headers = {
98
+ "X-Subscription-Token": self.api_key,
99
+ "Accept": "application/json",
100
+ "User-Agent": USER_AGENT,
101
+ }
102
+ try:
103
+ async with httpx.AsyncClient(timeout=20.0) as client:
104
+ response = await client.get(
105
+ "https://api.search.brave.com/res/v1/web/search",
106
+ params=params,
107
+ headers=headers,
108
+ )
109
+ except httpx.HTTPError as exc:
110
+ raise SearchError(self.name, f"brave request failed: {exc}") from exc
111
+ if response.status_code >= 400:
112
+ raise _status_error(self.name, response)
113
+ data = response.json()
114
+ results: list[Source] = []
115
+ for item in (data.get("web") or {}).get("results") or []:
116
+ source = _source(
117
+ item.get("title"),
118
+ item.get("url"),
119
+ item.get("description"),
120
+ self.name,
121
+ )
122
+ if source:
123
+ results.append(source)
124
+ return results
125
+
126
+
127
+ class SerperProvider:
128
+ name = "serper"
129
+
130
+ def __init__(self, api_key: str):
131
+ self.api_key = api_key
132
+
133
+ async def search(self, query: str, limit: int) -> list[Source]:
134
+ headers = {
135
+ "X-API-KEY": self.api_key,
136
+ "content-type": "application/json",
137
+ "user-agent": USER_AGENT,
138
+ }
139
+ try:
140
+ async with httpx.AsyncClient(timeout=20.0) as client:
141
+ response = await client.post(
142
+ "https://google.serper.dev/search",
143
+ json={"q": query, "num": min(max(limit, 1), 100)},
144
+ headers=headers,
145
+ )
146
+ except httpx.HTTPError as exc:
147
+ raise SearchError(self.name, f"serper request failed: {exc}") from exc
148
+ if response.status_code >= 400:
149
+ raise _status_error(self.name, response)
150
+ data = response.json()
151
+ results: list[Source] = []
152
+ for item in data.get("organic") or []:
153
+ source = _source(
154
+ item.get("title"),
155
+ item.get("link"),
156
+ item.get("snippet"),
157
+ self.name,
158
+ )
159
+ if source:
160
+ results.append(source)
161
+ return results
162
+
163
+
164
+ class SearxngProvider:
165
+ name = "searxng"
166
+
167
+ def __init__(self, base_url: str):
168
+ self.base_url = base_url.rstrip("/")
169
+
170
+ async def search(self, query: str, limit: int) -> list[Source]:
171
+ params = {"q": query, "format": "json", "safesearch": "1"}
172
+ headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
173
+ try:
174
+ async with httpx.AsyncClient(timeout=20.0) as client:
175
+ response = await client.get(
176
+ f"{self.base_url}/search",
177
+ params=params,
178
+ headers=headers,
179
+ )
180
+ except httpx.HTTPError as exc:
181
+ raise SearchError(self.name, f"searxng request failed: {exc}") from exc
182
+ if response.status_code >= 400:
183
+ raise _status_error(self.name, response)
184
+ data = response.json()
185
+ results: list[Source] = []
186
+ for item in data.get("results") or []:
187
+ source = _source(
188
+ item.get("title"),
189
+ item.get("url"),
190
+ item.get("content"),
191
+ self.name,
192
+ )
193
+ if source:
194
+ results.append(source)
195
+ return results
196
+
197
+
198
+ def create_provider(name: str, settings: Settings):
199
+ if name == "firecrawl" and settings.firecrawl_url:
200
+ return FirecrawlProvider(settings.firecrawl_url, settings.firecrawl_api_key)
201
+ if name == "brave" and settings.brave_api_key:
202
+ return BraveProvider(settings.brave_api_key)
203
+ if name == "serper" and settings.serper_api_key:
204
+ return SerperProvider(settings.serper_api_key)
205
+ if name == "searxng" and settings.searxng_url:
206
+ return SearxngProvider(settings.searxng_url)
207
+ return None
208
+
209
+
210
+ __all__ = [
211
+ "BraveProvider",
212
+ "FirecrawlProvider",
213
+ "SearxngProvider",
214
+ "SerperProvider",
215
+ "create_provider",
216
+ ]
indagar/store.py ADDED
@@ -0,0 +1,28 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ from .models import Run
7
+
8
+
9
+ def save_run(
10
+ run: Run,
11
+ out: Path | None = None,
12
+ runs_dir: Path = Path("runs"),
13
+ ) -> tuple[Path, Path]:
14
+ """Write the run record and Markdown dossier. Returns (json, markdown)."""
15
+ runs_dir.mkdir(parents=True, exist_ok=True)
16
+ json_path = runs_dir / f"{run.id}.json"
17
+ markdown_path = out or runs_dir / f"{run.id}.md"
18
+ markdown_path.parent.mkdir(parents=True, exist_ok=True)
19
+
20
+ json_path.write_text(
21
+ json.dumps(run.to_dict(), indent=2, ensure_ascii=False) + "\n",
22
+ encoding="utf-8",
23
+ )
24
+ markdown_path.write_text(run.report, encoding="utf-8")
25
+ return json_path, markdown_path
26
+
27
+
28
+ __all__ = ["save_run"]
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.5
2
+ Name: indagar
3
+ Version: 0.1.0
4
+ Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
5
+ Author: Michael Borck
6
+ License: MIT
7
+ License-File: LICENSE
8
+ Keywords: citations,cli,deep-research,research,web-search
9
+ Classifier: Environment :: Console
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Requires-Python: >=3.11
16
+ Requires-Dist: httpx>=0.27
17
+ Requires-Dist: python-dotenv>=1.0
18
+ Requires-Dist: rich>=13.7
19
+ Requires-Dist: typer>=0.12
20
+ Provides-Extra: dev
21
+ Requires-Dist: pytest>=8; extra == 'dev'
22
+ Requires-Dist: ruff>=0.6; extra == 'dev'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # Indagar
26
+
27
+ **Deep research from the command line.**
28
+
29
+ Indagar asks a question, plans sub-questions, searches multiple sources, reads
30
+ what it finds, and produces a cited research dossier. The name is Spanish for
31
+ *to investigate, inquire, or probe*.
32
+
33
+ This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
34
+ chain, a deterministic research plan, and a source dossier. LLM planning,
35
+ extraction, and synthesis are the next phase.
36
+
37
+ ## Quick start
38
+
39
+ ```bash
40
+ git clone https://github.com/michael-borck/indagar.git
41
+ cd indagar
42
+ uv venv
43
+ source .venv/bin/activate
44
+ uv pip install -e '.[dev]'
45
+
46
+ indagar doctor
47
+ indagar search "retrieval augmented generation" --limit 5
48
+ indagar research "Does retrieval augmented generation reduce hallucination?" \
49
+ --max-sources 10
50
+ ```
51
+
52
+ If the checkout is on an exFAT drive, create the virtual environment on a
53
+ local disk instead: `uv venv ~/.venvs/indagar`.
54
+
55
+ The `research` command writes a Markdown dossier and a JSON run record under
56
+ `runs/`. Every source keeps its URL, provider, and sub-question.
57
+
58
+ ## Configuration
59
+
60
+ Copy `.env.example` to `.env` and set the providers you have.
61
+
62
+ | Variable | Purpose |
63
+ |---|---|
64
+ | `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
65
+ | `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
66
+ | `BRAVE_API_KEY` | Brave Search API key |
67
+ | `SERPER_API_KEY` | Serper API key |
68
+ | `SEARXNG_URL` | Self-hosted SearXNG base URL |
69
+ | `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
70
+ | `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
71
+ | `OPENAI_API_KEY` | OpenAI-compatible key |
72
+ | `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
73
+ | `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
74
+ | `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
75
+ | `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
76
+
77
+ `indagar doctor` shows what is configured.
78
+
79
+ ## Status
80
+
81
+ - [x] CLI with `doctor`, `search`, and `research`
82
+ - [x] Firecrawl, Brave, Serper, and SearXNG adapters
83
+ - [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
84
+ - [x] Deterministic research plan and cited source dossier
85
+ - [ ] LLM planner, claim extraction, and synthesis
86
+ - [ ] Web UI and desktop shell
87
+ - [ ] Academic, podcast, and video sources
88
+
89
+ See [SPEC.md](SPEC.md) for the design and roadmap.
90
+
91
+ ## Development
92
+
93
+ ```bash
94
+ uv pip install -e '.[dev]'
95
+ ruff check .
96
+ ruff format --check .
97
+ pytest -q
98
+ ```
99
+
100
+ ## Licence
101
+
102
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,18 @@
1
+ indagar/__init__.py,sha256=Io3bD5DCVPKQK-d4CTwvXxYw8ggfnctl6DtA88y3siw,78
2
+ indagar/__main__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
3
+ indagar/cli.py,sha256=TpFzkp3z5FMZMOc472HwGXb9UwlUH6uLSKTmqi1M0uQ,4698
4
+ indagar/config.py,sha256=zXmSLxPav5PUGlnI3KeAhynUWrMN0v7YWadi1rcysng,3074
5
+ indagar/models.py,sha256=Se5XYNEQq64zqoTzg4426aVVqgs74HnVg2iAi0AP-WA,2041
6
+ indagar/store.py,sha256=z3E4EKnAbPavaVQdge_-OeeDH5JYOy3MlkY8QeM0kho,751
7
+ indagar/research/__init__.py,sha256=nCuhySlDmopoANC1OWlCC4QJmR7Mrw2Z5Q17cX5eLRc,111
8
+ indagar/research/loop.py,sha256=UHxxNVXBEPxKZhiRHMAzbaDoAjHk-gsb18qBPamoj9k,4727
9
+ indagar/research/planner.py,sha256=SL5uaihLABRvKGFZAGjWq_cxSUnFBxUThC_8uUC_orA,1061
10
+ indagar/search/__init__.py,sha256=LS0L_Lx4WYgqO_CXJNeRCFkecwKCRmUZgRhkegASXTg,197
11
+ indagar/search/base.py,sha256=oKv-bfHfdP1GJarMmRtErgVshu4IfGpDkTMB-igYUrk,567
12
+ indagar/search/chain.py,sha256=qly3OG4bf9uVsBYFxqsCGp-vDK1zhnf76gZKFQF4c08,5211
13
+ indagar/search/providers.py,sha256=N7zxlACL9UVk4eoI8O-VbJHeLmLfnGvN_IAy1a0ZFB0,6927
14
+ indagar-0.1.0.dist-info/METADATA,sha256=ejM7FGJGPwAXCdg0um4cvTK2p13QXJUaYTwoFB4OIdI,3364
15
+ indagar-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
16
+ indagar-0.1.0.dist-info/entry_points.txt,sha256=GHooEipv6OwNv2VLTtsprOFiIFl8sU8TQpwvTQlqlac,45
17
+ indagar-0.1.0.dist-info/licenses/LICENSE,sha256=tP28nznO05HKBQN8uZ9NSvmcpzCWMj4P8e6F7IDj24Q,1070
18
+ indagar-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ indagar = indagar.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Michael Borck
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.