indagar 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indagar/__init__.py +3 -0
- indagar/__main__.py +4 -0
- indagar/cli.py +140 -0
- indagar/config.py +82 -0
- indagar/models.py +90 -0
- indagar/research/__init__.py +4 -0
- indagar/research/loop.py +147 -0
- indagar/research/planner.py +25 -0
- indagar/search/__init__.py +4 -0
- indagar/search/base.py +22 -0
- indagar/search/chain.py +164 -0
- indagar/search/providers.py +216 -0
- indagar/store.py +28 -0
- indagar-0.1.0.dist-info/METADATA +102 -0
- indagar-0.1.0.dist-info/RECORD +18 -0
- indagar-0.1.0.dist-info/WHEEL +4 -0
- indagar-0.1.0.dist-info/entry_points.txt +2 -0
- indagar-0.1.0.dist-info/licenses/LICENSE +21 -0
indagar/__init__.py
ADDED
indagar/__main__.py
ADDED
indagar/cli.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from dataclasses import replace
|
|
5
|
+
from importlib.metadata import PackageNotFoundError
|
|
6
|
+
from importlib.metadata import version as package_version
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
|
|
13
|
+
from .config import Settings
|
|
14
|
+
from .research.loop import run_research
|
|
15
|
+
from .search.chain import build_chain
|
|
16
|
+
from .store import save_run
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(
|
|
19
|
+
add_completion=False,
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
help="Indagar — deep research from the command line.",
|
|
22
|
+
)
|
|
23
|
+
console = Console()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _version() -> str:
|
|
27
|
+
try:
|
|
28
|
+
return package_version("indagar")
|
|
29
|
+
except PackageNotFoundError:
|
|
30
|
+
return "0.1.0"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _settings_or_exit() -> Settings:
|
|
34
|
+
return Settings.from_env()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@app.command()
|
|
38
|
+
def version() -> None:
|
|
39
|
+
"""Print the installed version."""
|
|
40
|
+
console.print(_version())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@app.command()
|
|
44
|
+
def doctor() -> None:
|
|
45
|
+
"""Show configured search and LLM providers and the current budgets."""
|
|
46
|
+
settings = _settings_or_exit()
|
|
47
|
+
configured = settings.configured_search_providers()
|
|
48
|
+
|
|
49
|
+
console.print("[bold]Indagar doctor[/bold]")
|
|
50
|
+
console.print(f"Search order: {', '.join(settings.search_order)}")
|
|
51
|
+
console.print(f"Configured search: {', '.join(configured) or 'none'}")
|
|
52
|
+
console.print(f"LLM providers: {', '.join(settings.llm_providers()) or 'none'}")
|
|
53
|
+
console.print(
|
|
54
|
+
f"Budgets: sources={settings.max_sources}, "
|
|
55
|
+
f"seconds={settings.max_seconds}, search_limit={settings.search_limit}"
|
|
56
|
+
)
|
|
57
|
+
if not configured:
|
|
58
|
+
console.print(
|
|
59
|
+
"[yellow]No search provider configured. Set FIRECRAWL_URL, "
|
|
60
|
+
"BRAVE_API_KEY, SERPER_API_KEY, or SEARXNG_URL.[/yellow]"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command()
|
|
65
|
+
def search(
|
|
66
|
+
query: str = typer.Argument(..., help="Search query"),
|
|
67
|
+
limit: int = typer.Option(8, "--limit", "-n", min=1, max=20),
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Run the configured search chain and print the results."""
|
|
70
|
+
settings = _settings_or_exit()
|
|
71
|
+
chain = build_chain(settings)
|
|
72
|
+
if not chain.providers:
|
|
73
|
+
console.print("[red]No search provider configured.[/red]")
|
|
74
|
+
raise typer.Exit(1)
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
outcome = asyncio.run(chain.search(query, limit=limit))
|
|
78
|
+
except Exception as exc: # noqa: BLE001 - CLI boundary
|
|
79
|
+
console.print(f"[red]Search failed: {exc}[/red]")
|
|
80
|
+
raise typer.Exit(1) from exc
|
|
81
|
+
|
|
82
|
+
table = Table(title=f"Search: {query}", show_lines=False)
|
|
83
|
+
table.add_column("#", justify="right", style="dim")
|
|
84
|
+
table.add_column("Title", overflow="fold")
|
|
85
|
+
table.add_column("Domain", style="cyan")
|
|
86
|
+
table.add_column("Provider", style="green")
|
|
87
|
+
for index, source in enumerate(outcome.sources, start=1):
|
|
88
|
+
table.add_row(str(index), source.title, source.domain, source.provider)
|
|
89
|
+
console.print(table)
|
|
90
|
+
|
|
91
|
+
for attempt in outcome.attempts:
|
|
92
|
+
detail = f" — {attempt.error}" if attempt.error else ""
|
|
93
|
+
console.print(
|
|
94
|
+
f"[dim]{attempt.provider}: {attempt.status} "
|
|
95
|
+
f"({attempt.count}, {attempt.latency_ms}ms){detail}[/dim]"
|
|
96
|
+
)
|
|
97
|
+
if not outcome.sources:
|
|
98
|
+
console.print("[yellow]No results.[/yellow]")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@app.command()
|
|
102
|
+
def research(
|
|
103
|
+
question: str = typer.Argument(..., help="Question or topic to research"),
|
|
104
|
+
max_sources: int = typer.Option(0, "--max-sources", min=0, help="0 uses INDAGAR_MAX_SOURCES"),
|
|
105
|
+
out: Path | None = typer.Option(None, "--out", "-o", help="Markdown output path"),
|
|
106
|
+
) -> None:
|
|
107
|
+
"""Plan sub-questions, search, and write a source dossier."""
|
|
108
|
+
settings = _settings_or_exit()
|
|
109
|
+
if max_sources:
|
|
110
|
+
settings = replace(settings, max_sources=max_sources)
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
run = asyncio.run(run_research(question, settings))
|
|
114
|
+
except (RuntimeError, ValueError) as exc:
|
|
115
|
+
console.print(f"[red]{exc}[/red]")
|
|
116
|
+
raise typer.Exit(1) from exc
|
|
117
|
+
|
|
118
|
+
console.print(f"[bold]Plan[/bold] ({len(run.plan.sub_questions)} sub-questions)")
|
|
119
|
+
for index, sub_question in enumerate(run.plan.sub_questions, start=1):
|
|
120
|
+
console.print(f" {index}. {sub_question.question}")
|
|
121
|
+
|
|
122
|
+
table = Table(title=f"Sources: {len(run.sources)}")
|
|
123
|
+
table.add_column("#", justify="right", style="dim")
|
|
124
|
+
table.add_column("Title", overflow="fold")
|
|
125
|
+
table.add_column("Domain", style="cyan")
|
|
126
|
+
table.add_column("Provider", style="green")
|
|
127
|
+
for index, source in enumerate(run.sources, start=1):
|
|
128
|
+
table.add_row(str(index), source.title, source.domain, source.provider)
|
|
129
|
+
console.print(table)
|
|
130
|
+
|
|
131
|
+
json_path, markdown_path = save_run(run, out=out)
|
|
132
|
+
console.print(f"Run record: {json_path}")
|
|
133
|
+
console.print(f"Dossier: {markdown_path}")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def main() -> None:
|
|
137
|
+
app()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
__all__ = ["app", "main"]
|
indagar/config.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from dotenv import load_dotenv
|
|
7
|
+
|
|
8
|
+
DEFAULT_SEARCH_ORDER = ("firecrawl", "brave", "serper", "searxng")
|
|
9
|
+
DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _env_int(name: str, default: int) -> int:
|
|
13
|
+
raw = os.getenv(name)
|
|
14
|
+
if raw is None:
|
|
15
|
+
return default
|
|
16
|
+
try:
|
|
17
|
+
return max(1, int(raw))
|
|
18
|
+
except ValueError:
|
|
19
|
+
return default
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _search_order() -> tuple[str, ...]:
|
|
23
|
+
raw = os.getenv("INDAGAR_SEARCH_ORDER", "")
|
|
24
|
+
if not raw.strip():
|
|
25
|
+
return DEFAULT_SEARCH_ORDER
|
|
26
|
+
names = tuple(part.strip().lower() for part in raw.split(",") if part.strip())
|
|
27
|
+
return names or DEFAULT_SEARCH_ORDER
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Settings:
|
|
32
|
+
firecrawl_url: str | None = None
|
|
33
|
+
firecrawl_api_key: str | None = None
|
|
34
|
+
brave_api_key: str | None = None
|
|
35
|
+
serper_api_key: str | None = None
|
|
36
|
+
searxng_url: str | None = None
|
|
37
|
+
search_order: tuple[str, ...] = DEFAULT_SEARCH_ORDER
|
|
38
|
+
anthropic_api_key: str | None = None
|
|
39
|
+
openai_api_key: str | None = None
|
|
40
|
+
openai_base_url: str = DEFAULT_OPENAI_BASE_URL
|
|
41
|
+
max_sources: int = 12
|
|
42
|
+
max_seconds: int = 300
|
|
43
|
+
search_limit: int = 8
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def from_env(cls) -> Settings:
|
|
47
|
+
load_dotenv(override=False)
|
|
48
|
+
return cls(
|
|
49
|
+
firecrawl_url=os.getenv("FIRECRAWL_URL", "").strip() or None,
|
|
50
|
+
firecrawl_api_key=os.getenv("FIRECRAWL_API_KEY", "").strip() or None,
|
|
51
|
+
brave_api_key=os.getenv("BRAVE_API_KEY", "").strip() or None,
|
|
52
|
+
serper_api_key=os.getenv("SERPER_API_KEY", "").strip() or None,
|
|
53
|
+
searxng_url=os.getenv("SEARXNG_URL", "").strip() or None,
|
|
54
|
+
search_order=_search_order(),
|
|
55
|
+
anthropic_api_key=os.getenv("ANTHROPIC_API_KEY", "").strip() or None,
|
|
56
|
+
openai_api_key=os.getenv("OPENAI_API_KEY", "").strip() or None,
|
|
57
|
+
openai_base_url=(os.getenv("OPENAI_BASE_URL", "").strip() or DEFAULT_OPENAI_BASE_URL),
|
|
58
|
+
max_sources=_env_int("INDAGAR_MAX_SOURCES", 12),
|
|
59
|
+
max_seconds=_env_int("INDAGAR_MAX_SECONDS", 300),
|
|
60
|
+
search_limit=_env_int("INDAGAR_SEARCH_LIMIT", 8),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def configured_search_providers(self) -> tuple[str, ...]:
|
|
64
|
+
configured: list[str] = []
|
|
65
|
+
for name in self.search_order:
|
|
66
|
+
if name == "firecrawl" and self.firecrawl_url:
|
|
67
|
+
configured.append(name)
|
|
68
|
+
elif name == "brave" and self.brave_api_key:
|
|
69
|
+
configured.append(name)
|
|
70
|
+
elif name == "serper" and self.serper_api_key:
|
|
71
|
+
configured.append(name)
|
|
72
|
+
elif name == "searxng" and self.searxng_url:
|
|
73
|
+
configured.append(name)
|
|
74
|
+
return tuple(configured)
|
|
75
|
+
|
|
76
|
+
def llm_providers(self) -> tuple[str, ...]:
|
|
77
|
+
providers: list[str] = []
|
|
78
|
+
if self.anthropic_api_key:
|
|
79
|
+
providers.append("anthropic")
|
|
80
|
+
if self.openai_api_key or self.openai_base_url != DEFAULT_OPENAI_BASE_URL:
|
|
81
|
+
providers.append("openai-compatible")
|
|
82
|
+
return tuple(providers)
|
indagar/models.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict, dataclass, field
|
|
4
|
+
from datetime import UTC, datetime
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def utc_now() -> str:
|
|
9
|
+
return datetime.now(UTC).isoformat()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Source:
|
|
14
|
+
title: str
|
|
15
|
+
url: str
|
|
16
|
+
snippet: str = ""
|
|
17
|
+
provider: str = ""
|
|
18
|
+
source_type: str = "web"
|
|
19
|
+
sub_question: str | None = None
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def domain(self) -> str:
|
|
23
|
+
try:
|
|
24
|
+
return urlparse(self.url).hostname or ""
|
|
25
|
+
except ValueError:
|
|
26
|
+
return ""
|
|
27
|
+
|
|
28
|
+
def to_dict(self) -> dict[str, object]:
|
|
29
|
+
return asdict(self)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class SubQuestion:
|
|
34
|
+
question: str
|
|
35
|
+
rationale: str = ""
|
|
36
|
+
|
|
37
|
+
def to_dict(self) -> dict[str, object]:
|
|
38
|
+
return asdict(self)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class ResearchPlan:
|
|
43
|
+
question: str
|
|
44
|
+
sub_questions: tuple[SubQuestion, ...]
|
|
45
|
+
|
|
46
|
+
def to_dict(self) -> dict[str, object]:
|
|
47
|
+
return {
|
|
48
|
+
"question": self.question,
|
|
49
|
+
"sub_questions": [item.to_dict() for item in self.sub_questions],
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class SearchAttempt:
|
|
55
|
+
provider: str
|
|
56
|
+
status: str
|
|
57
|
+
count: int
|
|
58
|
+
latency_ms: int
|
|
59
|
+
error: str | None = None
|
|
60
|
+
|
|
61
|
+
def to_dict(self) -> dict[str, object]:
|
|
62
|
+
return asdict(self)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class SearchOutcome:
|
|
67
|
+
sources: tuple[Source, ...]
|
|
68
|
+
attempts: tuple[SearchAttempt, ...]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class Run:
|
|
73
|
+
id: str
|
|
74
|
+
question: str
|
|
75
|
+
plan: ResearchPlan
|
|
76
|
+
sources: tuple[Source, ...]
|
|
77
|
+
attempts: tuple[SearchAttempt, ...]
|
|
78
|
+
report: str
|
|
79
|
+
created_at: str = field(default_factory=utc_now)
|
|
80
|
+
|
|
81
|
+
def to_dict(self) -> dict[str, object]:
|
|
82
|
+
return {
|
|
83
|
+
"id": self.id,
|
|
84
|
+
"question": self.question,
|
|
85
|
+
"created_at": self.created_at,
|
|
86
|
+
"plan": self.plan.to_dict(),
|
|
87
|
+
"sources": [source.to_dict() for source in self.sources],
|
|
88
|
+
"attempts": [attempt.to_dict() for attempt in self.attempts],
|
|
89
|
+
"report": self.report,
|
|
90
|
+
}
|
indagar/research/loop.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import replace
|
|
5
|
+
from datetime import UTC, datetime
|
|
6
|
+
|
|
7
|
+
from ..config import Settings
|
|
8
|
+
from ..models import ResearchPlan, Run, SearchAttempt, SearchOutcome, Source, utc_now
|
|
9
|
+
from ..search.chain import SearchChain, build_chain, normalize_url
|
|
10
|
+
from .planner import plan_question
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _run_id(question: str) -> str:
|
|
14
|
+
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
15
|
+
slug = re.sub(r"[^a-z0-9]+", "-", question.lower()).strip("-")[:48]
|
|
16
|
+
return f"{stamp}-{slug or 'research'}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _escape(value: str) -> str:
|
|
20
|
+
return value.replace("|", "\\|").replace("\n", " ").strip()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def render_dossier(
|
|
24
|
+
plan: ResearchPlan,
|
|
25
|
+
sources: tuple[Source, ...],
|
|
26
|
+
attempts: tuple[SearchAttempt, ...],
|
|
27
|
+
) -> str:
|
|
28
|
+
lines = [
|
|
29
|
+
f"# Research dossier: {plan.question}",
|
|
30
|
+
"",
|
|
31
|
+
f"Generated: {utc_now()}",
|
|
32
|
+
"",
|
|
33
|
+
"> Phase 0: sources were discovered and deduplicated. LLM planning, claim "
|
|
34
|
+
"extraction, gap checks and synthesis are the next phase.",
|
|
35
|
+
"",
|
|
36
|
+
"## Plan",
|
|
37
|
+
"",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
for index, sub_question in enumerate(plan.sub_questions, start=1):
|
|
41
|
+
lines.append(f"{index}. **{sub_question.question}** — {sub_question.rationale}")
|
|
42
|
+
|
|
43
|
+
lines.extend(["", "## Sources", ""])
|
|
44
|
+
if not sources:
|
|
45
|
+
lines.append("_No sources were returned._")
|
|
46
|
+
for index, source in enumerate(sources, start=1):
|
|
47
|
+
lines.append(f"{index}. **{source.title}** — {source.url}")
|
|
48
|
+
meta = [f"provider: {source.provider or 'unknown'}", f"domain: {source.domain}"]
|
|
49
|
+
if source.sub_question:
|
|
50
|
+
meta.append(f"sub-question: {source.sub_question}")
|
|
51
|
+
lines.append(f" {' · '.join(meta)}")
|
|
52
|
+
if source.snippet:
|
|
53
|
+
lines.append(f" {source.snippet}")
|
|
54
|
+
lines.append("")
|
|
55
|
+
|
|
56
|
+
lines.extend(
|
|
57
|
+
[
|
|
58
|
+
"## Search attempts",
|
|
59
|
+
"",
|
|
60
|
+
"| Provider | Status | Count | Latency |",
|
|
61
|
+
"|---|---|---|---|",
|
|
62
|
+
]
|
|
63
|
+
)
|
|
64
|
+
for attempt in attempts:
|
|
65
|
+
detail = _escape(attempt.error) if attempt.error else ""
|
|
66
|
+
status = attempt.status if not detail else f"{attempt.status}: {detail}"
|
|
67
|
+
lines.append(
|
|
68
|
+
f"| {attempt.provider} | {status} | {attempt.count} | {attempt.latency_ms}ms |"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
lines.extend(
|
|
72
|
+
[
|
|
73
|
+
"",
|
|
74
|
+
"## Next",
|
|
75
|
+
"",
|
|
76
|
+
"- Add an LLM planner to refine sub-questions and search terms.",
|
|
77
|
+
"- Read sources and extract claims with source spans.",
|
|
78
|
+
"- Add a gap check and one or two follow-up rounds.",
|
|
79
|
+
"- Synthesize a report with inline citations and a citation ledger.",
|
|
80
|
+
]
|
|
81
|
+
)
|
|
82
|
+
return "\n".join(lines).rstrip() + "\n"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def run_research(
|
|
86
|
+
question: str,
|
|
87
|
+
settings: Settings,
|
|
88
|
+
chain: SearchChain | None = None,
|
|
89
|
+
plan: ResearchPlan | None = None,
|
|
90
|
+
max_sources: int | None = None,
|
|
91
|
+
) -> Run:
|
|
92
|
+
plan = plan or plan_question(question, max_sub_questions=4)
|
|
93
|
+
chain = chain or build_chain(settings)
|
|
94
|
+
if not chain.providers:
|
|
95
|
+
raise RuntimeError(
|
|
96
|
+
"No search provider configured. Set FIRECRAWL_URL, BRAVE_API_KEY, "
|
|
97
|
+
"SERPER_API_KEY, or SEARXNG_URL."
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
limit = max(1, max_sources or settings.max_sources)
|
|
101
|
+
buckets: dict[str, list[Source]] = {
|
|
102
|
+
sub_question.question: [] for sub_question in plan.sub_questions
|
|
103
|
+
}
|
|
104
|
+
seen: set[str] = set()
|
|
105
|
+
attempts: list[SearchAttempt] = []
|
|
106
|
+
|
|
107
|
+
for sub_question in plan.sub_questions:
|
|
108
|
+
outcome: SearchOutcome = await chain.search(
|
|
109
|
+
sub_question.question, limit=settings.search_limit
|
|
110
|
+
)
|
|
111
|
+
attempts.extend(outcome.attempts)
|
|
112
|
+
for source in outcome.sources:
|
|
113
|
+
key = normalize_url(source.url)
|
|
114
|
+
if key in seen:
|
|
115
|
+
continue
|
|
116
|
+
seen.add(key)
|
|
117
|
+
buckets[sub_question.question].append(
|
|
118
|
+
replace(source, sub_question=sub_question.question)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
selected: list[Source] = []
|
|
122
|
+
while len(selected) < limit:
|
|
123
|
+
added = False
|
|
124
|
+
for sub_question in plan.sub_questions:
|
|
125
|
+
bucket = buckets[sub_question.question]
|
|
126
|
+
if not bucket:
|
|
127
|
+
continue
|
|
128
|
+
selected.append(bucket.pop(0))
|
|
129
|
+
added = True
|
|
130
|
+
if len(selected) >= limit:
|
|
131
|
+
break
|
|
132
|
+
if not added:
|
|
133
|
+
break
|
|
134
|
+
|
|
135
|
+
selected = tuple(selected)
|
|
136
|
+
report = render_dossier(plan, selected, tuple(attempts))
|
|
137
|
+
return Run(
|
|
138
|
+
id=_run_id(plan.question),
|
|
139
|
+
question=plan.question,
|
|
140
|
+
plan=plan,
|
|
141
|
+
sources=selected,
|
|
142
|
+
attempts=tuple(attempts),
|
|
143
|
+
report=report,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
__all__ = ["render_dossier", "run_research"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from ..models import ResearchPlan, SubQuestion
|
|
4
|
+
|
|
5
|
+
FACETS: tuple[tuple[str, str], ...] = (
|
|
6
|
+
("overview and definitions", "Establish shared terms and background."),
|
|
7
|
+
("evidence and data", "Find quantitative or empirical evidence."),
|
|
8
|
+
("critiques and limitations", "Look for counterarguments, failures, and known limits."),
|
|
9
|
+
("recent developments", "Check what changed recently."),
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def plan_question(question: str, max_sub_questions: int = 4) -> ResearchPlan:
|
|
14
|
+
"""Deterministic Phase 0 plan: the question plus focused facets."""
|
|
15
|
+
normalized = " ".join(question.split())
|
|
16
|
+
if not normalized:
|
|
17
|
+
raise ValueError("question is required")
|
|
18
|
+
count = min(max(1, max_sub_questions), len(FACETS) + 1)
|
|
19
|
+
sub_questions = [SubQuestion(normalized, "Answer the question directly.")]
|
|
20
|
+
for suffix, rationale in FACETS[: count - 1]:
|
|
21
|
+
sub_questions.append(SubQuestion(f"{normalized} {suffix}", rationale))
|
|
22
|
+
return ResearchPlan(normalized, tuple(sub_questions))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = ["FACETS", "plan_question"]
|
indagar/search/base.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Protocol
|
|
4
|
+
|
|
5
|
+
from ..models import Source
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SearchError(Exception):
|
|
9
|
+
"""A provider failed. The chain records it and moves on."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, provider: str, message: str, status: int | None = None):
|
|
12
|
+
super().__init__(message)
|
|
13
|
+
self.provider = provider
|
|
14
|
+
self.status = status
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SearchProvider(Protocol):
|
|
18
|
+
name: str
|
|
19
|
+
|
|
20
|
+
async def search(self, query: str, limit: int) -> list[Source]:
|
|
21
|
+
"""Return up to limit results. Raise SearchError on failure."""
|
|
22
|
+
...
|
indagar/search/chain.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections import OrderedDict
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
|
7
|
+
|
|
8
|
+
from ..config import Settings
|
|
9
|
+
from ..models import SearchAttempt, SearchOutcome, Source
|
|
10
|
+
from .base import SearchProvider
|
|
11
|
+
from .providers import create_provider
|
|
12
|
+
|
|
13
|
+
TRACKING_PARAMS = {
|
|
14
|
+
"utm_source",
|
|
15
|
+
"utm_medium",
|
|
16
|
+
"utm_campaign",
|
|
17
|
+
"utm_term",
|
|
18
|
+
"utm_content",
|
|
19
|
+
"fbclid",
|
|
20
|
+
"gclid",
|
|
21
|
+
"mc_cid",
|
|
22
|
+
"mc_eid",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def normalize_url(raw: str) -> str:
|
|
27
|
+
parsed = urlparse(raw.strip())
|
|
28
|
+
hostname = (parsed.hostname or "").lower()
|
|
29
|
+
if hostname.startswith("www."):
|
|
30
|
+
hostname = hostname[4:]
|
|
31
|
+
query = [
|
|
32
|
+
(key, value)
|
|
33
|
+
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
|
|
34
|
+
if key.lower() not in TRACKING_PARAMS
|
|
35
|
+
]
|
|
36
|
+
path = parsed.path.rstrip("/") or "/"
|
|
37
|
+
return urlunparse(
|
|
38
|
+
(
|
|
39
|
+
parsed.scheme.lower(),
|
|
40
|
+
hostname,
|
|
41
|
+
path,
|
|
42
|
+
"",
|
|
43
|
+
urlencode(query),
|
|
44
|
+
"",
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _is_http_url(value: str) -> bool:
|
|
50
|
+
try:
|
|
51
|
+
parsed = urlparse(value)
|
|
52
|
+
except ValueError:
|
|
53
|
+
return False
|
|
54
|
+
return parsed.scheme in {"http", "https"} and bool(parsed.hostname)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class SearchChain:
|
|
58
|
+
"""Ordered providers with over-fetch, dedupe, and a short cache."""
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
providers: list[SearchProvider],
|
|
63
|
+
ttl_seconds: int = 600,
|
|
64
|
+
max_entries: int = 100,
|
|
65
|
+
):
|
|
66
|
+
self.providers = providers
|
|
67
|
+
self.ttl_seconds = ttl_seconds
|
|
68
|
+
self.max_entries = max_entries
|
|
69
|
+
self._cache: OrderedDict[str, tuple[float, SearchOutcome]] = OrderedDict()
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def provider_names(self) -> tuple[str, ...]:
|
|
73
|
+
return tuple(provider.name for provider in self.providers)
|
|
74
|
+
|
|
75
|
+
async def search(self, query: str, limit: int = 8) -> SearchOutcome:
|
|
76
|
+
key = self._cache_key(query, limit)
|
|
77
|
+
cached = self._cache_get(key)
|
|
78
|
+
if cached is not None:
|
|
79
|
+
return cached
|
|
80
|
+
|
|
81
|
+
attempts: list[SearchAttempt] = []
|
|
82
|
+
seen: set[str] = set()
|
|
83
|
+
|
|
84
|
+
for provider in self.providers:
|
|
85
|
+
started = time.perf_counter()
|
|
86
|
+
try:
|
|
87
|
+
raw = await provider.search(query, limit * 2)
|
|
88
|
+
results: list[Source] = []
|
|
89
|
+
for source in raw:
|
|
90
|
+
if not _is_http_url(source.url):
|
|
91
|
+
continue
|
|
92
|
+
normalized = normalize_url(source.url)
|
|
93
|
+
if normalized in seen:
|
|
94
|
+
continue
|
|
95
|
+
seen.add(normalized)
|
|
96
|
+
results.append(
|
|
97
|
+
replace(
|
|
98
|
+
source,
|
|
99
|
+
provider=source.provider or provider.name,
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
if len(results) >= limit:
|
|
103
|
+
break
|
|
104
|
+
latency = int((time.perf_counter() - started) * 1000)
|
|
105
|
+
attempts.append(
|
|
106
|
+
SearchAttempt(
|
|
107
|
+
provider=provider.name,
|
|
108
|
+
status="ok" if results else "empty",
|
|
109
|
+
count=len(results),
|
|
110
|
+
latency_ms=latency,
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
if results:
|
|
114
|
+
outcome = SearchOutcome(tuple(results), tuple(attempts))
|
|
115
|
+
self._cache_set(key, outcome)
|
|
116
|
+
return outcome
|
|
117
|
+
except Exception as exc: # noqa: BLE001 - one provider must not stop the chain
|
|
118
|
+
latency = int((time.perf_counter() - started) * 1000)
|
|
119
|
+
attempts.append(
|
|
120
|
+
SearchAttempt(
|
|
121
|
+
provider=provider.name,
|
|
122
|
+
status="error",
|
|
123
|
+
count=0,
|
|
124
|
+
latency_ms=latency,
|
|
125
|
+
error=str(exc),
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
return SearchOutcome((), tuple(attempts))
|
|
130
|
+
|
|
131
|
+
def _cache_key(self, query: str, limit: int) -> str:
|
|
132
|
+
normalized_query = " ".join(query.lower().split())
|
|
133
|
+
providers = ",".join(self.provider_names)
|
|
134
|
+
return f"{normalized_query}|{limit}|{providers}"
|
|
135
|
+
|
|
136
|
+
def _cache_get(self, key: str) -> SearchOutcome | None:
|
|
137
|
+
entry = self._cache.get(key)
|
|
138
|
+
if entry is None:
|
|
139
|
+
return None
|
|
140
|
+
stored_at, outcome = entry
|
|
141
|
+
if time.monotonic() - stored_at >= self.ttl_seconds:
|
|
142
|
+
self._cache.pop(key, None)
|
|
143
|
+
return None
|
|
144
|
+
self._cache.move_to_end(key)
|
|
145
|
+
return outcome
|
|
146
|
+
|
|
147
|
+
def _cache_set(self, key: str, outcome: SearchOutcome) -> None:
|
|
148
|
+
if not outcome.sources:
|
|
149
|
+
return
|
|
150
|
+
if len(self._cache) >= self.max_entries:
|
|
151
|
+
self._cache.popitem(last=False)
|
|
152
|
+
self._cache[key] = (time.monotonic(), outcome)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def build_chain(settings: Settings) -> SearchChain:
|
|
156
|
+
providers = [
|
|
157
|
+
provider
|
|
158
|
+
for name in settings.configured_search_providers()
|
|
159
|
+
if (provider := create_provider(name, settings)) is not None
|
|
160
|
+
]
|
|
161
|
+
return SearchChain(providers)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
__all__ = ["SearchChain", "build_chain", "normalize_url"]
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from urllib.parse import urlparse
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from ..config import Settings
|
|
8
|
+
from ..models import Source
|
|
9
|
+
from .base import SearchError
|
|
10
|
+
|
|
11
|
+
USER_AGENT = "Indagar/0.1 (+https://github.com/michael-borck/indagar)"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _is_http_url(value: str) -> bool:
|
|
15
|
+
try:
|
|
16
|
+
parsed = urlparse(value)
|
|
17
|
+
except ValueError:
|
|
18
|
+
return False
|
|
19
|
+
return parsed.scheme in {"http", "https"} and bool(parsed.hostname)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _source(
|
|
23
|
+
title: str | None,
|
|
24
|
+
url: str | None,
|
|
25
|
+
snippet: str | None,
|
|
26
|
+
provider: str,
|
|
27
|
+
) -> Source | None:
|
|
28
|
+
if not title or not url or not _is_http_url(url):
|
|
29
|
+
return None
|
|
30
|
+
return Source(
|
|
31
|
+
title=" ".join(title.split()),
|
|
32
|
+
url=url,
|
|
33
|
+
snippet=" ".join((snippet or "").split())[:600],
|
|
34
|
+
provider=provider,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _status_error(provider: str, response: httpx.Response) -> SearchError:
|
|
39
|
+
return SearchError(
|
|
40
|
+
provider,
|
|
41
|
+
f"{provider} returned HTTP {response.status_code}",
|
|
42
|
+
status=response.status_code,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class FirecrawlProvider:
|
|
47
|
+
name = "firecrawl"
|
|
48
|
+
|
|
49
|
+
def __init__(self, base_url: str, api_key: str | None = None):
|
|
50
|
+
self.base_url = base_url.rstrip("/")
|
|
51
|
+
self.api_key = api_key
|
|
52
|
+
|
|
53
|
+
async def search(self, query: str, limit: int) -> list[Source]:
|
|
54
|
+
headers = {"content-type": "application/json", "user-agent": USER_AGENT}
|
|
55
|
+
if self.api_key:
|
|
56
|
+
headers["authorization"] = f"Bearer {self.api_key}"
|
|
57
|
+
try:
|
|
58
|
+
async with httpx.AsyncClient(timeout=30.0) as client:
|
|
59
|
+
response = await client.post(
|
|
60
|
+
f"{self.base_url}/v2/search",
|
|
61
|
+
json={"query": query, "limit": max(1, limit), "sources": ["web"]},
|
|
62
|
+
headers=headers,
|
|
63
|
+
)
|
|
64
|
+
except httpx.HTTPError as exc:
|
|
65
|
+
raise SearchError(self.name, f"firecrawl request failed: {exc}") from exc
|
|
66
|
+
if response.status_code >= 400:
|
|
67
|
+
raise _status_error(self.name, response)
|
|
68
|
+
data = response.json()
|
|
69
|
+
web = ((data.get("data") or {}).get("web")) or []
|
|
70
|
+
results: list[Source] = []
|
|
71
|
+
for item in web:
|
|
72
|
+
source = _source(
|
|
73
|
+
item.get("title"),
|
|
74
|
+
item.get("url"),
|
|
75
|
+
item.get("description"),
|
|
76
|
+
self.name,
|
|
77
|
+
)
|
|
78
|
+
if source:
|
|
79
|
+
results.append(source)
|
|
80
|
+
return results
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class BraveProvider:
|
|
84
|
+
name = "brave"
|
|
85
|
+
|
|
86
|
+
def __init__(self, api_key: str):
|
|
87
|
+
self.api_key = api_key
|
|
88
|
+
|
|
89
|
+
async def search(self, query: str, limit: int) -> list[Source]:
|
|
90
|
+
params = {
|
|
91
|
+
"q": query,
|
|
92
|
+
"count": min(max(limit, 1), 20),
|
|
93
|
+
"search_lang": "en",
|
|
94
|
+
"country": "US",
|
|
95
|
+
"safesearch": "moderate",
|
|
96
|
+
}
|
|
97
|
+
headers = {
|
|
98
|
+
"X-Subscription-Token": self.api_key,
|
|
99
|
+
"Accept": "application/json",
|
|
100
|
+
"User-Agent": USER_AGENT,
|
|
101
|
+
}
|
|
102
|
+
try:
|
|
103
|
+
async with httpx.AsyncClient(timeout=20.0) as client:
|
|
104
|
+
response = await client.get(
|
|
105
|
+
"https://api.search.brave.com/res/v1/web/search",
|
|
106
|
+
params=params,
|
|
107
|
+
headers=headers,
|
|
108
|
+
)
|
|
109
|
+
except httpx.HTTPError as exc:
|
|
110
|
+
raise SearchError(self.name, f"brave request failed: {exc}") from exc
|
|
111
|
+
if response.status_code >= 400:
|
|
112
|
+
raise _status_error(self.name, response)
|
|
113
|
+
data = response.json()
|
|
114
|
+
results: list[Source] = []
|
|
115
|
+
for item in (data.get("web") or {}).get("results") or []:
|
|
116
|
+
source = _source(
|
|
117
|
+
item.get("title"),
|
|
118
|
+
item.get("url"),
|
|
119
|
+
item.get("description"),
|
|
120
|
+
self.name,
|
|
121
|
+
)
|
|
122
|
+
if source:
|
|
123
|
+
results.append(source)
|
|
124
|
+
return results
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class SerperProvider:
|
|
128
|
+
name = "serper"
|
|
129
|
+
|
|
130
|
+
def __init__(self, api_key: str):
|
|
131
|
+
self.api_key = api_key
|
|
132
|
+
|
|
133
|
+
async def search(self, query: str, limit: int) -> list[Source]:
|
|
134
|
+
headers = {
|
|
135
|
+
"X-API-KEY": self.api_key,
|
|
136
|
+
"content-type": "application/json",
|
|
137
|
+
"user-agent": USER_AGENT,
|
|
138
|
+
}
|
|
139
|
+
try:
|
|
140
|
+
async with httpx.AsyncClient(timeout=20.0) as client:
|
|
141
|
+
response = await client.post(
|
|
142
|
+
"https://google.serper.dev/search",
|
|
143
|
+
json={"q": query, "num": min(max(limit, 1), 100)},
|
|
144
|
+
headers=headers,
|
|
145
|
+
)
|
|
146
|
+
except httpx.HTTPError as exc:
|
|
147
|
+
raise SearchError(self.name, f"serper request failed: {exc}") from exc
|
|
148
|
+
if response.status_code >= 400:
|
|
149
|
+
raise _status_error(self.name, response)
|
|
150
|
+
data = response.json()
|
|
151
|
+
results: list[Source] = []
|
|
152
|
+
for item in data.get("organic") or []:
|
|
153
|
+
source = _source(
|
|
154
|
+
item.get("title"),
|
|
155
|
+
item.get("link"),
|
|
156
|
+
item.get("snippet"),
|
|
157
|
+
self.name,
|
|
158
|
+
)
|
|
159
|
+
if source:
|
|
160
|
+
results.append(source)
|
|
161
|
+
return results
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class SearxngProvider:
|
|
165
|
+
name = "searxng"
|
|
166
|
+
|
|
167
|
+
def __init__(self, base_url: str):
|
|
168
|
+
self.base_url = base_url.rstrip("/")
|
|
169
|
+
|
|
170
|
+
async def search(self, query: str, limit: int) -> list[Source]:
|
|
171
|
+
params = {"q": query, "format": "json", "safesearch": "1"}
|
|
172
|
+
headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
|
|
173
|
+
try:
|
|
174
|
+
async with httpx.AsyncClient(timeout=20.0) as client:
|
|
175
|
+
response = await client.get(
|
|
176
|
+
f"{self.base_url}/search",
|
|
177
|
+
params=params,
|
|
178
|
+
headers=headers,
|
|
179
|
+
)
|
|
180
|
+
except httpx.HTTPError as exc:
|
|
181
|
+
raise SearchError(self.name, f"searxng request failed: {exc}") from exc
|
|
182
|
+
if response.status_code >= 400:
|
|
183
|
+
raise _status_error(self.name, response)
|
|
184
|
+
data = response.json()
|
|
185
|
+
results: list[Source] = []
|
|
186
|
+
for item in data.get("results") or []:
|
|
187
|
+
source = _source(
|
|
188
|
+
item.get("title"),
|
|
189
|
+
item.get("url"),
|
|
190
|
+
item.get("content"),
|
|
191
|
+
self.name,
|
|
192
|
+
)
|
|
193
|
+
if source:
|
|
194
|
+
results.append(source)
|
|
195
|
+
return results
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def create_provider(name: str, settings: Settings):
|
|
199
|
+
if name == "firecrawl" and settings.firecrawl_url:
|
|
200
|
+
return FirecrawlProvider(settings.firecrawl_url, settings.firecrawl_api_key)
|
|
201
|
+
if name == "brave" and settings.brave_api_key:
|
|
202
|
+
return BraveProvider(settings.brave_api_key)
|
|
203
|
+
if name == "serper" and settings.serper_api_key:
|
|
204
|
+
return SerperProvider(settings.serper_api_key)
|
|
205
|
+
if name == "searxng" and settings.searxng_url:
|
|
206
|
+
return SearxngProvider(settings.searxng_url)
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
__all__ = [
|
|
211
|
+
"BraveProvider",
|
|
212
|
+
"FirecrawlProvider",
|
|
213
|
+
"SearxngProvider",
|
|
214
|
+
"SerperProvider",
|
|
215
|
+
"create_provider",
|
|
216
|
+
]
|
indagar/store.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .models import Run
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def save_run(
|
|
10
|
+
run: Run,
|
|
11
|
+
out: Path | None = None,
|
|
12
|
+
runs_dir: Path = Path("runs"),
|
|
13
|
+
) -> tuple[Path, Path]:
|
|
14
|
+
"""Write the run record and Markdown dossier. Returns (json, markdown)."""
|
|
15
|
+
runs_dir.mkdir(parents=True, exist_ok=True)
|
|
16
|
+
json_path = runs_dir / f"{run.id}.json"
|
|
17
|
+
markdown_path = out or runs_dir / f"{run.id}.md"
|
|
18
|
+
markdown_path.parent.mkdir(parents=True, exist_ok=True)
|
|
19
|
+
|
|
20
|
+
json_path.write_text(
|
|
21
|
+
json.dumps(run.to_dict(), indent=2, ensure_ascii=False) + "\n",
|
|
22
|
+
encoding="utf-8",
|
|
23
|
+
)
|
|
24
|
+
markdown_path.write_text(run.report, encoding="utf-8")
|
|
25
|
+
return json_path, markdown_path
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
__all__ = ["save_run"]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: indagar
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deep research from the command line: plan, search, read, and synthesize with citations.
|
|
5
|
+
Author: Michael Borck
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: citations,cli,deep-research,research,web-search
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Requires-Dist: httpx>=0.27
|
|
17
|
+
Requires-Dist: python-dotenv>=1.0
|
|
18
|
+
Requires-Dist: rich>=13.7
|
|
19
|
+
Requires-Dist: typer>=0.12
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# Indagar
|
|
26
|
+
|
|
27
|
+
**Deep research from the command line.**
|
|
28
|
+
|
|
29
|
+
Indagar asks a question, plans sub-questions, searches multiple sources, reads
|
|
30
|
+
what it finds, and produces a cited research dossier. The name is Spanish for
|
|
31
|
+
*to investigate, inquire, or probe*.
|
|
32
|
+
|
|
33
|
+
This repository is the Phase 0 skeleton: a working CLI, a multi-provider search
|
|
34
|
+
chain, a deterministic research plan, and a source dossier. LLM planning,
|
|
35
|
+
extraction, and synthesis are the next phase.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
git clone https://github.com/michael-borck/indagar.git
|
|
41
|
+
cd indagar
|
|
42
|
+
uv venv
|
|
43
|
+
source .venv/bin/activate
|
|
44
|
+
uv pip install -e '.[dev]'
|
|
45
|
+
|
|
46
|
+
indagar doctor
|
|
47
|
+
indagar search "retrieval augmented generation" --limit 5
|
|
48
|
+
indagar research "Does retrieval augmented generation reduce hallucination?" \
|
|
49
|
+
--max-sources 10
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
If the checkout is on an exFAT drive, create the virtual environment on a
|
|
53
|
+
local disk instead: `uv venv ~/.venvs/indagar`.
|
|
54
|
+
|
|
55
|
+
The `research` command writes a Markdown dossier and a JSON run record under
|
|
56
|
+
`runs/`. Every source keeps its URL, provider, and sub-question.
|
|
57
|
+
|
|
58
|
+
## Configuration
|
|
59
|
+
|
|
60
|
+
Copy `.env.example` to `.env` and set the providers you have.
|
|
61
|
+
|
|
62
|
+
| Variable | Purpose |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `FIRECRAWL_URL` | Self-hosted or cloud Firecrawl base URL, e.g. `http://localhost:3002` |
|
|
65
|
+
| `FIRECRAWL_API_KEY` | Optional key when Firecrawl is behind an auth gate |
|
|
66
|
+
| `BRAVE_API_KEY` | Brave Search API key |
|
|
67
|
+
| `SERPER_API_KEY` | Serper API key |
|
|
68
|
+
| `SEARXNG_URL` | Self-hosted SearXNG base URL |
|
|
69
|
+
| `INDAGAR_SEARCH_ORDER` | Provider order. Default `firecrawl,brave,serper,searxng` |
|
|
70
|
+
| `ANTHROPIC_API_KEY` | Anthropic key for the next LLM phase |
|
|
71
|
+
| `OPENAI_API_KEY` | OpenAI-compatible key |
|
|
72
|
+
| `OPENAI_BASE_URL` | OpenAI-compatible base URL, including Ollama/LM Studio |
|
|
73
|
+
| `INDAGAR_MAX_SOURCES` | Sources per run. Default `12` |
|
|
74
|
+
| `INDAGAR_MAX_SECONDS` | Wall-clock budget. Default `300` |
|
|
75
|
+
| `INDAGAR_SEARCH_LIMIT` | Results requested per sub-question. Default `8` |
|
|
76
|
+
|
|
77
|
+
`indagar doctor` shows what is configured.
|
|
78
|
+
|
|
79
|
+
## Status
|
|
80
|
+
|
|
81
|
+
- [x] CLI with `doctor`, `search`, and `research`
|
|
82
|
+
- [x] Firecrawl, Brave, Serper, and SearXNG adapters
|
|
83
|
+
- [x] Ordered fallback, over-fetch, URL dedupe, and a 10-minute cache
|
|
84
|
+
- [x] Deterministic research plan and cited source dossier
|
|
85
|
+
- [ ] LLM planner, claim extraction, and synthesis
|
|
86
|
+
- [ ] Web UI and desktop shell
|
|
87
|
+
- [ ] Academic, podcast, and video sources
|
|
88
|
+
|
|
89
|
+
See [SPEC.md](SPEC.md) for the design and roadmap.
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
uv pip install -e '.[dev]'
|
|
95
|
+
ruff check .
|
|
96
|
+
ruff format --check .
|
|
97
|
+
pytest -q
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Licence
|
|
101
|
+
|
|
102
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
indagar/__init__.py,sha256=Io3bD5DCVPKQK-d4CTwvXxYw8ggfnctl6DtA88y3siw,78
|
|
2
|
+
indagar/__main__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
|
|
3
|
+
indagar/cli.py,sha256=TpFzkp3z5FMZMOc472HwGXb9UwlUH6uLSKTmqi1M0uQ,4698
|
|
4
|
+
indagar/config.py,sha256=zXmSLxPav5PUGlnI3KeAhynUWrMN0v7YWadi1rcysng,3074
|
|
5
|
+
indagar/models.py,sha256=Se5XYNEQq64zqoTzg4426aVVqgs74HnVg2iAi0AP-WA,2041
|
|
6
|
+
indagar/store.py,sha256=z3E4EKnAbPavaVQdge_-OeeDH5JYOy3MlkY8QeM0kho,751
|
|
7
|
+
indagar/research/__init__.py,sha256=nCuhySlDmopoANC1OWlCC4QJmR7Mrw2Z5Q17cX5eLRc,111
|
|
8
|
+
indagar/research/loop.py,sha256=UHxxNVXBEPxKZhiRHMAzbaDoAjHk-gsb18qBPamoj9k,4727
|
|
9
|
+
indagar/research/planner.py,sha256=SL5uaihLABRvKGFZAGjWq_cxSUnFBxUThC_8uUC_orA,1061
|
|
10
|
+
indagar/search/__init__.py,sha256=LS0L_Lx4WYgqO_CXJNeRCFkecwKCRmUZgRhkegASXTg,197
|
|
11
|
+
indagar/search/base.py,sha256=oKv-bfHfdP1GJarMmRtErgVshu4IfGpDkTMB-igYUrk,567
|
|
12
|
+
indagar/search/chain.py,sha256=qly3OG4bf9uVsBYFxqsCGp-vDK1zhnf76gZKFQF4c08,5211
|
|
13
|
+
indagar/search/providers.py,sha256=N7zxlACL9UVk4eoI8O-VbJHeLmLfnGvN_IAy1a0ZFB0,6927
|
|
14
|
+
indagar-0.1.0.dist-info/METADATA,sha256=ejM7FGJGPwAXCdg0um4cvTK2p13QXJUaYTwoFB4OIdI,3364
|
|
15
|
+
indagar-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
16
|
+
indagar-0.1.0.dist-info/entry_points.txt,sha256=GHooEipv6OwNv2VLTtsprOFiIFl8sU8TQpwvTQlqlac,45
|
|
17
|
+
indagar-0.1.0.dist-info/licenses/LICENSE,sha256=tP28nznO05HKBQN8uZ9NSvmcpzCWMj4P8e6F7IDj24Q,1070
|
|
18
|
+
indagar-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Michael Borck
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|