deep-dossier 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ """
2
+ Deep Dossier - Autonomous multi-step research agent using Google Gemini.
3
+ """
4
+
5
+ __version__ = "0.1.0"
6
+ __author__ = "Shubhransh Gupta"
7
+
8
+ from .agent import DeepResearchAgent
9
+ from .config import ResearchConfig
10
+
11
+ __all__ = ["DeepResearchAgent", "ResearchConfig", "__version__"]
deep_dossier/agent.py ADDED
@@ -0,0 +1,222 @@
1
+ """
2
+ Core DeepResearchAgent orchestrator leveraging Gemini for autonomous multi-step research.
3
+ """
4
+
5
+ import os
6
+ import re
7
+ import json
8
+ import logging
9
+ from typing import List, Dict, Any, Optional
10
+ from datetime import datetime
11
+
12
+ from .config import ResearchConfig
13
+ from .search import SearchEngine, SearchResult
14
+ from .crawler import WebCrawler, ScrapedDocument
15
+ from .llm import UnifiedLLM
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ class DeepResearchAgent:
21
+ """
22
+ Autonomous research agent that decomposes questions, searches the web recursively,
23
+ crawls deep documentation/papers, and synthesizes publication-grade reports.
24
+ Supports Gemini, OpenAI, Claude, DeepSeek, Nemotron, and Ollama.
25
+ """
26
+
27
+ def __init__(self, config: Optional[ResearchConfig] = None):
28
+ self.config = config or ResearchConfig()
29
+ self.config.validate()
30
+
31
+ self.search_engine = SearchEngine(timeout=self.config.timeout_seconds)
32
+ self.crawler = WebCrawler(
33
+ user_agent=self.config.user_agent,
34
+ timeout=self.config.timeout_seconds,
35
+ )
36
+ self.llm = UnifiedLLM(
37
+ provider=self.config.provider,
38
+ model=self.config.model,
39
+ api_key=self.config.api_key,
40
+ base_url=self.config.base_url,
41
+ timeout=self.config.timeout_seconds * 3,
42
+ )
43
+
44
+ def _call_llm(self, prompt: str, system_instruction: Optional[str] = None) -> str:
45
+ """Invokes the unified multi-provider LLM."""
46
+ return self.llm.generate(prompt, system_instruction=system_instruction)
47
+
48
+ def research(self, topic: str, on_progress=None) -> Dict[str, Any]:
49
+ """
50
+ Executes an end-to-end autonomous deep research workflow.
51
+ """
52
+ def report_step(step_msg: str):
53
+ if on_progress:
54
+ on_progress(step_msg)
55
+ elif self.config.verbose:
56
+ print(f"[*] {step_msg}")
57
+
58
+ report_step(f"Starting deep research on: '{topic}'")
59
+
60
+ # Phase 1: Planning and Query Decomposition
61
+ report_step("Phase 1: Generating research plan and query decomposition...")
62
+ initial_queries = self._generate_search_queries(topic)
63
+ report_step(f"Generated {len(initial_queries)} research angles: {', '.join(initial_queries)}")
64
+
65
+ all_scraped_docs: Dict[str, ScrapedDocument] = {}
66
+ all_sources: List[Dict[str, str]] = []
67
+ visited_urls = set()
68
+
69
+ current_queries = initial_queries
70
+
71
+ # Phase 2: Recursive Deep Web Crawling Loop
72
+ for depth in range(1, self.config.max_depth + 1):
73
+ report_step(f"Phase 2 (Depth {depth}/{self.config.max_depth}): Searching and extracting evidence...")
74
+
75
+ new_urls = []
76
+ for q in current_queries:
77
+ search_results = self.search_engine.search(q, max_results=self.config.breadth)
78
+ for res in search_results:
79
+ if res.url not in visited_urls and len(visited_urls) < self.config.max_sources_to_scrape:
80
+ visited_urls.add(res.url)
81
+ new_urls.append(res.url)
82
+ all_sources.append({"title": res.title, "url": res.url, "snippet": res.snippet})
83
+
84
+ if not new_urls:
85
+ report_step("No new URLs found in this pass. Proceeding to synthesis...")
86
+ break
87
+
88
+ report_step(f"Crawling {len(new_urls)} primary sources in parallel...")
89
+ scraped = self.crawler.scrape_urls(new_urls)
90
+ for doc in scraped:
91
+ all_scraped_docs[doc.url] = doc
92
+
93
+ total_chars = sum(d.char_count for d in all_scraped_docs.values())
94
+ report_step(f"Aggregated {len(all_scraped_docs)} documents ({total_chars:,} characters of raw knowledge).")
95
+
96
+ # If more depth remaining, hypothesize next missing gaps
97
+ if depth < self.config.max_depth and len(visited_urls) < self.config.max_sources_to_scrape:
98
+ report_step("Analyzing knowledge gaps to formulate next follow-up queries...")
99
+ current_queries = self._generate_followup_queries(topic, list(all_scraped_docs.values()))
100
+ report_step(f"Follow-up queries: {', '.join(current_queries)}")
101
+
102
+ # Phase 3: Synthesis & Report Generation
103
+ report_step("Phase 3: Synthesizing publication-grade report with citations...")
104
+ markdown_report = self._synthesize_report(topic, list(all_scraped_docs.values()), all_sources)
105
+
106
+ report_step("Deep research completed successfully.")
107
+
108
+ slug = re.sub(r"[^a-zA-Z0-9_-]", "_", topic.lower())[:50].strip("_")
109
+ timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
110
+
111
+ return {
112
+ "topic": topic,
113
+ "slug": slug,
114
+ "timestamp": timestamp,
115
+ "markdown_report": markdown_report,
116
+ "sources": all_sources,
117
+ "document_count": len(all_scraped_docs),
118
+ "total_chars_analyzed": sum(d.char_count for d in all_scraped_docs.values()),
119
+ }
120
+
121
+ def _generate_search_queries(self, topic: str) -> List[str]:
122
+ """Prompt Gemini to break topic down into 3-4 diverse search queries."""
123
+ prompt = f"""
124
+ You are an expert investigative research director.
125
+ Given the research topic: "{topic}"
126
+ Formulate {self.config.max_queries_per_step} high-precision, diverse search engine queries.
127
+ Cover:
128
+ 1. Core technical definition / state of the art.
129
+ 2. Concrete benchmark numbers, metrics, or recent developments.
130
+ 3. Critiques, limitations, controversies, or competitor comparisons.
131
+
132
+ Return ONLY a JSON list of query strings. Example:
133
+ ["query 1", "query 2", "query 3"]
134
+ """
135
+ response = self._call_llm(prompt)
136
+ try:
137
+ match = re.search(r"\[.*\]", response, re.DOTALL)
138
+ if match:
139
+ queries = json.loads(match.group(0))
140
+ return [str(q).strip() for q in queries if str(q).strip()][:self.config.max_queries_per_step]
141
+ except Exception:
142
+ pass
143
+ return [f"{topic} overview", f"{topic} latest benchmarks", f"{topic} challenges"]
144
+
145
+ def _generate_followup_queries(self, topic: str, docs: List[ScrapedDocument]) -> List[str]:
146
+ """Determine what data is still ambiguous and generate next targeted queries."""
147
+ snippets = "\n\n".join([f"[{d.title}]: {d.text_content[:800]}" for d in docs[:5]])
148
+ prompt = f"""
149
+ Research Topic: "{topic}"
150
+ We have collected initial documents containing the following excerpts:
151
+ {snippets}
152
+
153
+ Identify the most critical missing pieces of data, contradictions, or unanswered questions.
154
+ Generate 2 targeted search queries to resolve these specific gaps.
155
+
156
+ Return ONLY a JSON array of query strings. Example:
157
+ ["specific query 1", "specific query 2"]
158
+ """
159
+ response = self._call_llm(prompt)
160
+ try:
161
+ match = re.search(r"\[.*\]", response, re.DOTALL)
162
+ if match:
163
+ queries = json.loads(match.group(0))
164
+ return [str(q).strip() for q in queries if str(q).strip()][:2]
165
+ except Exception:
166
+ pass
167
+ return [f"{topic} detailed analysis", f"{topic} case studies"]
168
+
169
+ def _synthesize_report(
170
+ self,
171
+ topic: str,
172
+ docs: List[ScrapedDocument],
173
+ sources: List[Dict[str, str]]
174
+ ) -> str:
175
+ """
176
+ Synthesizes an extensive, publication-grade research report.
177
+ Feeds the rich multi-document context to Gemini's large context window.
178
+ """
179
+ context_blocks = []
180
+ for idx, doc in enumerate(docs, 1):
181
+ context_blocks.append(
182
+ f"--- SOURCE [{idx}] ---\n"
183
+ f"Title: {doc.title}\n"
184
+ f"URL: {doc.url}\n"
185
+ f"Content:\n{doc.text_content}\n"
186
+ )
187
+
188
+ full_context = "\n\n".join(context_blocks)
189
+ if len(full_context) > 200000:
190
+ full_context = full_context[:200000] + "\n\n...[Truncated context]..."
191
+
192
+ system_instruction = (
193
+ "You are a world-class research scientist and principal technology analyst. "
194
+ "Write comprehensive, rigorous, deeply technical research reports. "
195
+ "Never use fluff, cliches, or shallow generalities. Use concrete figures, tables, and citations."
196
+ )
197
+
198
+ prompt = f"""
199
+ RESEARCH TOPIC: "{topic}"
200
+
201
+ BELOW IS THE VERIFIED RAW SOURCE MATERIAL CRAWLED FROM THE WEB:
202
+ {full_context}
203
+
204
+ TASK:
205
+ Write an authoritative, publication-ready research dossier addressing "{topic}".
206
+
207
+ STRUCTURE YOUR REPORT USING THE FOLLOWING SECTIONS:
208
+ 1. Executive Summary (Key takeaways, core findings, bottom-line verdict)
209
+ 2. Background & Context (Why this matters, historical evolution)
210
+ 3. Technical Architecture & Deep Dive (Specific mechanisms, formulations, protocols)
211
+ 4. Empirical Data & Comparative Analysis (Create clear Markdown tables comparing metrics, benchmarks, or approaches)
212
+ 5. Critical Challenges & Limitations (What are the bottlenecks, failures, or tradeoffs?)
213
+ 6. Future Trajectory & Strategic Implications (Where is this heading over the next 2-5 years?)
214
+ 7. Key Takeaways & Actionable Guidance
215
+
216
+ CITATION RULES:
217
+ - Use in-text numeric citations like [1], [2], [3] whenever citing specific facts, numbers, or claims.
218
+ - At the end of the report, include a 'References' section matching the numbers to the URLs from the source material.
219
+
220
+ Ensure the report is thorough, highly informative, and reads like a premier institutional research paper.
221
+ """
222
+ return self._call_llm(prompt, system_instruction=system_instruction)
deep_dossier/cli.py ADDED
@@ -0,0 +1,116 @@
1
+ """
2
+ Command-line interface for Open Deep Research with rich terminal styling.
3
+ """
4
+
5
+ import sys
6
+ import argparse
7
+ from pathlib import Path
8
+ from rich.console import Console
9
+ from rich.panel import Panel
10
+ from rich.table import Table
11
+ from rich.markdown import Markdown
12
+
13
+ from .config import ResearchConfig
14
+ from .agent import DeepResearchAgent
15
+ from .report import save_reports
16
+
17
+ console = Console()
18
+
19
+ BANNER = """[bold cyan]
20
+ ___ ___ _
21
+ / _ \\___ ___ ___ / _ \\___ ___ ___ (_)__ ____
22
+ / // / -_) -_) _ \\ / // / _ \\(_-<-_-'/ / -_) __/
23
+ /____/\\__/\\__/ .__/ /____/\\___/___/___//_/\\__/_/
24
+ /_/
25
+ [/bold cyan]
26
+ [dim]⚡ Autonomous Multi-Step Research Agent • Gemini | Claude | GPT | DeepSeek | Nemotron[/dim]
27
+ """
28
+
29
+
30
+ def main():
31
+ parser = argparse.ArgumentParser(
32
+ description="Deep Dossier — Autonomous multi-step deep research agent supporting Gemini, OpenAI, Anthropic, DeepSeek, and Ollama."
33
+ )
34
+ parser.add_argument("topic", nargs="?", help="The topic or research question to investigate.")
35
+ parser.add_argument("--depth", "-d", type=int, default=2, help="Research depth iterations (default: 2)")
36
+ parser.add_argument("--breadth", "-b", type=int, default=4, help="Breadth per search step (default: 4)")
37
+ parser.add_argument("--provider", "-p", type=str, default=None, help="LLM Provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter")
38
+ parser.add_argument("--model", "-m", type=str, default=None, help="Model identifier (e.g., gemini-2.5-flash, claude-3-5-sonnet, gpt-4o, etc.)")
39
+ parser.add_argument("--output", "-o", type=str, default="./research_outputs", help="Output directory")
40
+ parser.add_argument("--api-key", "-k", type=str, default=None, help="API Key override")
41
+ parser.add_argument("--base-url", type=str, default=None, help="Custom API Base URL (for Ollama, vLLM, or self-hosted endpoints)")
42
+ parser.add_argument("--no-html", action="store_true", help="Disable HTML report generation")
43
+
44
+ args = parser.parse_args()
45
+
46
+ console.print(BANNER)
47
+
48
+ topic = args.topic
49
+ if not topic:
50
+ topic = console.input("[bold yellow]Enter your research topic/question: [/bold yellow]").strip()
51
+
52
+ if not topic:
53
+ console.print("[red]Error: Research topic cannot be empty.[/red]")
54
+ sys.exit(1)
55
+
56
+ config = ResearchConfig(
57
+ provider=args.provider,
58
+ model=args.model,
59
+ api_key=args.api_key,
60
+ base_url=args.base_url,
61
+ max_depth=args.depth,
62
+ breadth=args.breadth,
63
+ output_dir=Path(args.output),
64
+ generate_html=not args.no_html,
65
+ )
66
+
67
+ try:
68
+ config.validate()
69
+ except ValueError as e:
70
+ console.print(Panel(f"[bold red]Configuration Error:[/bold red]\n{str(e)}", border_style="red"))
71
+ sys.exit(1)
72
+
73
+ agent = DeepResearchAgent(config=config)
74
+
75
+ console.print(Panel(
76
+ f"[bold white]Topic:[/bold white] {topic}\n"
77
+ f"[bold white]Provider:[/bold white] {agent.llm.provider.upper()} | [bold white]Model:[/bold white] {agent.llm.model}\n"
78
+ f"[bold white]Depth:[/bold white] {config.max_depth} | [bold white]Breadth:[/bold white] {config.breadth}",
79
+ title="[bold green]Research Mission Activated[/bold green]",
80
+ border_style="cyan"
81
+ ))
82
+
83
+ with console.status("[bold green]Agent orchestrating deep research...[/bold green]", spinner="dots") as status:
84
+ def on_progress(msg: str):
85
+ status.update(f"[bold cyan]{msg}[/bold cyan]")
86
+ console.print(f" [dim]•[/dim] {msg}")
87
+
88
+ try:
89
+ result = agent.research(topic=topic, on_progress=on_progress)
90
+ except Exception as e:
91
+ console.print(f"\n[bold red]Error during research execution:[/bold red] {e}")
92
+ sys.exit(1)
93
+
94
+ # Save reports
95
+ paths = save_reports(result, config.output_dir, result["slug"])
96
+
97
+ # Output Summary Table
98
+ table = Table(title="Generated Research Deliverables", border_style="green")
99
+ table.add_column("Format", style="cyan", no_wrap=True)
100
+ table.add_column("Location", style="white")
101
+
102
+ if "markdown" in paths:
103
+ table.add_row("Markdown (.md)", str(paths["markdown"].resolve()))
104
+ if "html" in paths:
105
+ table.add_row("Interactive HTML", str(paths["html"].resolve()))
106
+ if "json" in paths:
107
+ table.add_row("JSON Dossier", str(paths["json"].resolve()))
108
+
109
+ console.print("\n")
110
+ console.print(table)
111
+ console.print(f"\n[bold green]✨ Research Complete![/bold green] Analyzed [bold]{result['document_count']}[/bold] sources ({result['total_chars_analyzed']:,} characters).")
112
+ console.print(f"To view HTML report in browser: [bold cyan]open {paths.get('html')}[/bold cyan]\n")
113
+
114
+
115
+ if __name__ == "__main__":
116
+ main()
deep_dossier/config.py ADDED
@@ -0,0 +1,85 @@
1
+ """
2
+ Configuration settings and environment management for Deep Dossier.
3
+ """
4
+
5
+ import os
6
+ from pathlib import Path
7
+ from dataclasses import dataclass, field
8
+ from typing import Optional, List
9
+
10
+
11
+ def load_env_file(dotenv_path: Optional[str] = None):
12
+ """Simple robust .env loader."""
13
+ paths_to_check = [
14
+ dotenv_path,
15
+ Path.cwd() / ".env",
16
+ Path.cwd() / ".env.local",
17
+ Path.home() / ".config" / "deep-dossier" / ".env",
18
+ ]
19
+ for p in paths_to_check:
20
+ if p and Path(p).is_file():
21
+ try:
22
+ with open(p, "r", encoding="utf-8") as f:
23
+ for line in f:
24
+ line = line.strip()
25
+ if not line or line.startswith("#") or "=" not in line:
26
+ continue
27
+ k, v = line.split("=", 1)
28
+ k = k.strip()
29
+ v = v.strip().strip("'\"")
30
+ if k and k not in os.environ:
31
+ os.environ[k] = v
32
+ break
33
+ except Exception:
34
+ pass
35
+
36
+
37
+ load_env_file()
38
+
39
+
40
+ @dataclass
41
+ class ResearchConfig:
42
+ """Config options for the Deep Dossier Agent supporting multiple LLM providers."""
43
+ provider: Optional[str] = None # 'gemini', 'openai', 'anthropic', 'deepseek', 'nvidia', 'ollama', 'openrouter'
44
+ model: Optional[str] = None
45
+ api_key: Optional[str] = None
46
+ base_url: Optional[str] = None
47
+ max_depth: int = 2
48
+ breadth: int = 4
49
+ max_queries_per_step: int = 3
50
+ max_sources_to_scrape: int = 15
51
+ timeout_seconds: int = 20
52
+ user_agent: str = (
53
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
54
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 (DeepDossier/0.1.0)"
55
+ )
56
+ output_dir: Path = field(default_factory=lambda: Path.cwd() / "research_outputs")
57
+ generate_html: bool = True
58
+ generate_markdown: bool = True
59
+ generate_json: bool = True
60
+ verbose: bool = True
61
+
62
+ def validate(self):
63
+ """Validates that at least one provider key or base_url is accessible."""
64
+ available_keys = [
65
+ self.api_key,
66
+ os.getenv("GEMINI_API_KEY"),
67
+ os.getenv("OPENAI_API_KEY"),
68
+ os.getenv("ANTHROPIC_API_KEY"),
69
+ os.getenv("OPENROUTER_API_KEY"),
70
+ os.getenv("DEEPSEEK_API_KEY"),
71
+ os.getenv("NVIDIA_API_KEY"),
72
+ self.base_url,
73
+ ]
74
+ if not any(available_keys):
75
+ raise ValueError(
76
+ "No API Key found.\n"
77
+ "Please set one of the following environment variables:\n"
78
+ " • export GEMINI_API_KEY='...'\n"
79
+ " • export OPENAI_API_KEY='...'\n"
80
+ " • export ANTHROPIC_API_KEY='...'\n"
81
+ " • export OPENROUTER_API_KEY='...'\n"
82
+ " • export DEEPSEEK_API_KEY='...'\n"
83
+ " • export NVIDIA_API_KEY='...'\n"
84
+ "Or specify --api-key / --base-url (for local Ollama/vLLM)."
85
+ )
@@ -0,0 +1,145 @@
1
+ """
2
+ Web crawler and content extractor for deep document analysis.
3
+ """
4
+
5
+ import re
6
+ import logging
7
+ from typing import Optional, List, Dict
8
+ from dataclasses import dataclass
9
+ from concurrent.futures import ThreadPoolExecutor, as_completed
10
+ import requests
11
+ from bs4 import BeautifulSoup
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ @dataclass
17
+ class ScrapedDocument:
18
+ url: str
19
+ title: str
20
+ text_content: str
21
+ char_count: int
22
+ success: bool
23
+ error: Optional[str] = None
24
+
25
+
26
+ class WebCrawler:
27
+ """Extracts clean article content from URLs in parallel."""
28
+
29
+ def __init__(self, user_agent: str, timeout: int = 15, max_workers: int = 5):
30
+ self.user_agent = user_agent
31
+ self.timeout = timeout
32
+ self.max_workers = max_workers
33
+ self.session = requests.Session()
34
+ self.session.headers.update({
35
+ "User-Agent": self.user_agent,
36
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
37
+ "Accept-Language": "en-US,en;q=0.5",
38
+ })
39
+
40
+ def scrape_url(self, url: str) -> ScrapedDocument:
41
+ """Fetch and extract readable text from a single webpage."""
42
+ try:
43
+ resp = self.session.get(url, timeout=self.timeout, allow_redirects=True)
44
+ resp.raise_for_status()
45
+
46
+ # Check content type
47
+ content_type = resp.headers.get("Content-Type", "").lower()
48
+ if "pdf" in content_type or url.lower().endswith(".pdf"):
49
+ return self._extract_pdf(resp.content, url)
50
+
51
+ return self._extract_html(resp.text, url)
52
+ except Exception as e:
53
+ logger.debug(f"Failed to crawl {url}: {e}")
54
+ return ScrapedDocument(
55
+ url=url,
56
+ title="",
57
+ text_content="",
58
+ char_count=0,
59
+ success=False,
60
+ error=str(e),
61
+ )
62
+
63
+ def scrape_urls(self, urls: List[str]) -> List[ScrapedDocument]:
64
+ """Crawl multiple URLs concurrently."""
65
+ results: List[ScrapedDocument] = []
66
+ unique_urls = list(dict.fromkeys(urls)) # Preserve order, deduplicate
67
+
68
+ with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
69
+ future_to_url = {executor.submit(self.scrape_url, u): u for u in unique_urls}
70
+ for future in as_completed(future_to_url):
71
+ res = future.result()
72
+ if res.success and res.char_count > 100:
73
+ results.append(res)
74
+ return results
75
+
76
+ def _extract_html(self, html: str, url: str) -> ScrapedDocument:
77
+ """Parse HTML, eliminate clutter, and extract dense readable text."""
78
+ soup = BeautifulSoup(html, "html.parser")
79
+
80
+ # Extract title
81
+ title = ""
82
+ if soup.title and soup.title.string:
83
+ title = soup.title.string.strip()
84
+ elif soup.find("h1"):
85
+ title = soup.find("h1").get_text(strip=True)
86
+
87
+ # Remove clutter
88
+ for unwanted in soup(["script", "style", "nav", "footer", "header", "noscript", "svg", "aside"]):
89
+ unwanted.decompose()
90
+
91
+ # Target main content containers if present
92
+ main_content = (
93
+ soup.find("article")
94
+ or soup.find("main")
95
+ or soup.find("div", {"id": re.compile(r"(content|main|article)", re.I)})
96
+ or soup.find("div", {"class": re.compile(r"(content|main|article|post)", re.I)})
97
+ or soup.body
98
+ )
99
+
100
+ if not main_content:
101
+ text = soup.get_text(separator="\n", strip=True)
102
+ else:
103
+ text = main_content.get_text(separator="\n", strip=True)
104
+
105
+ # Normalize extra blank lines
106
+ lines = [line.strip() for line in text.splitlines() if line.strip()]
107
+ cleaned_text = "\n\n".join(lines)
108
+
109
+ # Cap text length to avoid token explosion (e.g. 50,000 characters per document)
110
+ if len(cleaned_text) > 60000:
111
+ cleaned_text = cleaned_text[:60000] + "\n...[truncated for length]..."
112
+
113
+ return ScrapedDocument(
114
+ url=url,
115
+ title=title or url,
116
+ text_content=cleaned_text,
117
+ char_count=len(cleaned_text),
118
+ success=True,
119
+ )
120
+
121
+ def _extract_pdf(self, content: bytes, url: str) -> ScrapedDocument:
122
+ """Extract text from PDF if pypdf or PyPDF2 is available, otherwise basic raw fallback."""
123
+ try:
124
+ import io
125
+ import pypdf
126
+ reader = pypdf.PdfReader(io.BytesIO(content))
127
+ text = ""
128
+ for page in reader.pages[:20]: # First 20 pages max
129
+ text += page.extract_text() + "\n"
130
+ return ScrapedDocument(
131
+ url=url,
132
+ title=f"PDF Document: {url.split('/')[-1]}",
133
+ text_content=text.strip(),
134
+ char_count=len(text),
135
+ success=True,
136
+ )
137
+ except Exception:
138
+ return ScrapedDocument(
139
+ url=url,
140
+ title=f"PDF: {url.split('/')[-1]}",
141
+ text_content="[Binary PDF content could not be parsed without pypdf]",
142
+ char_count=0,
143
+ success=False,
144
+ error="pypdf not installed",
145
+ )
deep_dossier/llm.py ADDED
@@ -0,0 +1,188 @@
1
+ """
2
+ Unified multi-provider LLM interface supporting Gemini, OpenAI, Anthropic, DeepSeek, Nemotron, and Ollama.
3
+ """
4
+
5
+ import os
6
+ import json
7
+ import logging
8
+ from typing import Optional, Dict, Any, List
9
+ import requests
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ class UnifiedLLM:
15
+ """
16
+ Universal LLM client that routes requests to:
17
+ - Google Gemini
18
+ - OpenAI (GPT-4o, o1, o3-mini, etc.)
19
+ - Anthropic (Claude 3.5 Sonnet, Claude 3.7 Sonnet, etc.)
20
+ - OpenAI-Compatible endpoints (OpenRouter, DeepSeek, NVIDIA Nemotron, Ollama, vLLM)
21
+ """
22
+
23
+ def __init__(
24
+ self,
25
+ provider: Optional[str] = None,
26
+ model: Optional[str] = None,
27
+ api_key: Optional[str] = None,
28
+ base_url: Optional[str] = None,
29
+ timeout: int = 60,
30
+ ):
31
+ self.api_key = api_key
32
+ self.base_url = base_url
33
+ self.timeout = timeout
34
+ self.provider, self.model = self._resolve_provider_and_model(provider, model)
35
+ self._resolve_api_key()
36
+
37
+ def _resolve_provider_and_model(self, provider: Optional[str], model: Optional[str]) -> tuple:
38
+ """Auto-detect provider based on model name or available environment variables."""
39
+ if provider:
40
+ p = provider.lower()
41
+ m = model or self._default_model_for_provider(p)
42
+ return p, m
43
+
44
+ m = (model or "").lower()
45
+ if "claude" in m:
46
+ return "anthropic", model
47
+ if any(x in m for x in ["gpt", "o1", "o3", "chatgpt"]):
48
+ return "openai", model
49
+ if "gemini" in m:
50
+ return "gemini", model
51
+ if "deepseek" in m:
52
+ return "deepseek", model
53
+ if "nemotron" in m or "nvidia" in m:
54
+ return "nvidia", model
55
+
56
+ # Check environment variable presence
57
+ if os.getenv("GEMINI_API_KEY"):
58
+ return "gemini", model or "gemini-2.5-flash"
59
+ if os.getenv("OPENAI_API_KEY"):
60
+ return "openai", model or "gpt-4o"
61
+ if os.getenv("ANTHROPIC_API_KEY"):
62
+ return "anthropic", model or "claude-3-5-sonnet-20241022"
63
+ if os.getenv("OPENROUTER_API_KEY"):
64
+ return "openrouter", model or "meta-llama/llama-3.3-70b-instruct"
65
+ if os.getenv("DEEPSEEK_API_KEY"):
66
+ return "deepseek", model or "deepseek-chat"
67
+
68
+ return "gemini", model or "gemini-2.5-flash"
69
+
70
+ def _default_model_for_provider(self, provider: str) -> str:
71
+ defaults = {
72
+ "gemini": "gemini-2.5-flash",
73
+ "openai": "gpt-4o",
74
+ "anthropic": "claude-3-5-sonnet-20241022",
75
+ "openrouter": "meta-llama/llama-3.3-70b-instruct",
76
+ "deepseek": "deepseek-chat",
77
+ "nvidia": "nvidia/nemotron-4-340b-instruct",
78
+ "ollama": "llama3.2",
79
+ }
80
+ return defaults.get(provider, "gemini-2.5-flash")
81
+
82
+ def _resolve_api_key(self):
83
+ if self.api_key:
84
+ return
85
+
86
+ key_map = {
87
+ "gemini": "GEMINI_API_KEY",
88
+ "openai": "OPENAI_API_KEY",
89
+ "anthropic": "ANTHROPIC_API_KEY",
90
+ "openrouter": "OPENROUTER_API_KEY",
91
+ "deepseek": "DEEPSEEK_API_KEY",
92
+ "nvidia": "NVIDIA_API_KEY",
93
+ }
94
+ env_var = key_map.get(self.provider)
95
+ if env_var:
96
+ self.api_key = os.getenv(env_var)
97
+
98
+ def generate(self, prompt: str, system_instruction: Optional[str] = None) -> str:
99
+ """Dispatch generation to appropriate provider implementation."""
100
+ if self.provider == "gemini":
101
+ return self._call_gemini(prompt, system_instruction)
102
+ elif self.provider == "anthropic":
103
+ return self._call_anthropic(prompt, system_instruction)
104
+ elif self.provider in ("openai", "openrouter", "deepseek", "nvidia", "ollama") or self.base_url:
105
+ return self._call_openai_compatible(prompt, system_instruction)
106
+ else:
107
+ return self._call_gemini(prompt, system_instruction)
108
+
109
+ def _call_gemini(self, prompt: str, system_instruction: Optional[str]) -> str:
110
+ # Try google-genai SDK first
111
+ try:
112
+ from google import genai
113
+ from google.genai import types
114
+ client = genai.Client(api_key=self.api_key)
115
+ cfg = {}
116
+ if system_instruction:
117
+ cfg["system_instruction"] = system_instruction
118
+ res = client.models.generate_content(
119
+ model=self.model,
120
+ contents=prompt,
121
+ config=types.GenerateContentConfig(**cfg) if cfg else None
122
+ )
123
+ if res and res.text:
124
+ return res.text
125
+ except Exception:
126
+ pass
127
+
128
+ # Fallback to direct REST endpoint
129
+ url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent?key={self.api_key}"
130
+ contents = []
131
+ if system_instruction:
132
+ contents.append({"role": "user", "parts": [{"text": f"SYSTEM: {system_instruction}"}]})
133
+ contents.append({"role": "user", "parts": [{"text": prompt}]})
134
+ resp = requests.post(url, json={"contents": contents}, timeout=self.timeout)
135
+ resp.raise_for_status()
136
+ data = resp.json()
137
+ return data["candidates"][0]["content"]["parts"][0]["text"]
138
+
139
+ def _call_anthropic(self, prompt: str, system_instruction: Optional[str]) -> str:
140
+ url = "https://api.anthropic.com/v1/messages"
141
+ headers = {
142
+ "x-api-key": self.api_key or "",
143
+ "anthropic-version": "2023-06-01",
144
+ "content-type": "application/json",
145
+ }
146
+ payload = {
147
+ "model": self.model,
148
+ "max_tokens": 4096,
149
+ "messages": [{"role": "user", "content": prompt}],
150
+ }
151
+ if system_instruction:
152
+ payload["system"] = system_instruction
153
+
154
+ resp = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
155
+ resp.raise_for_status()
156
+ data = resp.json()
157
+ content = data.get("content", [])
158
+ if content and "text" in content[0]:
159
+ return content[0]["text"]
160
+ return ""
161
+
162
+ def _call_openai_compatible(self, prompt: str, system_instruction: Optional[str]) -> str:
163
+ endpoint_map = {
164
+ "openai": "https://api.openai.com/v1/chat/completions",
165
+ "openrouter": "https://openrouter.ai/api/v1/chat/completions",
166
+ "deepseek": "https://api.deepseek.com/v1/chat/completions",
167
+ "nvidia": "https://integrate.api.nvidia.com/v1/chat/completions",
168
+ "ollama": "http://localhost:11434/v1/chat/completions",
169
+ }
170
+ url = self.base_url or endpoint_map.get(self.provider, "https://api.openai.com/v1/chat/completions")
171
+ headers = {
172
+ "Authorization": f"Bearer {self.api_key or ''}",
173
+ "Content-Type": "application/json",
174
+ }
175
+ messages = []
176
+ if system_instruction:
177
+ messages.append({"role": "system", "content": system_instruction})
178
+ messages.append({"role": "user", "content": prompt})
179
+
180
+ payload = {
181
+ "model": self.model,
182
+ "messages": messages,
183
+ }
184
+
185
+ resp = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
186
+ resp.raise_for_status()
187
+ data = resp.json()
188
+ return data["choices"][0]["message"]["content"]
deep_dossier/report.py ADDED
@@ -0,0 +1,274 @@
1
+ """
2
+ Report generators for Markdown, interactive HTML, and JSON dossiers.
3
+ """
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Dict, Any, List
8
+ from datetime import datetime
9
+
10
+
11
+ def save_reports(data: Dict[str, Any], output_dir: Path, slug: str) -> Dict[str, Path]:
12
+ """Saves report in Markdown, HTML, and JSON formats."""
13
+ output_dir.mkdir(parents=True, exist_ok=True)
14
+ paths = {}
15
+
16
+ # 1. Save Markdown
17
+ md_path = output_dir / f"{slug}.md"
18
+ with open(md_path, "w", encoding="utf-8") as f:
19
+ f.write(data["markdown_report"])
20
+ paths["markdown"] = md_path
21
+
22
+ # 2. Save JSON Dossier
23
+ json_path = output_dir / f"{slug}.json"
24
+ with open(json_path, "w", encoding="utf-8") as f:
25
+ json.dump(data, f, indent=2, ensure_ascii=False)
26
+ paths["json"] = json_path
27
+
28
+ # 3. Save Interactive HTML
29
+ html_content = generate_html_report(data)
30
+ html_path = output_dir / f"{slug}.html"
31
+ with open(html_path, "w", encoding="utf-8") as f:
32
+ f.write(html_content)
33
+ paths["html"] = html_path
34
+
35
+ return paths
36
+
37
+
38
+ def generate_html_report(data: Dict[str, Any]) -> str:
39
+ """Generates an executive-grade HTML report with modern dark theme and responsive typography."""
40
+ title = data.get("topic", "Research Report")
41
+ timestamp = data.get("timestamp", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"))
42
+ sources: List[Dict[str, str]] = data.get("sources", [])
43
+ markdown_content = data.get("markdown_report", "")
44
+
45
+ # Basic markdown to html conversion or simple rendering
46
+ try:
47
+ import markdown
48
+ rendered_body = markdown.markdown(
49
+ markdown_content,
50
+ extensions=["tables", "fenced_code", "toc"]
51
+ )
52
+ except Exception:
53
+ # Fallback rendering if markdown package is missing
54
+ rendered_body = (
55
+ markdown_content
56
+ .replace("\n\n", "</p><p>")
57
+ .replace("# ", "<h1>")
58
+ .replace("## ", "<h2>")
59
+ .replace("### ", "<h3>")
60
+ )
61
+
62
+ sources_html = ""
63
+ for idx, s in enumerate(sources, 1):
64
+ url = s.get("url", "#")
65
+ stitle = s.get("title", url)
66
+ sources_html += f"""
67
+ <div class="source-card">
68
+ <span class="source-num">[{idx}]</span>
69
+ <div class="source-info">
70
+ <a href="{url}" target="_blank" rel="noopener noreferrer" class="source-title">{stitle}</a>
71
+ <span class="source-url">{url}</span>
72
+ </div>
73
+ </div>
74
+ """
75
+
76
+ html = f"""<!DOCTYPE html>
77
+ <html lang="en">
78
+ <head>
79
+ <meta charset="UTF-8">
80
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
81
+ <title>{title} — Deep Research Dossier</title>
82
+ <link rel="preconnect" href="https://fonts.googleapis.com">
83
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
84
+ <link href="https://fonts.googleapis.com/css2?family=Plus+Jakarta+Sans:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
85
+ <style>
86
+ :root {{
87
+ --bg: #090d16;
88
+ --surface: #111726;
89
+ --surface-border: #1e293b;
90
+ --primary: #38bdf8;
91
+ --primary-glow: rgba(56, 189, 248, 0.15);
92
+ --text-main: #f1f5f9;
93
+ --text-muted: #94a3b8;
94
+ --accent: #818cf8;
95
+ --success: #34d399;
96
+ }}
97
+ * {{ box-sizing: border-box; margin: 0; padding: 0; }}
98
+ body {{
99
+ background-color: var(--bg);
100
+ color: var(--text-main);
101
+ font-family: 'Plus Jakarta Sans', -apple-system, BlinkMacSystemFont, sans-serif;
102
+ line-height: 1.7;
103
+ padding: 2.5rem 1.5rem;
104
+ }}
105
+ .container {{
106
+ max-width: 900px;
107
+ margin: 0 auto;
108
+ }}
109
+ header {{
110
+ border-bottom: 1px solid var(--surface-border);
111
+ padding-bottom: 2rem;
112
+ margin-bottom: 2.5rem;
113
+ }}
114
+ .badge {{
115
+ display: inline-block;
116
+ background: var(--primary-glow);
117
+ color: var(--primary);
118
+ border: 1px solid rgba(56, 189, 248, 0.3);
119
+ font-size: 0.75rem;
120
+ font-weight: 600;
121
+ text-transform: uppercase;
122
+ letter-spacing: 0.05em;
123
+ padding: 0.25rem 0.75rem;
124
+ border-radius: 9999px;
125
+ margin-bottom: 1rem;
126
+ }}
127
+ h1 {{
128
+ font-size: 2.25rem;
129
+ font-weight: 700;
130
+ line-height: 1.25;
131
+ color: #ffffff;
132
+ margin-bottom: 0.75rem;
133
+ }}
134
+ .meta {{
135
+ color: var(--text-muted);
136
+ font-size: 0.875rem;
137
+ display: flex;
138
+ gap: 1.5rem;
139
+ }}
140
+ .content {{
141
+ background: var(--surface);
142
+ border: 1px solid var(--surface-border);
143
+ border-radius: 1rem;
144
+ padding: 2.5rem;
145
+ margin-bottom: 3rem;
146
+ box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.3);
147
+ }}
148
+ .content h2 {{
149
+ color: #ffffff;
150
+ font-size: 1.5rem;
151
+ margin-top: 2rem;
152
+ margin-bottom: 1rem;
153
+ border-bottom: 1px solid var(--surface-border);
154
+ padding-bottom: 0.5rem;
155
+ }}
156
+ .content h3 {{
157
+ color: var(--primary);
158
+ font-size: 1.2rem;
159
+ margin-top: 1.5rem;
160
+ margin-bottom: 0.75rem;
161
+ }}
162
+ .content p {{
163
+ margin-bottom: 1.25rem;
164
+ color: #cbd5e1;
165
+ font-size: 1rem;
166
+ }}
167
+ .content ul, .content ol {{
168
+ margin-left: 1.5rem;
169
+ margin-bottom: 1.25rem;
170
+ color: #cbd5e1;
171
+ }}
172
+ .content li {{ margin-bottom: 0.5rem; }}
173
+ .content blockquote {{
174
+ border-left: 4px solid var(--primary);
175
+ padding-left: 1rem;
176
+ margin: 1.5rem 0;
177
+ color: var(--text-muted);
178
+ font-style: italic;
179
+ }}
180
+ .content table {{
181
+ width: 100%;
182
+ border-collapse: collapse;
183
+ margin: 1.5rem 0;
184
+ }}
185
+ .content th, .content td {{
186
+ padding: 0.75rem 1rem;
187
+ border: 1px solid var(--surface-border);
188
+ text-align: left;
189
+ }}
190
+ .content th {{ background: #1e293b; color: #ffffff; }}
191
+ code {{
192
+ font-family: 'JetBrains Mono', monospace;
193
+ background: #1e293b;
194
+ color: #38bdf8;
195
+ padding: 0.2rem 0.4rem;
196
+ border-radius: 4px;
197
+ font-size: 0.9em;
198
+ }}
199
+ .sources-section {{
200
+ margin-top: 2rem;
201
+ }}
202
+ .source-card {{
203
+ background: var(--surface);
204
+ border: 1px solid var(--surface-border);
205
+ border-radius: 0.5rem;
206
+ padding: 1rem 1.25rem;
207
+ margin-bottom: 0.75rem;
208
+ display: flex;
209
+ align-items: flex-start;
210
+ gap: 1rem;
211
+ transition: border-color 0.2s ease;
212
+ }}
213
+ .source-card:hover {{
214
+ border-color: var(--primary);
215
+ }}
216
+ .source-num {{
217
+ color: var(--primary);
218
+ font-weight: 600;
219
+ font-family: 'JetBrains Mono', monospace;
220
+ }}
221
+ .source-title {{
222
+ color: #ffffff;
223
+ font-weight: 500;
224
+ text-decoration: none;
225
+ display: block;
226
+ margin-bottom: 0.25rem;
227
+ }}
228
+ .source-title:hover {{
229
+ text-decoration: underline;
230
+ color: var(--primary);
231
+ }}
232
+ .source-url {{
233
+ color: var(--text-muted);
234
+ font-size: 0.8rem;
235
+ word-break: break-all;
236
+ }}
237
+ footer {{
238
+ text-align: center;
239
+ color: var(--text-muted);
240
+ font-size: 0.875rem;
241
+ margin-top: 3rem;
242
+ border-top: 1px solid var(--surface-border);
243
+ padding-top: 1.5rem;
244
+ }}
245
+ </style>
246
+ </head>
247
+ <body>
248
+ <div class="container">
249
+ <header>
250
+ <span class="badge">Open Deep Research &bull; Gemini Autonomous Agent</span>
251
+ <h1>{title}</h1>
252
+ <div class="meta">
253
+ <span>Generated: {timestamp}</span>
254
+ <span>Sources verified: {len(sources)}</span>
255
+ </div>
256
+ </header>
257
+
258
+ <main class="content">
259
+ {rendered_body}
260
+ </main>
261
+
262
+ <section class="sources-section">
263
+ <h2>Verified Citations & Bibliography ({len(sources)})</h2>
264
+ {sources_html}
265
+ </section>
266
+
267
+ <footer>
268
+ Generated by <a href="https://github.com/shubhransh-gupta/open-deep-research" style="color:var(--primary);text-decoration:none;">Open Deep Research</a> &bull; Powered by Google Gemini
269
+ </footer>
270
+ </div>
271
+ </body>
272
+ </html>
273
+ """
274
+ return html
deep_dossier/search.py ADDED
@@ -0,0 +1,97 @@
1
+ """
2
+ Search module providing search capabilities via DuckDuckGo and custom search backends.
3
+ """
4
+
5
+ import time
6
+ import logging
7
+ from typing import List, Dict, Optional
8
+ from dataclasses import dataclass
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ @dataclass
14
+ class SearchResult:
15
+ title: str
16
+ url: str
17
+ snippet: str
18
+ source: str = "duckduckgo"
19
+
20
+
21
+ class SearchEngine:
22
+ """Multi-backend search engine supporting DuckDuckGo and fallback methods."""
23
+
24
+ def __init__(self, timeout: int = 15):
25
+ self.timeout = timeout
26
+
27
+ def search(self, query: str, max_results: int = 5) -> List[SearchResult]:
28
+ """Perform search with retries and fallback."""
29
+ results: List[SearchResult] = []
30
+ try:
31
+ results = self._search_duckduckgo(query, max_results)
32
+ if results:
33
+ return results
34
+ except Exception as e:
35
+ logger.warning(f"DuckDuckGo search failed for '{query}': {e}")
36
+
37
+ # Fallback search if DDG failed
38
+ try:
39
+ results = self._search_html_fallback(query, max_results)
40
+ except Exception as e:
41
+ logger.error(f"Fallback search also failed for '{query}': {e}")
42
+
43
+ return results
44
+
45
+ def _search_duckduckgo(self, query: str, max_results: int) -> List[SearchResult]:
46
+ """Search using duckduckgo_search library."""
47
+ try:
48
+ from duckduckgo_search import DDGS
49
+ with DDGS() as ddgs:
50
+ raw_results = list(ddgs.text(query, max_results=max_results))
51
+ results = []
52
+ for r in raw_results:
53
+ title = r.get("title", "")
54
+ url = r.get("href", "")
55
+ body = r.get("body", "")
56
+ if url and (title or body):
57
+ results.append(SearchResult(title=title, url=url, snippet=body))
58
+ return results
59
+ except Exception as e:
60
+ logger.debug(f"DDGS error: {e}")
61
+ raise
62
+
63
+ def _search_html_fallback(self, query: str, max_results: int) -> List[SearchResult]:
64
+ """Direct lightweight HTTP fallback for DuckDuckGo HTML endpoint."""
65
+ import requests
66
+ from bs4 import BeautifulSoup
67
+ import urllib.parse
68
+
69
+ url = "https://html.duckduckgo.com/html/"
70
+ headers = {
71
+ "User-Agent": (
72
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
73
+ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
74
+ )
75
+ }
76
+ resp = requests.post(
77
+ url,
78
+ data={"q": query},
79
+ headers=headers,
80
+ timeout=self.timeout
81
+ )
82
+ resp.raise_for_status()
83
+
84
+ soup = BeautifulSoup(resp.text, "html.parser")
85
+ results = []
86
+ for result in soup.find_all("div", class_="result", limit=max_results):
87
+ title_tag = result.find("a", class_="result__a")
88
+ snippet_tag = result.find("a", class_="result__snippet")
89
+ if title_tag:
90
+ raw_link = title_tag.get("href", "")
91
+ parsed_link = urllib.parse.parse_qs(urllib.parse.urlparse(raw_link).query).get("uddg")
92
+ clean_url = parsed_link[0] if parsed_link else raw_link
93
+ title = title_tag.get_text(strip=True)
94
+ snippet = snippet_tag.get_text(strip=True) if snippet_tag else ""
95
+ if clean_url.startswith("http"):
96
+ results.append(SearchResult(title=title, url=clean_url, snippet=snippet, source="ddg-html"))
97
+ return results
@@ -0,0 +1,144 @@
1
+ Metadata-Version: 2.4
2
+ Name: deep-dossier
3
+ Version: 0.1.0
4
+ Summary: Autonomous multi-step research agent using Gemini for recursive web discovery, synthesis, and verified intelligence dossiers.
5
+ Author: Shubhransh Gupta
6
+ License: MIT
7
+ Keywords: deep-research,gemini,ai-agent,research-agent,autonomous-agent,llm,google-gemini,dossier,report-generator,python,open-source
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.10
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: google-genai>=3.0.0
20
+ Requires-Dist: rich>=13.0.0
21
+ Requires-Dist: requests>=2.28.0
22
+ Requires-Dist: duckduckgo-search>=7.0.0
23
+ Requires-Dist: beautifulsoup4>=4.12.0
24
+ Requires-Dist: pydantic>=2.0.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
27
+
28
+ # ⚡ Deep Dossier
29
+
30
+ [![Documentation](https://img.shields.io/badge/Documentation-Apple%20HIG%20Style-blue?style=for-the-badge&logo=vercel)](https://agent-docs-coral.vercel.app)
31
+ [![GitHub Stars](https://img.shields.io/github/stars/shubhransh-gupta/deep-dossier?style=for-the-badge&color=2997ff)](https://github.com/shubhransh-gupta/deep-dossier/stargazers)
32
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg?style=for-the-badge)](https://opensource.org/licenses/MIT)
33
+
34
+ > **📖 Live Documentation & Interactive FAQs:** [https://agent-docs-coral.vercel.app](https://agent-docs-coral.vercel.app)
35
+
36
+ <p align="center">
37
+ <img src="assets/demo.svg" alt="Deep Dossier Live Terminal Demo" width="100%" />
38
+ </p>
39
+
40
+ An open-source autonomous multi-step research agent with **universal model support**.
41
+
42
+ Deep Dossier decomposes complex research prompts into targeted search plans, recursively crawls dozens of web pages and documents in parallel, resolves information gaps, and produces publication-grade Markdown and interactive HTML dossiers with verified in-text citations.
43
+
44
+ ## 🌐 Universal Model Support (Bring Your Own Key)
45
+
46
+ Deep Dossier is model-agnostic. Choose your preferred AI provider or run completely local:
47
+
48
+ | Provider | Supported Models | Environment Variable |
49
+ | :--- | :--- | :--- |
50
+ | **Google Gemini** | `gemini-2.5-flash`, `gemini-2.5-pro`, `gemini-1.5-pro` | `export GEMINI_API_KEY="..."` |
51
+ | **Anthropic Claude** | `claude-3-5-sonnet-20241022`, `claude-3-7-sonnet`, `claude-3-opus` | `export ANTHROPIC_API_KEY="..."` |
52
+ | **OpenAI** | `gpt-4o`, `gpt-4o-mini`, `o1`, `o3-mini` | `export OPENAI_API_KEY="..."` |
53
+ | **DeepSeek** | `deepseek-chat`, `deepseek-reasoner` | `export DEEPSEEK_API_KEY="..."` |
54
+ | **NVIDIA Nemotron** | `nvidia/nemotron-4-340b-instruct` | `export NVIDIA_API_KEY="..."` |
55
+ | **OpenRouter** | 200+ models across all frontier labs | `export OPENROUTER_API_KEY="..."` |
56
+ | **Local / Self-Hosted** | Ollama (`llama3.2`, `mistral`, `qwen2.5`), vLLM | `--base-url http://localhost:11434/v1` |
57
+
58
+ ## 🌟 Key Highlights
59
+
60
+ - **Multi-Provider Flexibility**: Switch between Gemini, Claude, GPT, DeepSeek, or local Ollama with a single flag.
61
+ - **Recursive Search & Discovery**: Explores multi-hop questions by identifying knowledge gaps after each crawl iteration.
62
+ - **Zero-Config Search**: Built-in DuckDuckGo search integration requiring zero additional API keys.
63
+ - **Publication Deliverables**: Exports structured Markdown, JSON intelligence dossiers, and self-contained dark-mode interactive HTML reports.
64
+
65
+ ## 🚀 Quick Start
66
+
67
+ ### 1. Installation
68
+
69
+ ```bash
70
+ git clone https://github.com/shubhransh-gupta/deep-dossier.git
71
+ cd deep-dossier
72
+ pip install -e .
73
+ ```
74
+
75
+ ### 2. Set Your Preferred API Key
76
+
77
+ Choose any provider:
78
+ ```bash
79
+ # Option A: Google Gemini
80
+ export GEMINI_API_KEY="AIzaSy..."
81
+
82
+ # Option B: Anthropic Claude
83
+ export ANTHROPIC_API_KEY="sk-ant-..."
84
+
85
+ # Option C: OpenAI
86
+ export OPENAI_API_KEY="sk-proj-..."
87
+
88
+ # Option D: DeepSeek
89
+ export DEEPSEEK_API_KEY="sk-..."
90
+ ```
91
+
92
+ ### 3. Run a Research Mission
93
+
94
+ Using default/auto-detected provider:
95
+ ```bash
96
+ deep-dossier "Analyze battery degradation patterns of LFP vs NMC cells in commercial EVs"
97
+ ```
98
+
99
+ Explicitly choose provider and model:
100
+ ```bash
101
+ # Using Anthropic Claude 3.5 Sonnet
102
+ deep-dossier "Solid-state battery commercial readiness" --provider anthropic --model claude-3-5-sonnet-20241022
103
+
104
+ # Using OpenAI GPT-4o
105
+ deep-dossier "Cross-border payment rail comparison" --provider openai --model gpt-4o
106
+
107
+ # Using Local Ollama
108
+ deep-dossier "Local AI agent architectures" --base-url http://localhost:11434/v1 --model llama3.2
109
+ ```
110
+
111
+ ## ⚙️ Options
112
+
113
+ ```bash
114
+ deep-dossier [TOPIC] [OPTIONS]
115
+
116
+ Options:
117
+ --provider, -p TEXT LLM provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter
118
+ --model, -m TEXT Specific model name
119
+ --base-url TEXT Custom API base URL (for Ollama, vLLM, or custom proxies)
120
+ --depth, -d INTEGER Recursive search depth (default: 2)
121
+ --breadth, -b INTEGER Number of sources to retrieve per search query (default: 4)
122
+ --output, -o PATH Output directory for generated dossiers (default: ./research_outputs)
123
+ --api-key, -k TEXT Explicit API key override
124
+ --no-html Disable HTML report generation
125
+ ```
126
+
127
+ ## 📦 Project Structure
128
+
129
+ ```
130
+ deep-dossier/
131
+ ├── deep_dossier/
132
+ │ ├── agent.py # Core autonomous orchestrator and synthesis engine
133
+ │ ├── llm.py # Universal multi-provider LLM interface (Gemini/Claude/GPT/Ollama)
134
+ │ ├── crawler.py # Concurrent HTML & document content extractor
135
+ │ ├── search.py # Search engine interface
136
+ │ ├── report.py # Markdown and interactive HTML renderer
137
+ │ ├── config.py # Configuration and environment loader
138
+ │ └── cli.py # Rich terminal interface
139
+ └── pyproject.toml # Build metadata and entry points
140
+ ```
141
+
142
+ ## 📄 License
143
+
144
+ MIT License. Copyright (c) 2026 Shubhransh Gupta.
@@ -0,0 +1,13 @@
1
+ deep_dossier/__init__.py,sha256=C0mlOTi4jYCw59wnPfHCSqfEQcR7Vo5jkvFJ3QbGozk,275
2
+ deep_dossier/agent.py,sha256=P05foZXGMnZX3mr4FPCoc4vpbAjC5BOHar4wUOzEYIQ,9480
3
+ deep_dossier/cli.py,sha256=8Q8-1IWzRh2dVCdAdG0AJC2H9yqh6mC__jtmQWu1hBg,4805
4
+ deep_dossier/config.py,sha256=T1MlJxtriOgiHeq2xBf-jFJQ6rpZEw5Onuu5P2oaXKU,3054
5
+ deep_dossier/crawler.py,sha256=9Qejw_Sd-4uxFQtTyRL-9XHgDEK5cBlnlEnYTbNOHRU,5212
6
+ deep_dossier/llm.py,sha256=3riyTdETThp8zit6OWlaFxBY7d6y9gCQ41xW9XOWHnU,7305
7
+ deep_dossier/report.py,sha256=C-OOYK7O0X5SQ-oJZgHhYK3shCkiQcHvOhwrfBwN23Q,8814
8
+ deep_dossier/search.py,sha256=WSSBHTCH_vLiq05lGryaCdNNH2SH1UFVBt4NtR54QiA,3550
9
+ deep_dossier-0.1.0.dist-info/METADATA,sha256=ySuN05IyYyaUBEqdkyV4Z2io_tnCkWLklAZbLdk2fdY,6133
10
+ deep_dossier-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
11
+ deep_dossier-0.1.0.dist-info/entry_points.txt,sha256=fMwjyvybhDrkWdWAM7B8ChZXoesSHGguqxmR7XXcJ2c,55
12
+ deep_dossier-0.1.0.dist-info/top_level.txt,sha256=yMpgleYZ0W_lpONVB6pmznhTtS3yURuBALaTVevzag0,13
13
+ deep_dossier-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ deep-dossier = deep_dossier.cli:main
@@ -0,0 +1 @@
1
+ deep_dossier