deep-dossier 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deep_dossier-0.1.0/PKG-INFO +144 -0
- deep_dossier-0.1.0/README.md +117 -0
- deep_dossier-0.1.0/deep_dossier/__init__.py +11 -0
- deep_dossier-0.1.0/deep_dossier/agent.py +222 -0
- deep_dossier-0.1.0/deep_dossier/cli.py +116 -0
- deep_dossier-0.1.0/deep_dossier/config.py +85 -0
- deep_dossier-0.1.0/deep_dossier/crawler.py +145 -0
- deep_dossier-0.1.0/deep_dossier/llm.py +188 -0
- deep_dossier-0.1.0/deep_dossier/report.py +274 -0
- deep_dossier-0.1.0/deep_dossier/search.py +97 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/PKG-INFO +144 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/SOURCES.txt +16 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/dependency_links.txt +1 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/entry_points.txt +2 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/requires.txt +9 -0
- deep_dossier-0.1.0/deep_dossier.egg-info/top_level.txt +1 -0
- deep_dossier-0.1.0/pyproject.toml +55 -0
- deep_dossier-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deep-dossier
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Autonomous multi-step research agent using Gemini for recursive web discovery, synthesis, and verified intelligence dossiers.
|
|
5
|
+
Author: Shubhransh Gupta
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: deep-research,gemini,ai-agent,research-agent,autonomous-agent,llm,google-gemini,dossier,report-generator,python,open-source
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: google-genai>=3.0.0
|
|
20
|
+
Requires-Dist: rich>=13.0.0
|
|
21
|
+
Requires-Dist: requests>=2.28.0
|
|
22
|
+
Requires-Dist: duckduckgo-search>=7.0.0
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
24
|
+
Requires-Dist: pydantic>=2.0.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
27
|
+
|
|
28
|
+
# ⚡ Deep Dossier
|
|
29
|
+
|
|
30
|
+
[](https://agent-docs-coral.vercel.app)
|
|
31
|
+
[](https://github.com/shubhransh-gupta/deep-dossier/stargazers)
|
|
32
|
+
[](https://opensource.org/licenses/MIT)
|
|
33
|
+
|
|
34
|
+
> **📖 Live Documentation & Interactive FAQs:** [https://agent-docs-coral.vercel.app](https://agent-docs-coral.vercel.app)
|
|
35
|
+
|
|
36
|
+
<p align="center">
|
|
37
|
+
<img src="assets/demo.svg" alt="Deep Dossier Live Terminal Demo" width="100%" />
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
An open-source autonomous multi-step research agent with **universal model support**.
|
|
41
|
+
|
|
42
|
+
Deep Dossier decomposes complex research prompts into targeted search plans, recursively crawls dozens of web pages and documents in parallel, resolves information gaps, and produces publication-grade Markdown and interactive HTML dossiers with verified in-text citations.
|
|
43
|
+
|
|
44
|
+
## 🌐 Universal Model Support (Bring Your Own Key)
|
|
45
|
+
|
|
46
|
+
Deep Dossier is model-agnostic. Choose your preferred AI provider or run completely local:
|
|
47
|
+
|
|
48
|
+
| Provider | Supported Models | Environment Variable |
|
|
49
|
+
| :--- | :--- | :--- |
|
|
50
|
+
| **Google Gemini** | `gemini-2.5-flash`, `gemini-2.5-pro`, `gemini-1.5-pro` | `export GEMINI_API_KEY="..."` |
|
|
51
|
+
| **Anthropic Claude** | `claude-3-5-sonnet-20241022`, `claude-3-7-sonnet`, `claude-3-opus` | `export ANTHROPIC_API_KEY="..."` |
|
|
52
|
+
| **OpenAI** | `gpt-4o`, `gpt-4o-mini`, `o1`, `o3-mini` | `export OPENAI_API_KEY="..."` |
|
|
53
|
+
| **DeepSeek** | `deepseek-chat`, `deepseek-reasoner` | `export DEEPSEEK_API_KEY="..."` |
|
|
54
|
+
| **NVIDIA Nemotron** | `nvidia/nemotron-4-340b-instruct` | `export NVIDIA_API_KEY="..."` |
|
|
55
|
+
| **OpenRouter** | 200+ models across all frontier labs | `export OPENROUTER_API_KEY="..."` |
|
|
56
|
+
| **Local / Self-Hosted** | Ollama (`llama3.2`, `mistral`, `qwen2.5`), vLLM | `--base-url http://localhost:11434/v1` |
|
|
57
|
+
|
|
58
|
+
## 🌟 Key Highlights
|
|
59
|
+
|
|
60
|
+
- **Multi-Provider Flexibility**: Switch between Gemini, Claude, GPT, DeepSeek, or local Ollama with a single flag.
|
|
61
|
+
- **Recursive Search & Discovery**: Explores multi-hop questions by identifying knowledge gaps after each crawl iteration.
|
|
62
|
+
- **Zero-Config Search**: Built-in DuckDuckGo search integration requiring zero additional API keys.
|
|
63
|
+
- **Publication Deliverables**: Exports structured Markdown, JSON intelligence dossiers, and self-contained dark-mode interactive HTML reports.
|
|
64
|
+
|
|
65
|
+
## 🚀 Quick Start
|
|
66
|
+
|
|
67
|
+
### 1. Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
git clone https://github.com/shubhransh-gupta/deep-dossier.git
|
|
71
|
+
cd deep-dossier
|
|
72
|
+
pip install -e .
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### 2. Set Your Preferred API Key
|
|
76
|
+
|
|
77
|
+
Choose any provider:
|
|
78
|
+
```bash
|
|
79
|
+
# Option A: Google Gemini
|
|
80
|
+
export GEMINI_API_KEY="AIzaSy..."
|
|
81
|
+
|
|
82
|
+
# Option B: Anthropic Claude
|
|
83
|
+
export ANTHROPIC_API_KEY="sk-ant-..."
|
|
84
|
+
|
|
85
|
+
# Option C: OpenAI
|
|
86
|
+
export OPENAI_API_KEY="sk-proj-..."
|
|
87
|
+
|
|
88
|
+
# Option D: DeepSeek
|
|
89
|
+
export DEEPSEEK_API_KEY="sk-..."
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### 3. Run a Research Mission
|
|
93
|
+
|
|
94
|
+
Using default/auto-detected provider:
|
|
95
|
+
```bash
|
|
96
|
+
deep-dossier "Analyze battery degradation patterns of LFP vs NMC cells in commercial EVs"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Explicitly choose provider and model:
|
|
100
|
+
```bash
|
|
101
|
+
# Using Anthropic Claude 3.5 Sonnet
|
|
102
|
+
deep-dossier "Solid-state battery commercial readiness" --provider anthropic --model claude-3-5-sonnet-20241022
|
|
103
|
+
|
|
104
|
+
# Using OpenAI GPT-4o
|
|
105
|
+
deep-dossier "Cross-border payment rail comparison" --provider openai --model gpt-4o
|
|
106
|
+
|
|
107
|
+
# Using Local Ollama
|
|
108
|
+
deep-dossier "Local AI agent architectures" --base-url http://localhost:11434/v1 --model llama3.2
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## ⚙️ Options
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
deep-dossier [TOPIC] [OPTIONS]
|
|
115
|
+
|
|
116
|
+
Options:
|
|
117
|
+
--provider, -p TEXT LLM provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter
|
|
118
|
+
--model, -m TEXT Specific model name
|
|
119
|
+
--base-url TEXT Custom API base URL (for Ollama, vLLM, or custom proxies)
|
|
120
|
+
--depth, -d INTEGER Recursive search depth (default: 2)
|
|
121
|
+
--breadth, -b INTEGER Number of sources to retrieve per search query (default: 4)
|
|
122
|
+
--output, -o PATH Output directory for generated dossiers (default: ./research_outputs)
|
|
123
|
+
--api-key, -k TEXT Explicit API key override
|
|
124
|
+
--no-html Disable HTML report generation
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## 📦 Project Structure
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
deep-dossier/
|
|
131
|
+
├── deep_dossier/
|
|
132
|
+
│ ├── agent.py # Core autonomous orchestrator and synthesis engine
|
|
133
|
+
│ ├── llm.py # Universal multi-provider LLM interface (Gemini/Claude/GPT/Ollama)
|
|
134
|
+
│ ├── crawler.py # Concurrent HTML & document content extractor
|
|
135
|
+
│ ├── search.py # Search engine interface
|
|
136
|
+
│ ├── report.py # Markdown and interactive HTML renderer
|
|
137
|
+
│ ├── config.py # Configuration and environment loader
|
|
138
|
+
│ └── cli.py # Rich terminal interface
|
|
139
|
+
└── pyproject.toml # Build metadata and entry points
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## 📄 License
|
|
143
|
+
|
|
144
|
+
MIT License. Copyright (c) 2026 Shubhransh Gupta.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# ⚡ Deep Dossier
|
|
2
|
+
|
|
3
|
+
[](https://agent-docs-coral.vercel.app)
|
|
4
|
+
[](https://github.com/shubhransh-gupta/deep-dossier/stargazers)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
|
|
7
|
+
> **📖 Live Documentation & Interactive FAQs:** [https://agent-docs-coral.vercel.app](https://agent-docs-coral.vercel.app)
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<img src="assets/demo.svg" alt="Deep Dossier Live Terminal Demo" width="100%" />
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
An open-source autonomous multi-step research agent with **universal model support**.
|
|
14
|
+
|
|
15
|
+
Deep Dossier decomposes complex research prompts into targeted search plans, recursively crawls dozens of web pages and documents in parallel, resolves information gaps, and produces publication-grade Markdown and interactive HTML dossiers with verified in-text citations.
|
|
16
|
+
|
|
17
|
+
## 🌐 Universal Model Support (Bring Your Own Key)
|
|
18
|
+
|
|
19
|
+
Deep Dossier is model-agnostic. Choose your preferred AI provider or run completely local:
|
|
20
|
+
|
|
21
|
+
| Provider | Supported Models | Environment Variable |
|
|
22
|
+
| :--- | :--- | :--- |
|
|
23
|
+
| **Google Gemini** | `gemini-2.5-flash`, `gemini-2.5-pro`, `gemini-1.5-pro` | `export GEMINI_API_KEY="..."` |
|
|
24
|
+
| **Anthropic Claude** | `claude-3-5-sonnet-20241022`, `claude-3-7-sonnet`, `claude-3-opus` | `export ANTHROPIC_API_KEY="..."` |
|
|
25
|
+
| **OpenAI** | `gpt-4o`, `gpt-4o-mini`, `o1`, `o3-mini` | `export OPENAI_API_KEY="..."` |
|
|
26
|
+
| **DeepSeek** | `deepseek-chat`, `deepseek-reasoner` | `export DEEPSEEK_API_KEY="..."` |
|
|
27
|
+
| **NVIDIA Nemotron** | `nvidia/nemotron-4-340b-instruct` | `export NVIDIA_API_KEY="..."` |
|
|
28
|
+
| **OpenRouter** | 200+ models across all frontier labs | `export OPENROUTER_API_KEY="..."` |
|
|
29
|
+
| **Local / Self-Hosted** | Ollama (`llama3.2`, `mistral`, `qwen2.5`), vLLM | `--base-url http://localhost:11434/v1` |
|
|
30
|
+
|
|
31
|
+
## 🌟 Key Highlights
|
|
32
|
+
|
|
33
|
+
- **Multi-Provider Flexibility**: Switch between Gemini, Claude, GPT, DeepSeek, or local Ollama with a single flag.
|
|
34
|
+
- **Recursive Search & Discovery**: Explores multi-hop questions by identifying knowledge gaps after each crawl iteration.
|
|
35
|
+
- **Zero-Config Search**: Built-in DuckDuckGo search integration requiring zero additional API keys.
|
|
36
|
+
- **Publication Deliverables**: Exports structured Markdown, JSON intelligence dossiers, and self-contained dark-mode interactive HTML reports.
|
|
37
|
+
|
|
38
|
+
## 🚀 Quick Start
|
|
39
|
+
|
|
40
|
+
### 1. Installation
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
git clone https://github.com/shubhransh-gupta/deep-dossier.git
|
|
44
|
+
cd deep-dossier
|
|
45
|
+
pip install -e .
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### 2. Set Your Preferred API Key
|
|
49
|
+
|
|
50
|
+
Choose any provider:
|
|
51
|
+
```bash
|
|
52
|
+
# Option A: Google Gemini
|
|
53
|
+
export GEMINI_API_KEY="AIzaSy..."
|
|
54
|
+
|
|
55
|
+
# Option B: Anthropic Claude
|
|
56
|
+
export ANTHROPIC_API_KEY="sk-ant-..."
|
|
57
|
+
|
|
58
|
+
# Option C: OpenAI
|
|
59
|
+
export OPENAI_API_KEY="sk-proj-..."
|
|
60
|
+
|
|
61
|
+
# Option D: DeepSeek
|
|
62
|
+
export DEEPSEEK_API_KEY="sk-..."
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### 3. Run a Research Mission
|
|
66
|
+
|
|
67
|
+
Using default/auto-detected provider:
|
|
68
|
+
```bash
|
|
69
|
+
deep-dossier "Analyze battery degradation patterns of LFP vs NMC cells in commercial EVs"
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Explicitly choose provider and model:
|
|
73
|
+
```bash
|
|
74
|
+
# Using Anthropic Claude 3.5 Sonnet
|
|
75
|
+
deep-dossier "Solid-state battery commercial readiness" --provider anthropic --model claude-3-5-sonnet-20241022
|
|
76
|
+
|
|
77
|
+
# Using OpenAI GPT-4o
|
|
78
|
+
deep-dossier "Cross-border payment rail comparison" --provider openai --model gpt-4o
|
|
79
|
+
|
|
80
|
+
# Using Local Ollama
|
|
81
|
+
deep-dossier "Local AI agent architectures" --base-url http://localhost:11434/v1 --model llama3.2
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## ⚙️ Options
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
deep-dossier [TOPIC] [OPTIONS]
|
|
88
|
+
|
|
89
|
+
Options:
|
|
90
|
+
--provider, -p TEXT LLM provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter
|
|
91
|
+
--model, -m TEXT Specific model name
|
|
92
|
+
--base-url TEXT Custom API base URL (for Ollama, vLLM, or custom proxies)
|
|
93
|
+
--depth, -d INTEGER Recursive search depth (default: 2)
|
|
94
|
+
--breadth, -b INTEGER Number of sources to retrieve per search query (default: 4)
|
|
95
|
+
--output, -o PATH Output directory for generated dossiers (default: ./research_outputs)
|
|
96
|
+
--api-key, -k TEXT Explicit API key override
|
|
97
|
+
--no-html Disable HTML report generation
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## 📦 Project Structure
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
deep-dossier/
|
|
104
|
+
├── deep_dossier/
|
|
105
|
+
│ ├── agent.py # Core autonomous orchestrator and synthesis engine
|
|
106
|
+
│ ├── llm.py # Universal multi-provider LLM interface (Gemini/Claude/GPT/Ollama)
|
|
107
|
+
│ ├── crawler.py # Concurrent HTML & document content extractor
|
|
108
|
+
│ ├── search.py # Search engine interface
|
|
109
|
+
│ ├── report.py # Markdown and interactive HTML renderer
|
|
110
|
+
│ ├── config.py # Configuration and environment loader
|
|
111
|
+
│ └── cli.py # Rich terminal interface
|
|
112
|
+
└── pyproject.toml # Build metadata and entry points
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## 📄 License
|
|
116
|
+
|
|
117
|
+
MIT License. Copyright (c) 2026 Shubhransh Gupta.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Deep Dossier - Autonomous multi-step research agent using Google Gemini.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__author__ = "Shubhransh Gupta"
|
|
7
|
+
|
|
8
|
+
from .agent import DeepResearchAgent
|
|
9
|
+
from .config import ResearchConfig
|
|
10
|
+
|
|
11
|
+
__all__ = ["DeepResearchAgent", "ResearchConfig", "__version__"]
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core DeepResearchAgent orchestrator leveraging Gemini for autonomous multi-step research.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from typing import List, Dict, Any, Optional
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
|
|
12
|
+
from .config import ResearchConfig
|
|
13
|
+
from .search import SearchEngine, SearchResult
|
|
14
|
+
from .crawler import WebCrawler, ScrapedDocument
|
|
15
|
+
from .llm import UnifiedLLM
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DeepResearchAgent:
|
|
21
|
+
"""
|
|
22
|
+
Autonomous research agent that decomposes questions, searches the web recursively,
|
|
23
|
+
crawls deep documentation/papers, and synthesizes publication-grade reports.
|
|
24
|
+
Supports Gemini, OpenAI, Claude, DeepSeek, Nemotron, and Ollama.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, config: Optional[ResearchConfig] = None):
|
|
28
|
+
self.config = config or ResearchConfig()
|
|
29
|
+
self.config.validate()
|
|
30
|
+
|
|
31
|
+
self.search_engine = SearchEngine(timeout=self.config.timeout_seconds)
|
|
32
|
+
self.crawler = WebCrawler(
|
|
33
|
+
user_agent=self.config.user_agent,
|
|
34
|
+
timeout=self.config.timeout_seconds,
|
|
35
|
+
)
|
|
36
|
+
self.llm = UnifiedLLM(
|
|
37
|
+
provider=self.config.provider,
|
|
38
|
+
model=self.config.model,
|
|
39
|
+
api_key=self.config.api_key,
|
|
40
|
+
base_url=self.config.base_url,
|
|
41
|
+
timeout=self.config.timeout_seconds * 3,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def _call_llm(self, prompt: str, system_instruction: Optional[str] = None) -> str:
|
|
45
|
+
"""Invokes the unified multi-provider LLM."""
|
|
46
|
+
return self.llm.generate(prompt, system_instruction=system_instruction)
|
|
47
|
+
|
|
48
|
+
def research(self, topic: str, on_progress=None) -> Dict[str, Any]:
|
|
49
|
+
"""
|
|
50
|
+
Executes an end-to-end autonomous deep research workflow.
|
|
51
|
+
"""
|
|
52
|
+
def report_step(step_msg: str):
|
|
53
|
+
if on_progress:
|
|
54
|
+
on_progress(step_msg)
|
|
55
|
+
elif self.config.verbose:
|
|
56
|
+
print(f"[*] {step_msg}")
|
|
57
|
+
|
|
58
|
+
report_step(f"Starting deep research on: '{topic}'")
|
|
59
|
+
|
|
60
|
+
# Phase 1: Planning and Query Decomposition
|
|
61
|
+
report_step("Phase 1: Generating research plan and query decomposition...")
|
|
62
|
+
initial_queries = self._generate_search_queries(topic)
|
|
63
|
+
report_step(f"Generated {len(initial_queries)} research angles: {', '.join(initial_queries)}")
|
|
64
|
+
|
|
65
|
+
all_scraped_docs: Dict[str, ScrapedDocument] = {}
|
|
66
|
+
all_sources: List[Dict[str, str]] = []
|
|
67
|
+
visited_urls = set()
|
|
68
|
+
|
|
69
|
+
current_queries = initial_queries
|
|
70
|
+
|
|
71
|
+
# Phase 2: Recursive Deep Web Crawling Loop
|
|
72
|
+
for depth in range(1, self.config.max_depth + 1):
|
|
73
|
+
report_step(f"Phase 2 (Depth {depth}/{self.config.max_depth}): Searching and extracting evidence...")
|
|
74
|
+
|
|
75
|
+
new_urls = []
|
|
76
|
+
for q in current_queries:
|
|
77
|
+
search_results = self.search_engine.search(q, max_results=self.config.breadth)
|
|
78
|
+
for res in search_results:
|
|
79
|
+
if res.url not in visited_urls and len(visited_urls) < self.config.max_sources_to_scrape:
|
|
80
|
+
visited_urls.add(res.url)
|
|
81
|
+
new_urls.append(res.url)
|
|
82
|
+
all_sources.append({"title": res.title, "url": res.url, "snippet": res.snippet})
|
|
83
|
+
|
|
84
|
+
if not new_urls:
|
|
85
|
+
report_step("No new URLs found in this pass. Proceeding to synthesis...")
|
|
86
|
+
break
|
|
87
|
+
|
|
88
|
+
report_step(f"Crawling {len(new_urls)} primary sources in parallel...")
|
|
89
|
+
scraped = self.crawler.scrape_urls(new_urls)
|
|
90
|
+
for doc in scraped:
|
|
91
|
+
all_scraped_docs[doc.url] = doc
|
|
92
|
+
|
|
93
|
+
total_chars = sum(d.char_count for d in all_scraped_docs.values())
|
|
94
|
+
report_step(f"Aggregated {len(all_scraped_docs)} documents ({total_chars:,} characters of raw knowledge).")
|
|
95
|
+
|
|
96
|
+
# If more depth remaining, hypothesize next missing gaps
|
|
97
|
+
if depth < self.config.max_depth and len(visited_urls) < self.config.max_sources_to_scrape:
|
|
98
|
+
report_step("Analyzing knowledge gaps to formulate next follow-up queries...")
|
|
99
|
+
current_queries = self._generate_followup_queries(topic, list(all_scraped_docs.values()))
|
|
100
|
+
report_step(f"Follow-up queries: {', '.join(current_queries)}")
|
|
101
|
+
|
|
102
|
+
# Phase 3: Synthesis & Report Generation
|
|
103
|
+
report_step("Phase 3: Synthesizing publication-grade report with citations...")
|
|
104
|
+
markdown_report = self._synthesize_report(topic, list(all_scraped_docs.values()), all_sources)
|
|
105
|
+
|
|
106
|
+
report_step("Deep research completed successfully.")
|
|
107
|
+
|
|
108
|
+
slug = re.sub(r"[^a-zA-Z0-9_-]", "_", topic.lower())[:50].strip("_")
|
|
109
|
+
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
110
|
+
|
|
111
|
+
return {
|
|
112
|
+
"topic": topic,
|
|
113
|
+
"slug": slug,
|
|
114
|
+
"timestamp": timestamp,
|
|
115
|
+
"markdown_report": markdown_report,
|
|
116
|
+
"sources": all_sources,
|
|
117
|
+
"document_count": len(all_scraped_docs),
|
|
118
|
+
"total_chars_analyzed": sum(d.char_count for d in all_scraped_docs.values()),
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
def _generate_search_queries(self, topic: str) -> List[str]:
|
|
122
|
+
"""Prompt Gemini to break topic down into 3-4 diverse search queries."""
|
|
123
|
+
prompt = f"""
|
|
124
|
+
You are an expert investigative research director.
|
|
125
|
+
Given the research topic: "{topic}"
|
|
126
|
+
Formulate {self.config.max_queries_per_step} high-precision, diverse search engine queries.
|
|
127
|
+
Cover:
|
|
128
|
+
1. Core technical definition / state of the art.
|
|
129
|
+
2. Concrete benchmark numbers, metrics, or recent developments.
|
|
130
|
+
3. Critiques, limitations, controversies, or competitor comparisons.
|
|
131
|
+
|
|
132
|
+
Return ONLY a JSON list of query strings. Example:
|
|
133
|
+
["query 1", "query 2", "query 3"]
|
|
134
|
+
"""
|
|
135
|
+
response = self._call_llm(prompt)
|
|
136
|
+
try:
|
|
137
|
+
match = re.search(r"\[.*\]", response, re.DOTALL)
|
|
138
|
+
if match:
|
|
139
|
+
queries = json.loads(match.group(0))
|
|
140
|
+
return [str(q).strip() for q in queries if str(q).strip()][:self.config.max_queries_per_step]
|
|
141
|
+
except Exception:
|
|
142
|
+
pass
|
|
143
|
+
return [f"{topic} overview", f"{topic} latest benchmarks", f"{topic} challenges"]
|
|
144
|
+
|
|
145
|
+
def _generate_followup_queries(self, topic: str, docs: List[ScrapedDocument]) -> List[str]:
|
|
146
|
+
"""Determine what data is still ambiguous and generate next targeted queries."""
|
|
147
|
+
snippets = "\n\n".join([f"[{d.title}]: {d.text_content[:800]}" for d in docs[:5]])
|
|
148
|
+
prompt = f"""
|
|
149
|
+
Research Topic: "{topic}"
|
|
150
|
+
We have collected initial documents containing the following excerpts:
|
|
151
|
+
{snippets}
|
|
152
|
+
|
|
153
|
+
Identify the most critical missing pieces of data, contradictions, or unanswered questions.
|
|
154
|
+
Generate 2 targeted search queries to resolve these specific gaps.
|
|
155
|
+
|
|
156
|
+
Return ONLY a JSON array of query strings. Example:
|
|
157
|
+
["specific query 1", "specific query 2"]
|
|
158
|
+
"""
|
|
159
|
+
response = self._call_llm(prompt)
|
|
160
|
+
try:
|
|
161
|
+
match = re.search(r"\[.*\]", response, re.DOTALL)
|
|
162
|
+
if match:
|
|
163
|
+
queries = json.loads(match.group(0))
|
|
164
|
+
return [str(q).strip() for q in queries if str(q).strip()][:2]
|
|
165
|
+
except Exception:
|
|
166
|
+
pass
|
|
167
|
+
return [f"{topic} detailed analysis", f"{topic} case studies"]
|
|
168
|
+
|
|
169
|
+
def _synthesize_report(
|
|
170
|
+
self,
|
|
171
|
+
topic: str,
|
|
172
|
+
docs: List[ScrapedDocument],
|
|
173
|
+
sources: List[Dict[str, str]]
|
|
174
|
+
) -> str:
|
|
175
|
+
"""
|
|
176
|
+
Synthesizes an extensive, publication-grade research report.
|
|
177
|
+
Feeds the rich multi-document context to Gemini's large context window.
|
|
178
|
+
"""
|
|
179
|
+
context_blocks = []
|
|
180
|
+
for idx, doc in enumerate(docs, 1):
|
|
181
|
+
context_blocks.append(
|
|
182
|
+
f"--- SOURCE [{idx}] ---\n"
|
|
183
|
+
f"Title: {doc.title}\n"
|
|
184
|
+
f"URL: {doc.url}\n"
|
|
185
|
+
f"Content:\n{doc.text_content}\n"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
full_context = "\n\n".join(context_blocks)
|
|
189
|
+
if len(full_context) > 200000:
|
|
190
|
+
full_context = full_context[:200000] + "\n\n...[Truncated context]..."
|
|
191
|
+
|
|
192
|
+
system_instruction = (
|
|
193
|
+
"You are a world-class research scientist and principal technology analyst. "
|
|
194
|
+
"Write comprehensive, rigorous, deeply technical research reports. "
|
|
195
|
+
"Never use fluff, cliches, or shallow generalities. Use concrete figures, tables, and citations."
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
prompt = f"""
|
|
199
|
+
RESEARCH TOPIC: "{topic}"
|
|
200
|
+
|
|
201
|
+
BELOW IS THE VERIFIED RAW SOURCE MATERIAL CRAWLED FROM THE WEB:
|
|
202
|
+
{full_context}
|
|
203
|
+
|
|
204
|
+
TASK:
|
|
205
|
+
Write an authoritative, publication-ready research dossier addressing "{topic}".
|
|
206
|
+
|
|
207
|
+
STRUCTURE YOUR REPORT USING THE FOLLOWING SECTIONS:
|
|
208
|
+
1. Executive Summary (Key takeaways, core findings, bottom-line verdict)
|
|
209
|
+
2. Background & Context (Why this matters, historical evolution)
|
|
210
|
+
3. Technical Architecture & Deep Dive (Specific mechanisms, formulations, protocols)
|
|
211
|
+
4. Empirical Data & Comparative Analysis (Create clear Markdown tables comparing metrics, benchmarks, or approaches)
|
|
212
|
+
5. Critical Challenges & Limitations (What are the bottlenecks, failures, or tradeoffs?)
|
|
213
|
+
6. Future Trajectory & Strategic Implications (Where is this heading over the next 2-5 years?)
|
|
214
|
+
7. Key Takeaways & Actionable Guidance
|
|
215
|
+
|
|
216
|
+
CITATION RULES:
|
|
217
|
+
- Use in-text numeric citations like [1], [2], [3] whenever citing specific facts, numbers, or claims.
|
|
218
|
+
- At the end of the report, include a 'References' section matching the numbers to the URLs from the source material.
|
|
219
|
+
|
|
220
|
+
Ensure the report is thorough, highly informative, and reads like a premier institutional research paper.
|
|
221
|
+
"""
|
|
222
|
+
return self._call_llm(prompt, system_instruction=system_instruction)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for Open Deep Research with rich terminal styling.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import argparse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.table import Table
|
|
11
|
+
from rich.markdown import Markdown
|
|
12
|
+
|
|
13
|
+
from .config import ResearchConfig
|
|
14
|
+
from .agent import DeepResearchAgent
|
|
15
|
+
from .report import save_reports
|
|
16
|
+
|
|
17
|
+
console = Console()
|
|
18
|
+
|
|
19
|
+
BANNER = """[bold cyan]
|
|
20
|
+
___ ___ _
|
|
21
|
+
/ _ \\___ ___ ___ / _ \\___ ___ ___ (_)__ ____
|
|
22
|
+
/ // / -_) -_) _ \\ / // / _ \\(_-<-_-'/ / -_) __/
|
|
23
|
+
/____/\\__/\\__/ .__/ /____/\\___/___/___//_/\\__/_/
|
|
24
|
+
/_/
|
|
25
|
+
[/bold cyan]
|
|
26
|
+
[dim]⚡ Autonomous Multi-Step Research Agent • Gemini | Claude | GPT | DeepSeek | Nemotron[/dim]
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main():
|
|
31
|
+
parser = argparse.ArgumentParser(
|
|
32
|
+
description="Deep Dossier — Autonomous multi-step deep research agent supporting Gemini, OpenAI, Anthropic, DeepSeek, and Ollama."
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument("topic", nargs="?", help="The topic or research question to investigate.")
|
|
35
|
+
parser.add_argument("--depth", "-d", type=int, default=2, help="Research depth iterations (default: 2)")
|
|
36
|
+
parser.add_argument("--breadth", "-b", type=int, default=4, help="Breadth per search step (default: 4)")
|
|
37
|
+
parser.add_argument("--provider", "-p", type=str, default=None, help="LLM Provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter")
|
|
38
|
+
parser.add_argument("--model", "-m", type=str, default=None, help="Model identifier (e.g., gemini-2.5-flash, claude-3-5-sonnet, gpt-4o, etc.)")
|
|
39
|
+
parser.add_argument("--output", "-o", type=str, default="./research_outputs", help="Output directory")
|
|
40
|
+
parser.add_argument("--api-key", "-k", type=str, default=None, help="API Key override")
|
|
41
|
+
parser.add_argument("--base-url", type=str, default=None, help="Custom API Base URL (for Ollama, vLLM, or self-hosted endpoints)")
|
|
42
|
+
parser.add_argument("--no-html", action="store_true", help="Disable HTML report generation")
|
|
43
|
+
|
|
44
|
+
args = parser.parse_args()
|
|
45
|
+
|
|
46
|
+
console.print(BANNER)
|
|
47
|
+
|
|
48
|
+
topic = args.topic
|
|
49
|
+
if not topic:
|
|
50
|
+
topic = console.input("[bold yellow]Enter your research topic/question: [/bold yellow]").strip()
|
|
51
|
+
|
|
52
|
+
if not topic:
|
|
53
|
+
console.print("[red]Error: Research topic cannot be empty.[/red]")
|
|
54
|
+
sys.exit(1)
|
|
55
|
+
|
|
56
|
+
config = ResearchConfig(
|
|
57
|
+
provider=args.provider,
|
|
58
|
+
model=args.model,
|
|
59
|
+
api_key=args.api_key,
|
|
60
|
+
base_url=args.base_url,
|
|
61
|
+
max_depth=args.depth,
|
|
62
|
+
breadth=args.breadth,
|
|
63
|
+
output_dir=Path(args.output),
|
|
64
|
+
generate_html=not args.no_html,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
config.validate()
|
|
69
|
+
except ValueError as e:
|
|
70
|
+
console.print(Panel(f"[bold red]Configuration Error:[/bold red]\n{str(e)}", border_style="red"))
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
agent = DeepResearchAgent(config=config)
|
|
74
|
+
|
|
75
|
+
console.print(Panel(
|
|
76
|
+
f"[bold white]Topic:[/bold white] {topic}\n"
|
|
77
|
+
f"[bold white]Provider:[/bold white] {agent.llm.provider.upper()} | [bold white]Model:[/bold white] {agent.llm.model}\n"
|
|
78
|
+
f"[bold white]Depth:[/bold white] {config.max_depth} | [bold white]Breadth:[/bold white] {config.breadth}",
|
|
79
|
+
title="[bold green]Research Mission Activated[/bold green]",
|
|
80
|
+
border_style="cyan"
|
|
81
|
+
))
|
|
82
|
+
|
|
83
|
+
with console.status("[bold green]Agent orchestrating deep research...[/bold green]", spinner="dots") as status:
|
|
84
|
+
def on_progress(msg: str):
|
|
85
|
+
status.update(f"[bold cyan]{msg}[/bold cyan]")
|
|
86
|
+
console.print(f" [dim]•[/dim] {msg}")
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
result = agent.research(topic=topic, on_progress=on_progress)
|
|
90
|
+
except Exception as e:
|
|
91
|
+
console.print(f"\n[bold red]Error during research execution:[/bold red] {e}")
|
|
92
|
+
sys.exit(1)
|
|
93
|
+
|
|
94
|
+
# Save reports
|
|
95
|
+
paths = save_reports(result, config.output_dir, result["slug"])
|
|
96
|
+
|
|
97
|
+
# Output Summary Table
|
|
98
|
+
table = Table(title="Generated Research Deliverables", border_style="green")
|
|
99
|
+
table.add_column("Format", style="cyan", no_wrap=True)
|
|
100
|
+
table.add_column("Location", style="white")
|
|
101
|
+
|
|
102
|
+
if "markdown" in paths:
|
|
103
|
+
table.add_row("Markdown (.md)", str(paths["markdown"].resolve()))
|
|
104
|
+
if "html" in paths:
|
|
105
|
+
table.add_row("Interactive HTML", str(paths["html"].resolve()))
|
|
106
|
+
if "json" in paths:
|
|
107
|
+
table.add_row("JSON Dossier", str(paths["json"].resolve()))
|
|
108
|
+
|
|
109
|
+
console.print("\n")
|
|
110
|
+
console.print(table)
|
|
111
|
+
console.print(f"\n[bold green]✨ Research Complete![/bold green] Analyzed [bold]{result['document_count']}[/bold] sources ({result['total_chars_analyzed']:,} characters).")
|
|
112
|
+
console.print(f"To view HTML report in browser: [bold cyan]open {paths.get('html')}[/bold cyan]\n")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
main()
|