deep-dossier 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deep_dossier/__init__.py +11 -0
- deep_dossier/agent.py +222 -0
- deep_dossier/cli.py +116 -0
- deep_dossier/config.py +85 -0
- deep_dossier/crawler.py +145 -0
- deep_dossier/llm.py +188 -0
- deep_dossier/report.py +274 -0
- deep_dossier/search.py +97 -0
- deep_dossier-0.1.0.dist-info/METADATA +144 -0
- deep_dossier-0.1.0.dist-info/RECORD +13 -0
- deep_dossier-0.1.0.dist-info/WHEEL +5 -0
- deep_dossier-0.1.0.dist-info/entry_points.txt +2 -0
- deep_dossier-0.1.0.dist-info/top_level.txt +1 -0
deep_dossier/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Deep Dossier - Autonomous multi-step research agent using Google Gemini.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__author__ = "Shubhransh Gupta"
|
|
7
|
+
|
|
8
|
+
from .agent import DeepResearchAgent
|
|
9
|
+
from .config import ResearchConfig
|
|
10
|
+
|
|
11
|
+
__all__ = ["DeepResearchAgent", "ResearchConfig", "__version__"]
|
deep_dossier/agent.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core DeepResearchAgent orchestrator leveraging Gemini for autonomous multi-step research.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from typing import List, Dict, Any, Optional
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
|
|
12
|
+
from .config import ResearchConfig
|
|
13
|
+
from .search import SearchEngine, SearchResult
|
|
14
|
+
from .crawler import WebCrawler, ScrapedDocument
|
|
15
|
+
from .llm import UnifiedLLM
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DeepResearchAgent:
|
|
21
|
+
"""
|
|
22
|
+
Autonomous research agent that decomposes questions, searches the web recursively,
|
|
23
|
+
crawls deep documentation/papers, and synthesizes publication-grade reports.
|
|
24
|
+
Supports Gemini, OpenAI, Claude, DeepSeek, Nemotron, and Ollama.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, config: Optional[ResearchConfig] = None):
|
|
28
|
+
self.config = config or ResearchConfig()
|
|
29
|
+
self.config.validate()
|
|
30
|
+
|
|
31
|
+
self.search_engine = SearchEngine(timeout=self.config.timeout_seconds)
|
|
32
|
+
self.crawler = WebCrawler(
|
|
33
|
+
user_agent=self.config.user_agent,
|
|
34
|
+
timeout=self.config.timeout_seconds,
|
|
35
|
+
)
|
|
36
|
+
self.llm = UnifiedLLM(
|
|
37
|
+
provider=self.config.provider,
|
|
38
|
+
model=self.config.model,
|
|
39
|
+
api_key=self.config.api_key,
|
|
40
|
+
base_url=self.config.base_url,
|
|
41
|
+
timeout=self.config.timeout_seconds * 3,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def _call_llm(self, prompt: str, system_instruction: Optional[str] = None) -> str:
|
|
45
|
+
"""Invokes the unified multi-provider LLM."""
|
|
46
|
+
return self.llm.generate(prompt, system_instruction=system_instruction)
|
|
47
|
+
|
|
48
|
+
def research(self, topic: str, on_progress=None) -> Dict[str, Any]:
|
|
49
|
+
"""
|
|
50
|
+
Executes an end-to-end autonomous deep research workflow.
|
|
51
|
+
"""
|
|
52
|
+
def report_step(step_msg: str):
|
|
53
|
+
if on_progress:
|
|
54
|
+
on_progress(step_msg)
|
|
55
|
+
elif self.config.verbose:
|
|
56
|
+
print(f"[*] {step_msg}")
|
|
57
|
+
|
|
58
|
+
report_step(f"Starting deep research on: '{topic}'")
|
|
59
|
+
|
|
60
|
+
# Phase 1: Planning and Query Decomposition
|
|
61
|
+
report_step("Phase 1: Generating research plan and query decomposition...")
|
|
62
|
+
initial_queries = self._generate_search_queries(topic)
|
|
63
|
+
report_step(f"Generated {len(initial_queries)} research angles: {', '.join(initial_queries)}")
|
|
64
|
+
|
|
65
|
+
all_scraped_docs: Dict[str, ScrapedDocument] = {}
|
|
66
|
+
all_sources: List[Dict[str, str]] = []
|
|
67
|
+
visited_urls = set()
|
|
68
|
+
|
|
69
|
+
current_queries = initial_queries
|
|
70
|
+
|
|
71
|
+
# Phase 2: Recursive Deep Web Crawling Loop
|
|
72
|
+
for depth in range(1, self.config.max_depth + 1):
|
|
73
|
+
report_step(f"Phase 2 (Depth {depth}/{self.config.max_depth}): Searching and extracting evidence...")
|
|
74
|
+
|
|
75
|
+
new_urls = []
|
|
76
|
+
for q in current_queries:
|
|
77
|
+
search_results = self.search_engine.search(q, max_results=self.config.breadth)
|
|
78
|
+
for res in search_results:
|
|
79
|
+
if res.url not in visited_urls and len(visited_urls) < self.config.max_sources_to_scrape:
|
|
80
|
+
visited_urls.add(res.url)
|
|
81
|
+
new_urls.append(res.url)
|
|
82
|
+
all_sources.append({"title": res.title, "url": res.url, "snippet": res.snippet})
|
|
83
|
+
|
|
84
|
+
if not new_urls:
|
|
85
|
+
report_step("No new URLs found in this pass. Proceeding to synthesis...")
|
|
86
|
+
break
|
|
87
|
+
|
|
88
|
+
report_step(f"Crawling {len(new_urls)} primary sources in parallel...")
|
|
89
|
+
scraped = self.crawler.scrape_urls(new_urls)
|
|
90
|
+
for doc in scraped:
|
|
91
|
+
all_scraped_docs[doc.url] = doc
|
|
92
|
+
|
|
93
|
+
total_chars = sum(d.char_count for d in all_scraped_docs.values())
|
|
94
|
+
report_step(f"Aggregated {len(all_scraped_docs)} documents ({total_chars:,} characters of raw knowledge).")
|
|
95
|
+
|
|
96
|
+
# If more depth remaining, hypothesize next missing gaps
|
|
97
|
+
if depth < self.config.max_depth and len(visited_urls) < self.config.max_sources_to_scrape:
|
|
98
|
+
report_step("Analyzing knowledge gaps to formulate next follow-up queries...")
|
|
99
|
+
current_queries = self._generate_followup_queries(topic, list(all_scraped_docs.values()))
|
|
100
|
+
report_step(f"Follow-up queries: {', '.join(current_queries)}")
|
|
101
|
+
|
|
102
|
+
# Phase 3: Synthesis & Report Generation
|
|
103
|
+
report_step("Phase 3: Synthesizing publication-grade report with citations...")
|
|
104
|
+
markdown_report = self._synthesize_report(topic, list(all_scraped_docs.values()), all_sources)
|
|
105
|
+
|
|
106
|
+
report_step("Deep research completed successfully.")
|
|
107
|
+
|
|
108
|
+
slug = re.sub(r"[^a-zA-Z0-9_-]", "_", topic.lower())[:50].strip("_")
|
|
109
|
+
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
110
|
+
|
|
111
|
+
return {
|
|
112
|
+
"topic": topic,
|
|
113
|
+
"slug": slug,
|
|
114
|
+
"timestamp": timestamp,
|
|
115
|
+
"markdown_report": markdown_report,
|
|
116
|
+
"sources": all_sources,
|
|
117
|
+
"document_count": len(all_scraped_docs),
|
|
118
|
+
"total_chars_analyzed": sum(d.char_count for d in all_scraped_docs.values()),
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
def _generate_search_queries(self, topic: str) -> List[str]:
|
|
122
|
+
"""Prompt Gemini to break topic down into 3-4 diverse search queries."""
|
|
123
|
+
prompt = f"""
|
|
124
|
+
You are an expert investigative research director.
|
|
125
|
+
Given the research topic: "{topic}"
|
|
126
|
+
Formulate {self.config.max_queries_per_step} high-precision, diverse search engine queries.
|
|
127
|
+
Cover:
|
|
128
|
+
1. Core technical definition / state of the art.
|
|
129
|
+
2. Concrete benchmark numbers, metrics, or recent developments.
|
|
130
|
+
3. Critiques, limitations, controversies, or competitor comparisons.
|
|
131
|
+
|
|
132
|
+
Return ONLY a JSON list of query strings. Example:
|
|
133
|
+
["query 1", "query 2", "query 3"]
|
|
134
|
+
"""
|
|
135
|
+
response = self._call_llm(prompt)
|
|
136
|
+
try:
|
|
137
|
+
match = re.search(r"\[.*\]", response, re.DOTALL)
|
|
138
|
+
if match:
|
|
139
|
+
queries = json.loads(match.group(0))
|
|
140
|
+
return [str(q).strip() for q in queries if str(q).strip()][:self.config.max_queries_per_step]
|
|
141
|
+
except Exception:
|
|
142
|
+
pass
|
|
143
|
+
return [f"{topic} overview", f"{topic} latest benchmarks", f"{topic} challenges"]
|
|
144
|
+
|
|
145
|
+
def _generate_followup_queries(self, topic: str, docs: List[ScrapedDocument]) -> List[str]:
|
|
146
|
+
"""Determine what data is still ambiguous and generate next targeted queries."""
|
|
147
|
+
snippets = "\n\n".join([f"[{d.title}]: {d.text_content[:800]}" for d in docs[:5]])
|
|
148
|
+
prompt = f"""
|
|
149
|
+
Research Topic: "{topic}"
|
|
150
|
+
We have collected initial documents containing the following excerpts:
|
|
151
|
+
{snippets}
|
|
152
|
+
|
|
153
|
+
Identify the most critical missing pieces of data, contradictions, or unanswered questions.
|
|
154
|
+
Generate 2 targeted search queries to resolve these specific gaps.
|
|
155
|
+
|
|
156
|
+
Return ONLY a JSON array of query strings. Example:
|
|
157
|
+
["specific query 1", "specific query 2"]
|
|
158
|
+
"""
|
|
159
|
+
response = self._call_llm(prompt)
|
|
160
|
+
try:
|
|
161
|
+
match = re.search(r"\[.*\]", response, re.DOTALL)
|
|
162
|
+
if match:
|
|
163
|
+
queries = json.loads(match.group(0))
|
|
164
|
+
return [str(q).strip() for q in queries if str(q).strip()][:2]
|
|
165
|
+
except Exception:
|
|
166
|
+
pass
|
|
167
|
+
return [f"{topic} detailed analysis", f"{topic} case studies"]
|
|
168
|
+
|
|
169
|
+
def _synthesize_report(
|
|
170
|
+
self,
|
|
171
|
+
topic: str,
|
|
172
|
+
docs: List[ScrapedDocument],
|
|
173
|
+
sources: List[Dict[str, str]]
|
|
174
|
+
) -> str:
|
|
175
|
+
"""
|
|
176
|
+
Synthesizes an extensive, publication-grade research report.
|
|
177
|
+
Feeds the rich multi-document context to Gemini's large context window.
|
|
178
|
+
"""
|
|
179
|
+
context_blocks = []
|
|
180
|
+
for idx, doc in enumerate(docs, 1):
|
|
181
|
+
context_blocks.append(
|
|
182
|
+
f"--- SOURCE [{idx}] ---\n"
|
|
183
|
+
f"Title: {doc.title}\n"
|
|
184
|
+
f"URL: {doc.url}\n"
|
|
185
|
+
f"Content:\n{doc.text_content}\n"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
full_context = "\n\n".join(context_blocks)
|
|
189
|
+
if len(full_context) > 200000:
|
|
190
|
+
full_context = full_context[:200000] + "\n\n...[Truncated context]..."
|
|
191
|
+
|
|
192
|
+
system_instruction = (
|
|
193
|
+
"You are a world-class research scientist and principal technology analyst. "
|
|
194
|
+
"Write comprehensive, rigorous, deeply technical research reports. "
|
|
195
|
+
"Never use fluff, cliches, or shallow generalities. Use concrete figures, tables, and citations."
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
prompt = f"""
|
|
199
|
+
RESEARCH TOPIC: "{topic}"
|
|
200
|
+
|
|
201
|
+
BELOW IS THE VERIFIED RAW SOURCE MATERIAL CRAWLED FROM THE WEB:
|
|
202
|
+
{full_context}
|
|
203
|
+
|
|
204
|
+
TASK:
|
|
205
|
+
Write an authoritative, publication-ready research dossier addressing "{topic}".
|
|
206
|
+
|
|
207
|
+
STRUCTURE YOUR REPORT USING THE FOLLOWING SECTIONS:
|
|
208
|
+
1. Executive Summary (Key takeaways, core findings, bottom-line verdict)
|
|
209
|
+
2. Background & Context (Why this matters, historical evolution)
|
|
210
|
+
3. Technical Architecture & Deep Dive (Specific mechanisms, formulations, protocols)
|
|
211
|
+
4. Empirical Data & Comparative Analysis (Create clear Markdown tables comparing metrics, benchmarks, or approaches)
|
|
212
|
+
5. Critical Challenges & Limitations (What are the bottlenecks, failures, or tradeoffs?)
|
|
213
|
+
6. Future Trajectory & Strategic Implications (Where is this heading over the next 2-5 years?)
|
|
214
|
+
7. Key Takeaways & Actionable Guidance
|
|
215
|
+
|
|
216
|
+
CITATION RULES:
|
|
217
|
+
- Use in-text numeric citations like [1], [2], [3] whenever citing specific facts, numbers, or claims.
|
|
218
|
+
- At the end of the report, include a 'References' section matching the numbers to the URLs from the source material.
|
|
219
|
+
|
|
220
|
+
Ensure the report is thorough, highly informative, and reads like a premier institutional research paper.
|
|
221
|
+
"""
|
|
222
|
+
return self._call_llm(prompt, system_instruction=system_instruction)
|
deep_dossier/cli.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for Open Deep Research with rich terminal styling.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import argparse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.table import Table
|
|
11
|
+
from rich.markdown import Markdown
|
|
12
|
+
|
|
13
|
+
from .config import ResearchConfig
|
|
14
|
+
from .agent import DeepResearchAgent
|
|
15
|
+
from .report import save_reports
|
|
16
|
+
|
|
17
|
+
console = Console()
|
|
18
|
+
|
|
19
|
+
BANNER = """[bold cyan]
|
|
20
|
+
___ ___ _
|
|
21
|
+
/ _ \\___ ___ ___ / _ \\___ ___ ___ (_)__ ____
|
|
22
|
+
/ // / -_) -_) _ \\ / // / _ \\(_-<-_-'/ / -_) __/
|
|
23
|
+
/____/\\__/\\__/ .__/ /____/\\___/___/___//_/\\__/_/
|
|
24
|
+
/_/
|
|
25
|
+
[/bold cyan]
|
|
26
|
+
[dim]⚡ Autonomous Multi-Step Research Agent • Gemini | Claude | GPT | DeepSeek | Nemotron[/dim]
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main():
|
|
31
|
+
parser = argparse.ArgumentParser(
|
|
32
|
+
description="Deep Dossier — Autonomous multi-step deep research agent supporting Gemini, OpenAI, Anthropic, DeepSeek, and Ollama."
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument("topic", nargs="?", help="The topic or research question to investigate.")
|
|
35
|
+
parser.add_argument("--depth", "-d", type=int, default=2, help="Research depth iterations (default: 2)")
|
|
36
|
+
parser.add_argument("--breadth", "-b", type=int, default=4, help="Breadth per search step (default: 4)")
|
|
37
|
+
parser.add_argument("--provider", "-p", type=str, default=None, help="LLM Provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter")
|
|
38
|
+
parser.add_argument("--model", "-m", type=str, default=None, help="Model identifier (e.g., gemini-2.5-flash, claude-3-5-sonnet, gpt-4o, etc.)")
|
|
39
|
+
parser.add_argument("--output", "-o", type=str, default="./research_outputs", help="Output directory")
|
|
40
|
+
parser.add_argument("--api-key", "-k", type=str, default=None, help="API Key override")
|
|
41
|
+
parser.add_argument("--base-url", type=str, default=None, help="Custom API Base URL (for Ollama, vLLM, or self-hosted endpoints)")
|
|
42
|
+
parser.add_argument("--no-html", action="store_true", help="Disable HTML report generation")
|
|
43
|
+
|
|
44
|
+
args = parser.parse_args()
|
|
45
|
+
|
|
46
|
+
console.print(BANNER)
|
|
47
|
+
|
|
48
|
+
topic = args.topic
|
|
49
|
+
if not topic:
|
|
50
|
+
topic = console.input("[bold yellow]Enter your research topic/question: [/bold yellow]").strip()
|
|
51
|
+
|
|
52
|
+
if not topic:
|
|
53
|
+
console.print("[red]Error: Research topic cannot be empty.[/red]")
|
|
54
|
+
sys.exit(1)
|
|
55
|
+
|
|
56
|
+
config = ResearchConfig(
|
|
57
|
+
provider=args.provider,
|
|
58
|
+
model=args.model,
|
|
59
|
+
api_key=args.api_key,
|
|
60
|
+
base_url=args.base_url,
|
|
61
|
+
max_depth=args.depth,
|
|
62
|
+
breadth=args.breadth,
|
|
63
|
+
output_dir=Path(args.output),
|
|
64
|
+
generate_html=not args.no_html,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
config.validate()
|
|
69
|
+
except ValueError as e:
|
|
70
|
+
console.print(Panel(f"[bold red]Configuration Error:[/bold red]\n{str(e)}", border_style="red"))
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
agent = DeepResearchAgent(config=config)
|
|
74
|
+
|
|
75
|
+
console.print(Panel(
|
|
76
|
+
f"[bold white]Topic:[/bold white] {topic}\n"
|
|
77
|
+
f"[bold white]Provider:[/bold white] {agent.llm.provider.upper()} | [bold white]Model:[/bold white] {agent.llm.model}\n"
|
|
78
|
+
f"[bold white]Depth:[/bold white] {config.max_depth} | [bold white]Breadth:[/bold white] {config.breadth}",
|
|
79
|
+
title="[bold green]Research Mission Activated[/bold green]",
|
|
80
|
+
border_style="cyan"
|
|
81
|
+
))
|
|
82
|
+
|
|
83
|
+
with console.status("[bold green]Agent orchestrating deep research...[/bold green]", spinner="dots") as status:
|
|
84
|
+
def on_progress(msg: str):
|
|
85
|
+
status.update(f"[bold cyan]{msg}[/bold cyan]")
|
|
86
|
+
console.print(f" [dim]•[/dim] {msg}")
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
result = agent.research(topic=topic, on_progress=on_progress)
|
|
90
|
+
except Exception as e:
|
|
91
|
+
console.print(f"\n[bold red]Error during research execution:[/bold red] {e}")
|
|
92
|
+
sys.exit(1)
|
|
93
|
+
|
|
94
|
+
# Save reports
|
|
95
|
+
paths = save_reports(result, config.output_dir, result["slug"])
|
|
96
|
+
|
|
97
|
+
# Output Summary Table
|
|
98
|
+
table = Table(title="Generated Research Deliverables", border_style="green")
|
|
99
|
+
table.add_column("Format", style="cyan", no_wrap=True)
|
|
100
|
+
table.add_column("Location", style="white")
|
|
101
|
+
|
|
102
|
+
if "markdown" in paths:
|
|
103
|
+
table.add_row("Markdown (.md)", str(paths["markdown"].resolve()))
|
|
104
|
+
if "html" in paths:
|
|
105
|
+
table.add_row("Interactive HTML", str(paths["html"].resolve()))
|
|
106
|
+
if "json" in paths:
|
|
107
|
+
table.add_row("JSON Dossier", str(paths["json"].resolve()))
|
|
108
|
+
|
|
109
|
+
console.print("\n")
|
|
110
|
+
console.print(table)
|
|
111
|
+
console.print(f"\n[bold green]✨ Research Complete![/bold green] Analyzed [bold]{result['document_count']}[/bold] sources ({result['total_chars_analyzed']:,} characters).")
|
|
112
|
+
console.print(f"To view HTML report in browser: [bold cyan]open {paths.get('html')}[/bold cyan]\n")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
main()
|
deep_dossier/config.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configuration settings and environment management for Deep Dossier.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Optional, List
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_env_file(dotenv_path: Optional[str] = None):
|
|
12
|
+
"""Simple robust .env loader."""
|
|
13
|
+
paths_to_check = [
|
|
14
|
+
dotenv_path,
|
|
15
|
+
Path.cwd() / ".env",
|
|
16
|
+
Path.cwd() / ".env.local",
|
|
17
|
+
Path.home() / ".config" / "deep-dossier" / ".env",
|
|
18
|
+
]
|
|
19
|
+
for p in paths_to_check:
|
|
20
|
+
if p and Path(p).is_file():
|
|
21
|
+
try:
|
|
22
|
+
with open(p, "r", encoding="utf-8") as f:
|
|
23
|
+
for line in f:
|
|
24
|
+
line = line.strip()
|
|
25
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
26
|
+
continue
|
|
27
|
+
k, v = line.split("=", 1)
|
|
28
|
+
k = k.strip()
|
|
29
|
+
v = v.strip().strip("'\"")
|
|
30
|
+
if k and k not in os.environ:
|
|
31
|
+
os.environ[k] = v
|
|
32
|
+
break
|
|
33
|
+
except Exception:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
load_env_file()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class ResearchConfig:
|
|
42
|
+
"""Config options for the Deep Dossier Agent supporting multiple LLM providers."""
|
|
43
|
+
provider: Optional[str] = None # 'gemini', 'openai', 'anthropic', 'deepseek', 'nvidia', 'ollama', 'openrouter'
|
|
44
|
+
model: Optional[str] = None
|
|
45
|
+
api_key: Optional[str] = None
|
|
46
|
+
base_url: Optional[str] = None
|
|
47
|
+
max_depth: int = 2
|
|
48
|
+
breadth: int = 4
|
|
49
|
+
max_queries_per_step: int = 3
|
|
50
|
+
max_sources_to_scrape: int = 15
|
|
51
|
+
timeout_seconds: int = 20
|
|
52
|
+
user_agent: str = (
|
|
53
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
|
54
|
+
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 (DeepDossier/0.1.0)"
|
|
55
|
+
)
|
|
56
|
+
output_dir: Path = field(default_factory=lambda: Path.cwd() / "research_outputs")
|
|
57
|
+
generate_html: bool = True
|
|
58
|
+
generate_markdown: bool = True
|
|
59
|
+
generate_json: bool = True
|
|
60
|
+
verbose: bool = True
|
|
61
|
+
|
|
62
|
+
def validate(self):
|
|
63
|
+
"""Validates that at least one provider key or base_url is accessible."""
|
|
64
|
+
available_keys = [
|
|
65
|
+
self.api_key,
|
|
66
|
+
os.getenv("GEMINI_API_KEY"),
|
|
67
|
+
os.getenv("OPENAI_API_KEY"),
|
|
68
|
+
os.getenv("ANTHROPIC_API_KEY"),
|
|
69
|
+
os.getenv("OPENROUTER_API_KEY"),
|
|
70
|
+
os.getenv("DEEPSEEK_API_KEY"),
|
|
71
|
+
os.getenv("NVIDIA_API_KEY"),
|
|
72
|
+
self.base_url,
|
|
73
|
+
]
|
|
74
|
+
if not any(available_keys):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
"No API Key found.\n"
|
|
77
|
+
"Please set one of the following environment variables:\n"
|
|
78
|
+
" • export GEMINI_API_KEY='...'\n"
|
|
79
|
+
" • export OPENAI_API_KEY='...'\n"
|
|
80
|
+
" • export ANTHROPIC_API_KEY='...'\n"
|
|
81
|
+
" • export OPENROUTER_API_KEY='...'\n"
|
|
82
|
+
" • export DEEPSEEK_API_KEY='...'\n"
|
|
83
|
+
" • export NVIDIA_API_KEY='...'\n"
|
|
84
|
+
"Or specify --api-key / --base-url (for local Ollama/vLLM)."
|
|
85
|
+
)
|
deep_dossier/crawler.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Web crawler and content extractor for deep document analysis.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Optional, List, Dict
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
10
|
+
import requests
|
|
11
|
+
from bs4 import BeautifulSoup
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class ScrapedDocument:
|
|
18
|
+
url: str
|
|
19
|
+
title: str
|
|
20
|
+
text_content: str
|
|
21
|
+
char_count: int
|
|
22
|
+
success: bool
|
|
23
|
+
error: Optional[str] = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class WebCrawler:
|
|
27
|
+
"""Extracts clean article content from URLs in parallel."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, user_agent: str, timeout: int = 15, max_workers: int = 5):
|
|
30
|
+
self.user_agent = user_agent
|
|
31
|
+
self.timeout = timeout
|
|
32
|
+
self.max_workers = max_workers
|
|
33
|
+
self.session = requests.Session()
|
|
34
|
+
self.session.headers.update({
|
|
35
|
+
"User-Agent": self.user_agent,
|
|
36
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
37
|
+
"Accept-Language": "en-US,en;q=0.5",
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
def scrape_url(self, url: str) -> ScrapedDocument:
|
|
41
|
+
"""Fetch and extract readable text from a single webpage."""
|
|
42
|
+
try:
|
|
43
|
+
resp = self.session.get(url, timeout=self.timeout, allow_redirects=True)
|
|
44
|
+
resp.raise_for_status()
|
|
45
|
+
|
|
46
|
+
# Check content type
|
|
47
|
+
content_type = resp.headers.get("Content-Type", "").lower()
|
|
48
|
+
if "pdf" in content_type or url.lower().endswith(".pdf"):
|
|
49
|
+
return self._extract_pdf(resp.content, url)
|
|
50
|
+
|
|
51
|
+
return self._extract_html(resp.text, url)
|
|
52
|
+
except Exception as e:
|
|
53
|
+
logger.debug(f"Failed to crawl {url}: {e}")
|
|
54
|
+
return ScrapedDocument(
|
|
55
|
+
url=url,
|
|
56
|
+
title="",
|
|
57
|
+
text_content="",
|
|
58
|
+
char_count=0,
|
|
59
|
+
success=False,
|
|
60
|
+
error=str(e),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def scrape_urls(self, urls: List[str]) -> List[ScrapedDocument]:
|
|
64
|
+
"""Crawl multiple URLs concurrently."""
|
|
65
|
+
results: List[ScrapedDocument] = []
|
|
66
|
+
unique_urls = list(dict.fromkeys(urls)) # Preserve order, deduplicate
|
|
67
|
+
|
|
68
|
+
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
69
|
+
future_to_url = {executor.submit(self.scrape_url, u): u for u in unique_urls}
|
|
70
|
+
for future in as_completed(future_to_url):
|
|
71
|
+
res = future.result()
|
|
72
|
+
if res.success and res.char_count > 100:
|
|
73
|
+
results.append(res)
|
|
74
|
+
return results
|
|
75
|
+
|
|
76
|
+
def _extract_html(self, html: str, url: str) -> ScrapedDocument:
|
|
77
|
+
"""Parse HTML, eliminate clutter, and extract dense readable text."""
|
|
78
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
79
|
+
|
|
80
|
+
# Extract title
|
|
81
|
+
title = ""
|
|
82
|
+
if soup.title and soup.title.string:
|
|
83
|
+
title = soup.title.string.strip()
|
|
84
|
+
elif soup.find("h1"):
|
|
85
|
+
title = soup.find("h1").get_text(strip=True)
|
|
86
|
+
|
|
87
|
+
# Remove clutter
|
|
88
|
+
for unwanted in soup(["script", "style", "nav", "footer", "header", "noscript", "svg", "aside"]):
|
|
89
|
+
unwanted.decompose()
|
|
90
|
+
|
|
91
|
+
# Target main content containers if present
|
|
92
|
+
main_content = (
|
|
93
|
+
soup.find("article")
|
|
94
|
+
or soup.find("main")
|
|
95
|
+
or soup.find("div", {"id": re.compile(r"(content|main|article)", re.I)})
|
|
96
|
+
or soup.find("div", {"class": re.compile(r"(content|main|article|post)", re.I)})
|
|
97
|
+
or soup.body
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if not main_content:
|
|
101
|
+
text = soup.get_text(separator="\n", strip=True)
|
|
102
|
+
else:
|
|
103
|
+
text = main_content.get_text(separator="\n", strip=True)
|
|
104
|
+
|
|
105
|
+
# Normalize extra blank lines
|
|
106
|
+
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
107
|
+
cleaned_text = "\n\n".join(lines)
|
|
108
|
+
|
|
109
|
+
# Cap text length to avoid token explosion (e.g. 50,000 characters per document)
|
|
110
|
+
if len(cleaned_text) > 60000:
|
|
111
|
+
cleaned_text = cleaned_text[:60000] + "\n...[truncated for length]..."
|
|
112
|
+
|
|
113
|
+
return ScrapedDocument(
|
|
114
|
+
url=url,
|
|
115
|
+
title=title or url,
|
|
116
|
+
text_content=cleaned_text,
|
|
117
|
+
char_count=len(cleaned_text),
|
|
118
|
+
success=True,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def _extract_pdf(self, content: bytes, url: str) -> ScrapedDocument:
|
|
122
|
+
"""Extract text from PDF if pypdf or PyPDF2 is available, otherwise basic raw fallback."""
|
|
123
|
+
try:
|
|
124
|
+
import io
|
|
125
|
+
import pypdf
|
|
126
|
+
reader = pypdf.PdfReader(io.BytesIO(content))
|
|
127
|
+
text = ""
|
|
128
|
+
for page in reader.pages[:20]: # First 20 pages max
|
|
129
|
+
text += page.extract_text() + "\n"
|
|
130
|
+
return ScrapedDocument(
|
|
131
|
+
url=url,
|
|
132
|
+
title=f"PDF Document: {url.split('/')[-1]}",
|
|
133
|
+
text_content=text.strip(),
|
|
134
|
+
char_count=len(text),
|
|
135
|
+
success=True,
|
|
136
|
+
)
|
|
137
|
+
except Exception:
|
|
138
|
+
return ScrapedDocument(
|
|
139
|
+
url=url,
|
|
140
|
+
title=f"PDF: {url.split('/')[-1]}",
|
|
141
|
+
text_content="[Binary PDF content could not be parsed without pypdf]",
|
|
142
|
+
char_count=0,
|
|
143
|
+
success=False,
|
|
144
|
+
error="pypdf not installed",
|
|
145
|
+
)
|
deep_dossier/llm.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unified multi-provider LLM interface supporting Gemini, OpenAI, Anthropic, DeepSeek, Nemotron, and Ollama.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
from typing import Optional, Dict, Any, List
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class UnifiedLLM:
|
|
15
|
+
"""
|
|
16
|
+
Universal LLM client that routes requests to:
|
|
17
|
+
- Google Gemini
|
|
18
|
+
- OpenAI (GPT-4o, o1, o3-mini, etc.)
|
|
19
|
+
- Anthropic (Claude 3.5 Sonnet, Claude 3.7 Sonnet, etc.)
|
|
20
|
+
- OpenAI-Compatible endpoints (OpenRouter, DeepSeek, NVIDIA Nemotron, Ollama, vLLM)
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
provider: Optional[str] = None,
|
|
26
|
+
model: Optional[str] = None,
|
|
27
|
+
api_key: Optional[str] = None,
|
|
28
|
+
base_url: Optional[str] = None,
|
|
29
|
+
timeout: int = 60,
|
|
30
|
+
):
|
|
31
|
+
self.api_key = api_key
|
|
32
|
+
self.base_url = base_url
|
|
33
|
+
self.timeout = timeout
|
|
34
|
+
self.provider, self.model = self._resolve_provider_and_model(provider, model)
|
|
35
|
+
self._resolve_api_key()
|
|
36
|
+
|
|
37
|
+
def _resolve_provider_and_model(self, provider: Optional[str], model: Optional[str]) -> tuple:
|
|
38
|
+
"""Auto-detect provider based on model name or available environment variables."""
|
|
39
|
+
if provider:
|
|
40
|
+
p = provider.lower()
|
|
41
|
+
m = model or self._default_model_for_provider(p)
|
|
42
|
+
return p, m
|
|
43
|
+
|
|
44
|
+
m = (model or "").lower()
|
|
45
|
+
if "claude" in m:
|
|
46
|
+
return "anthropic", model
|
|
47
|
+
if any(x in m for x in ["gpt", "o1", "o3", "chatgpt"]):
|
|
48
|
+
return "openai", model
|
|
49
|
+
if "gemini" in m:
|
|
50
|
+
return "gemini", model
|
|
51
|
+
if "deepseek" in m:
|
|
52
|
+
return "deepseek", model
|
|
53
|
+
if "nemotron" in m or "nvidia" in m:
|
|
54
|
+
return "nvidia", model
|
|
55
|
+
|
|
56
|
+
# Check environment variable presence
|
|
57
|
+
if os.getenv("GEMINI_API_KEY"):
|
|
58
|
+
return "gemini", model or "gemini-2.5-flash"
|
|
59
|
+
if os.getenv("OPENAI_API_KEY"):
|
|
60
|
+
return "openai", model or "gpt-4o"
|
|
61
|
+
if os.getenv("ANTHROPIC_API_KEY"):
|
|
62
|
+
return "anthropic", model or "claude-3-5-sonnet-20241022"
|
|
63
|
+
if os.getenv("OPENROUTER_API_KEY"):
|
|
64
|
+
return "openrouter", model or "meta-llama/llama-3.3-70b-instruct"
|
|
65
|
+
if os.getenv("DEEPSEEK_API_KEY"):
|
|
66
|
+
return "deepseek", model or "deepseek-chat"
|
|
67
|
+
|
|
68
|
+
return "gemini", model or "gemini-2.5-flash"
|
|
69
|
+
|
|
70
|
+
def _default_model_for_provider(self, provider: str) -> str:
|
|
71
|
+
defaults = {
|
|
72
|
+
"gemini": "gemini-2.5-flash",
|
|
73
|
+
"openai": "gpt-4o",
|
|
74
|
+
"anthropic": "claude-3-5-sonnet-20241022",
|
|
75
|
+
"openrouter": "meta-llama/llama-3.3-70b-instruct",
|
|
76
|
+
"deepseek": "deepseek-chat",
|
|
77
|
+
"nvidia": "nvidia/nemotron-4-340b-instruct",
|
|
78
|
+
"ollama": "llama3.2",
|
|
79
|
+
}
|
|
80
|
+
return defaults.get(provider, "gemini-2.5-flash")
|
|
81
|
+
|
|
82
|
+
def _resolve_api_key(self):
|
|
83
|
+
if self.api_key:
|
|
84
|
+
return
|
|
85
|
+
|
|
86
|
+
key_map = {
|
|
87
|
+
"gemini": "GEMINI_API_KEY",
|
|
88
|
+
"openai": "OPENAI_API_KEY",
|
|
89
|
+
"anthropic": "ANTHROPIC_API_KEY",
|
|
90
|
+
"openrouter": "OPENROUTER_API_KEY",
|
|
91
|
+
"deepseek": "DEEPSEEK_API_KEY",
|
|
92
|
+
"nvidia": "NVIDIA_API_KEY",
|
|
93
|
+
}
|
|
94
|
+
env_var = key_map.get(self.provider)
|
|
95
|
+
if env_var:
|
|
96
|
+
self.api_key = os.getenv(env_var)
|
|
97
|
+
|
|
98
|
+
def generate(self, prompt: str, system_instruction: Optional[str] = None) -> str:
|
|
99
|
+
"""Dispatch generation to appropriate provider implementation."""
|
|
100
|
+
if self.provider == "gemini":
|
|
101
|
+
return self._call_gemini(prompt, system_instruction)
|
|
102
|
+
elif self.provider == "anthropic":
|
|
103
|
+
return self._call_anthropic(prompt, system_instruction)
|
|
104
|
+
elif self.provider in ("openai", "openrouter", "deepseek", "nvidia", "ollama") or self.base_url:
|
|
105
|
+
return self._call_openai_compatible(prompt, system_instruction)
|
|
106
|
+
else:
|
|
107
|
+
return self._call_gemini(prompt, system_instruction)
|
|
108
|
+
|
|
109
|
+
def _call_gemini(self, prompt: str, system_instruction: Optional[str]) -> str:
|
|
110
|
+
# Try google-genai SDK first
|
|
111
|
+
try:
|
|
112
|
+
from google import genai
|
|
113
|
+
from google.genai import types
|
|
114
|
+
client = genai.Client(api_key=self.api_key)
|
|
115
|
+
cfg = {}
|
|
116
|
+
if system_instruction:
|
|
117
|
+
cfg["system_instruction"] = system_instruction
|
|
118
|
+
res = client.models.generate_content(
|
|
119
|
+
model=self.model,
|
|
120
|
+
contents=prompt,
|
|
121
|
+
config=types.GenerateContentConfig(**cfg) if cfg else None
|
|
122
|
+
)
|
|
123
|
+
if res and res.text:
|
|
124
|
+
return res.text
|
|
125
|
+
except Exception:
|
|
126
|
+
pass
|
|
127
|
+
|
|
128
|
+
# Fallback to direct REST endpoint
|
|
129
|
+
url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent?key={self.api_key}"
|
|
130
|
+
contents = []
|
|
131
|
+
if system_instruction:
|
|
132
|
+
contents.append({"role": "user", "parts": [{"text": f"SYSTEM: {system_instruction}"}]})
|
|
133
|
+
contents.append({"role": "user", "parts": [{"text": prompt}]})
|
|
134
|
+
resp = requests.post(url, json={"contents": contents}, timeout=self.timeout)
|
|
135
|
+
resp.raise_for_status()
|
|
136
|
+
data = resp.json()
|
|
137
|
+
return data["candidates"][0]["content"]["parts"][0]["text"]
|
|
138
|
+
|
|
139
|
+
def _call_anthropic(self, prompt: str, system_instruction: Optional[str]) -> str:
|
|
140
|
+
url = "https://api.anthropic.com/v1/messages"
|
|
141
|
+
headers = {
|
|
142
|
+
"x-api-key": self.api_key or "",
|
|
143
|
+
"anthropic-version": "2023-06-01",
|
|
144
|
+
"content-type": "application/json",
|
|
145
|
+
}
|
|
146
|
+
payload = {
|
|
147
|
+
"model": self.model,
|
|
148
|
+
"max_tokens": 4096,
|
|
149
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
150
|
+
}
|
|
151
|
+
if system_instruction:
|
|
152
|
+
payload["system"] = system_instruction
|
|
153
|
+
|
|
154
|
+
resp = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
|
|
155
|
+
resp.raise_for_status()
|
|
156
|
+
data = resp.json()
|
|
157
|
+
content = data.get("content", [])
|
|
158
|
+
if content and "text" in content[0]:
|
|
159
|
+
return content[0]["text"]
|
|
160
|
+
return ""
|
|
161
|
+
|
|
162
|
+
def _call_openai_compatible(self, prompt: str, system_instruction: Optional[str]) -> str:
|
|
163
|
+
endpoint_map = {
|
|
164
|
+
"openai": "https://api.openai.com/v1/chat/completions",
|
|
165
|
+
"openrouter": "https://openrouter.ai/api/v1/chat/completions",
|
|
166
|
+
"deepseek": "https://api.deepseek.com/v1/chat/completions",
|
|
167
|
+
"nvidia": "https://integrate.api.nvidia.com/v1/chat/completions",
|
|
168
|
+
"ollama": "http://localhost:11434/v1/chat/completions",
|
|
169
|
+
}
|
|
170
|
+
url = self.base_url or endpoint_map.get(self.provider, "https://api.openai.com/v1/chat/completions")
|
|
171
|
+
headers = {
|
|
172
|
+
"Authorization": f"Bearer {self.api_key or ''}",
|
|
173
|
+
"Content-Type": "application/json",
|
|
174
|
+
}
|
|
175
|
+
messages = []
|
|
176
|
+
if system_instruction:
|
|
177
|
+
messages.append({"role": "system", "content": system_instruction})
|
|
178
|
+
messages.append({"role": "user", "content": prompt})
|
|
179
|
+
|
|
180
|
+
payload = {
|
|
181
|
+
"model": self.model,
|
|
182
|
+
"messages": messages,
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
resp = requests.post(url, json=payload, headers=headers, timeout=self.timeout)
|
|
186
|
+
resp.raise_for_status()
|
|
187
|
+
data = resp.json()
|
|
188
|
+
return data["choices"][0]["message"]["content"]
|
deep_dossier/report.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Report generators for Markdown, interactive HTML, and JSON dossiers.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, Any, List
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def save_reports(data: Dict[str, Any], output_dir: Path, slug: str) -> Dict[str, Path]:
|
|
12
|
+
"""Saves report in Markdown, HTML, and JSON formats."""
|
|
13
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
14
|
+
paths = {}
|
|
15
|
+
|
|
16
|
+
# 1. Save Markdown
|
|
17
|
+
md_path = output_dir / f"{slug}.md"
|
|
18
|
+
with open(md_path, "w", encoding="utf-8") as f:
|
|
19
|
+
f.write(data["markdown_report"])
|
|
20
|
+
paths["markdown"] = md_path
|
|
21
|
+
|
|
22
|
+
# 2. Save JSON Dossier
|
|
23
|
+
json_path = output_dir / f"{slug}.json"
|
|
24
|
+
with open(json_path, "w", encoding="utf-8") as f:
|
|
25
|
+
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
26
|
+
paths["json"] = json_path
|
|
27
|
+
|
|
28
|
+
# 3. Save Interactive HTML
|
|
29
|
+
html_content = generate_html_report(data)
|
|
30
|
+
html_path = output_dir / f"{slug}.html"
|
|
31
|
+
with open(html_path, "w", encoding="utf-8") as f:
|
|
32
|
+
f.write(html_content)
|
|
33
|
+
paths["html"] = html_path
|
|
34
|
+
|
|
35
|
+
return paths
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def generate_html_report(data: Dict[str, Any]) -> str:
|
|
39
|
+
"""Generates an executive-grade HTML report with modern dark theme and responsive typography."""
|
|
40
|
+
title = data.get("topic", "Research Report")
|
|
41
|
+
timestamp = data.get("timestamp", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"))
|
|
42
|
+
sources: List[Dict[str, str]] = data.get("sources", [])
|
|
43
|
+
markdown_content = data.get("markdown_report", "")
|
|
44
|
+
|
|
45
|
+
# Basic markdown to html conversion or simple rendering
|
|
46
|
+
try:
|
|
47
|
+
import markdown
|
|
48
|
+
rendered_body = markdown.markdown(
|
|
49
|
+
markdown_content,
|
|
50
|
+
extensions=["tables", "fenced_code", "toc"]
|
|
51
|
+
)
|
|
52
|
+
except Exception:
|
|
53
|
+
# Fallback rendering if markdown package is missing
|
|
54
|
+
rendered_body = (
|
|
55
|
+
markdown_content
|
|
56
|
+
.replace("\n\n", "</p><p>")
|
|
57
|
+
.replace("# ", "<h1>")
|
|
58
|
+
.replace("## ", "<h2>")
|
|
59
|
+
.replace("### ", "<h3>")
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
sources_html = ""
|
|
63
|
+
for idx, s in enumerate(sources, 1):
|
|
64
|
+
url = s.get("url", "#")
|
|
65
|
+
stitle = s.get("title", url)
|
|
66
|
+
sources_html += f"""
|
|
67
|
+
<div class="source-card">
|
|
68
|
+
<span class="source-num">[{idx}]</span>
|
|
69
|
+
<div class="source-info">
|
|
70
|
+
<a href="{url}" target="_blank" rel="noopener noreferrer" class="source-title">{stitle}</a>
|
|
71
|
+
<span class="source-url">{url}</span>
|
|
72
|
+
</div>
|
|
73
|
+
</div>
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
html = f"""<!DOCTYPE html>
|
|
77
|
+
<html lang="en">
|
|
78
|
+
<head>
|
|
79
|
+
<meta charset="UTF-8">
|
|
80
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
81
|
+
<title>{title} — Deep Research Dossier</title>
|
|
82
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
83
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
84
|
+
<link href="https://fonts.googleapis.com/css2?family=Plus+Jakarta+Sans:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
|
85
|
+
<style>
|
|
86
|
+
:root {{
|
|
87
|
+
--bg: #090d16;
|
|
88
|
+
--surface: #111726;
|
|
89
|
+
--surface-border: #1e293b;
|
|
90
|
+
--primary: #38bdf8;
|
|
91
|
+
--primary-glow: rgba(56, 189, 248, 0.15);
|
|
92
|
+
--text-main: #f1f5f9;
|
|
93
|
+
--text-muted: #94a3b8;
|
|
94
|
+
--accent: #818cf8;
|
|
95
|
+
--success: #34d399;
|
|
96
|
+
}}
|
|
97
|
+
* {{ box-sizing: border-box; margin: 0; padding: 0; }}
|
|
98
|
+
body {{
|
|
99
|
+
background-color: var(--bg);
|
|
100
|
+
color: var(--text-main);
|
|
101
|
+
font-family: 'Plus Jakarta Sans', -apple-system, BlinkMacSystemFont, sans-serif;
|
|
102
|
+
line-height: 1.7;
|
|
103
|
+
padding: 2.5rem 1.5rem;
|
|
104
|
+
}}
|
|
105
|
+
.container {{
|
|
106
|
+
max-width: 900px;
|
|
107
|
+
margin: 0 auto;
|
|
108
|
+
}}
|
|
109
|
+
header {{
|
|
110
|
+
border-bottom: 1px solid var(--surface-border);
|
|
111
|
+
padding-bottom: 2rem;
|
|
112
|
+
margin-bottom: 2.5rem;
|
|
113
|
+
}}
|
|
114
|
+
.badge {{
|
|
115
|
+
display: inline-block;
|
|
116
|
+
background: var(--primary-glow);
|
|
117
|
+
color: var(--primary);
|
|
118
|
+
border: 1px solid rgba(56, 189, 248, 0.3);
|
|
119
|
+
font-size: 0.75rem;
|
|
120
|
+
font-weight: 600;
|
|
121
|
+
text-transform: uppercase;
|
|
122
|
+
letter-spacing: 0.05em;
|
|
123
|
+
padding: 0.25rem 0.75rem;
|
|
124
|
+
border-radius: 9999px;
|
|
125
|
+
margin-bottom: 1rem;
|
|
126
|
+
}}
|
|
127
|
+
h1 {{
|
|
128
|
+
font-size: 2.25rem;
|
|
129
|
+
font-weight: 700;
|
|
130
|
+
line-height: 1.25;
|
|
131
|
+
color: #ffffff;
|
|
132
|
+
margin-bottom: 0.75rem;
|
|
133
|
+
}}
|
|
134
|
+
.meta {{
|
|
135
|
+
color: var(--text-muted);
|
|
136
|
+
font-size: 0.875rem;
|
|
137
|
+
display: flex;
|
|
138
|
+
gap: 1.5rem;
|
|
139
|
+
}}
|
|
140
|
+
.content {{
|
|
141
|
+
background: var(--surface);
|
|
142
|
+
border: 1px solid var(--surface-border);
|
|
143
|
+
border-radius: 1rem;
|
|
144
|
+
padding: 2.5rem;
|
|
145
|
+
margin-bottom: 3rem;
|
|
146
|
+
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.3);
|
|
147
|
+
}}
|
|
148
|
+
.content h2 {{
|
|
149
|
+
color: #ffffff;
|
|
150
|
+
font-size: 1.5rem;
|
|
151
|
+
margin-top: 2rem;
|
|
152
|
+
margin-bottom: 1rem;
|
|
153
|
+
border-bottom: 1px solid var(--surface-border);
|
|
154
|
+
padding-bottom: 0.5rem;
|
|
155
|
+
}}
|
|
156
|
+
.content h3 {{
|
|
157
|
+
color: var(--primary);
|
|
158
|
+
font-size: 1.2rem;
|
|
159
|
+
margin-top: 1.5rem;
|
|
160
|
+
margin-bottom: 0.75rem;
|
|
161
|
+
}}
|
|
162
|
+
.content p {{
|
|
163
|
+
margin-bottom: 1.25rem;
|
|
164
|
+
color: #cbd5e1;
|
|
165
|
+
font-size: 1rem;
|
|
166
|
+
}}
|
|
167
|
+
.content ul, .content ol {{
|
|
168
|
+
margin-left: 1.5rem;
|
|
169
|
+
margin-bottom: 1.25rem;
|
|
170
|
+
color: #cbd5e1;
|
|
171
|
+
}}
|
|
172
|
+
.content li {{ margin-bottom: 0.5rem; }}
|
|
173
|
+
.content blockquote {{
|
|
174
|
+
border-left: 4px solid var(--primary);
|
|
175
|
+
padding-left: 1rem;
|
|
176
|
+
margin: 1.5rem 0;
|
|
177
|
+
color: var(--text-muted);
|
|
178
|
+
font-style: italic;
|
|
179
|
+
}}
|
|
180
|
+
.content table {{
|
|
181
|
+
width: 100%;
|
|
182
|
+
border-collapse: collapse;
|
|
183
|
+
margin: 1.5rem 0;
|
|
184
|
+
}}
|
|
185
|
+
.content th, .content td {{
|
|
186
|
+
padding: 0.75rem 1rem;
|
|
187
|
+
border: 1px solid var(--surface-border);
|
|
188
|
+
text-align: left;
|
|
189
|
+
}}
|
|
190
|
+
.content th {{ background: #1e293b; color: #ffffff; }}
|
|
191
|
+
code {{
|
|
192
|
+
font-family: 'JetBrains Mono', monospace;
|
|
193
|
+
background: #1e293b;
|
|
194
|
+
color: #38bdf8;
|
|
195
|
+
padding: 0.2rem 0.4rem;
|
|
196
|
+
border-radius: 4px;
|
|
197
|
+
font-size: 0.9em;
|
|
198
|
+
}}
|
|
199
|
+
.sources-section {{
|
|
200
|
+
margin-top: 2rem;
|
|
201
|
+
}}
|
|
202
|
+
.source-card {{
|
|
203
|
+
background: var(--surface);
|
|
204
|
+
border: 1px solid var(--surface-border);
|
|
205
|
+
border-radius: 0.5rem;
|
|
206
|
+
padding: 1rem 1.25rem;
|
|
207
|
+
margin-bottom: 0.75rem;
|
|
208
|
+
display: flex;
|
|
209
|
+
align-items: flex-start;
|
|
210
|
+
gap: 1rem;
|
|
211
|
+
transition: border-color 0.2s ease;
|
|
212
|
+
}}
|
|
213
|
+
.source-card:hover {{
|
|
214
|
+
border-color: var(--primary);
|
|
215
|
+
}}
|
|
216
|
+
.source-num {{
|
|
217
|
+
color: var(--primary);
|
|
218
|
+
font-weight: 600;
|
|
219
|
+
font-family: 'JetBrains Mono', monospace;
|
|
220
|
+
}}
|
|
221
|
+
.source-title {{
|
|
222
|
+
color: #ffffff;
|
|
223
|
+
font-weight: 500;
|
|
224
|
+
text-decoration: none;
|
|
225
|
+
display: block;
|
|
226
|
+
margin-bottom: 0.25rem;
|
|
227
|
+
}}
|
|
228
|
+
.source-title:hover {{
|
|
229
|
+
text-decoration: underline;
|
|
230
|
+
color: var(--primary);
|
|
231
|
+
}}
|
|
232
|
+
.source-url {{
|
|
233
|
+
color: var(--text-muted);
|
|
234
|
+
font-size: 0.8rem;
|
|
235
|
+
word-break: break-all;
|
|
236
|
+
}}
|
|
237
|
+
footer {{
|
|
238
|
+
text-align: center;
|
|
239
|
+
color: var(--text-muted);
|
|
240
|
+
font-size: 0.875rem;
|
|
241
|
+
margin-top: 3rem;
|
|
242
|
+
border-top: 1px solid var(--surface-border);
|
|
243
|
+
padding-top: 1.5rem;
|
|
244
|
+
}}
|
|
245
|
+
</style>
|
|
246
|
+
</head>
|
|
247
|
+
<body>
|
|
248
|
+
<div class="container">
|
|
249
|
+
<header>
|
|
250
|
+
<span class="badge">Open Deep Research • Gemini Autonomous Agent</span>
|
|
251
|
+
<h1>{title}</h1>
|
|
252
|
+
<div class="meta">
|
|
253
|
+
<span>Generated: {timestamp}</span>
|
|
254
|
+
<span>Sources verified: {len(sources)}</span>
|
|
255
|
+
</div>
|
|
256
|
+
</header>
|
|
257
|
+
|
|
258
|
+
<main class="content">
|
|
259
|
+
{rendered_body}
|
|
260
|
+
</main>
|
|
261
|
+
|
|
262
|
+
<section class="sources-section">
|
|
263
|
+
<h2>Verified Citations & Bibliography ({len(sources)})</h2>
|
|
264
|
+
{sources_html}
|
|
265
|
+
</section>
|
|
266
|
+
|
|
267
|
+
<footer>
|
|
268
|
+
Generated by <a href="https://github.com/shubhransh-gupta/open-deep-research" style="color:var(--primary);text-decoration:none;">Open Deep Research</a> • Powered by Google Gemini
|
|
269
|
+
</footer>
|
|
270
|
+
</div>
|
|
271
|
+
</body>
|
|
272
|
+
</html>
|
|
273
|
+
"""
|
|
274
|
+
return html
|
deep_dossier/search.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Search module providing search capabilities via DuckDuckGo and custom search backends.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
import logging
|
|
7
|
+
from typing import List, Dict, Optional
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class SearchResult:
|
|
15
|
+
title: str
|
|
16
|
+
url: str
|
|
17
|
+
snippet: str
|
|
18
|
+
source: str = "duckduckgo"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SearchEngine:
|
|
22
|
+
"""Multi-backend search engine supporting DuckDuckGo and fallback methods."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, timeout: int = 15):
|
|
25
|
+
self.timeout = timeout
|
|
26
|
+
|
|
27
|
+
def search(self, query: str, max_results: int = 5) -> List[SearchResult]:
|
|
28
|
+
"""Perform search with retries and fallback."""
|
|
29
|
+
results: List[SearchResult] = []
|
|
30
|
+
try:
|
|
31
|
+
results = self._search_duckduckgo(query, max_results)
|
|
32
|
+
if results:
|
|
33
|
+
return results
|
|
34
|
+
except Exception as e:
|
|
35
|
+
logger.warning(f"DuckDuckGo search failed for '{query}': {e}")
|
|
36
|
+
|
|
37
|
+
# Fallback search if DDG failed
|
|
38
|
+
try:
|
|
39
|
+
results = self._search_html_fallback(query, max_results)
|
|
40
|
+
except Exception as e:
|
|
41
|
+
logger.error(f"Fallback search also failed for '{query}': {e}")
|
|
42
|
+
|
|
43
|
+
return results
|
|
44
|
+
|
|
45
|
+
def _search_duckduckgo(self, query: str, max_results: int) -> List[SearchResult]:
|
|
46
|
+
"""Search using duckduckgo_search library."""
|
|
47
|
+
try:
|
|
48
|
+
from duckduckgo_search import DDGS
|
|
49
|
+
with DDGS() as ddgs:
|
|
50
|
+
raw_results = list(ddgs.text(query, max_results=max_results))
|
|
51
|
+
results = []
|
|
52
|
+
for r in raw_results:
|
|
53
|
+
title = r.get("title", "")
|
|
54
|
+
url = r.get("href", "")
|
|
55
|
+
body = r.get("body", "")
|
|
56
|
+
if url and (title or body):
|
|
57
|
+
results.append(SearchResult(title=title, url=url, snippet=body))
|
|
58
|
+
return results
|
|
59
|
+
except Exception as e:
|
|
60
|
+
logger.debug(f"DDGS error: {e}")
|
|
61
|
+
raise
|
|
62
|
+
|
|
63
|
+
def _search_html_fallback(self, query: str, max_results: int) -> List[SearchResult]:
|
|
64
|
+
"""Direct lightweight HTTP fallback for DuckDuckGo HTML endpoint."""
|
|
65
|
+
import requests
|
|
66
|
+
from bs4 import BeautifulSoup
|
|
67
|
+
import urllib.parse
|
|
68
|
+
|
|
69
|
+
url = "https://html.duckduckgo.com/html/"
|
|
70
|
+
headers = {
|
|
71
|
+
"User-Agent": (
|
|
72
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|
73
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
resp = requests.post(
|
|
77
|
+
url,
|
|
78
|
+
data={"q": query},
|
|
79
|
+
headers=headers,
|
|
80
|
+
timeout=self.timeout
|
|
81
|
+
)
|
|
82
|
+
resp.raise_for_status()
|
|
83
|
+
|
|
84
|
+
soup = BeautifulSoup(resp.text, "html.parser")
|
|
85
|
+
results = []
|
|
86
|
+
for result in soup.find_all("div", class_="result", limit=max_results):
|
|
87
|
+
title_tag = result.find("a", class_="result__a")
|
|
88
|
+
snippet_tag = result.find("a", class_="result__snippet")
|
|
89
|
+
if title_tag:
|
|
90
|
+
raw_link = title_tag.get("href", "")
|
|
91
|
+
parsed_link = urllib.parse.parse_qs(urllib.parse.urlparse(raw_link).query).get("uddg")
|
|
92
|
+
clean_url = parsed_link[0] if parsed_link else raw_link
|
|
93
|
+
title = title_tag.get_text(strip=True)
|
|
94
|
+
snippet = snippet_tag.get_text(strip=True) if snippet_tag else ""
|
|
95
|
+
if clean_url.startswith("http"):
|
|
96
|
+
results.append(SearchResult(title=title, url=clean_url, snippet=snippet, source="ddg-html"))
|
|
97
|
+
return results
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deep-dossier
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Autonomous multi-step research agent using Gemini for recursive web discovery, synthesis, and verified intelligence dossiers.
|
|
5
|
+
Author: Shubhransh Gupta
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: deep-research,gemini,ai-agent,research-agent,autonomous-agent,llm,google-gemini,dossier,report-generator,python,open-source
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: google-genai>=3.0.0
|
|
20
|
+
Requires-Dist: rich>=13.0.0
|
|
21
|
+
Requires-Dist: requests>=2.28.0
|
|
22
|
+
Requires-Dist: duckduckgo-search>=7.0.0
|
|
23
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
24
|
+
Requires-Dist: pydantic>=2.0.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
27
|
+
|
|
28
|
+
# ⚡ Deep Dossier
|
|
29
|
+
|
|
30
|
+
[](https://agent-docs-coral.vercel.app)
|
|
31
|
+
[](https://github.com/shubhransh-gupta/deep-dossier/stargazers)
|
|
32
|
+
[](https://opensource.org/licenses/MIT)
|
|
33
|
+
|
|
34
|
+
> **📖 Live Documentation & Interactive FAQs:** [https://agent-docs-coral.vercel.app](https://agent-docs-coral.vercel.app)
|
|
35
|
+
|
|
36
|
+
<p align="center">
|
|
37
|
+
<img src="assets/demo.svg" alt="Deep Dossier Live Terminal Demo" width="100%" />
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
An open-source autonomous multi-step research agent with **universal model support**.
|
|
41
|
+
|
|
42
|
+
Deep Dossier decomposes complex research prompts into targeted search plans, recursively crawls dozens of web pages and documents in parallel, resolves information gaps, and produces publication-grade Markdown and interactive HTML dossiers with verified in-text citations.
|
|
43
|
+
|
|
44
|
+
## 🌐 Universal Model Support (Bring Your Own Key)
|
|
45
|
+
|
|
46
|
+
Deep Dossier is model-agnostic. Choose your preferred AI provider or run completely local:
|
|
47
|
+
|
|
48
|
+
| Provider | Supported Models | Environment Variable |
|
|
49
|
+
| :--- | :--- | :--- |
|
|
50
|
+
| **Google Gemini** | `gemini-2.5-flash`, `gemini-2.5-pro`, `gemini-1.5-pro` | `export GEMINI_API_KEY="..."` |
|
|
51
|
+
| **Anthropic Claude** | `claude-3-5-sonnet-20241022`, `claude-3-7-sonnet`, `claude-3-opus` | `export ANTHROPIC_API_KEY="..."` |
|
|
52
|
+
| **OpenAI** | `gpt-4o`, `gpt-4o-mini`, `o1`, `o3-mini` | `export OPENAI_API_KEY="..."` |
|
|
53
|
+
| **DeepSeek** | `deepseek-chat`, `deepseek-reasoner` | `export DEEPSEEK_API_KEY="..."` |
|
|
54
|
+
| **NVIDIA Nemotron** | `nvidia/nemotron-4-340b-instruct` | `export NVIDIA_API_KEY="..."` |
|
|
55
|
+
| **OpenRouter** | 200+ models across all frontier labs | `export OPENROUTER_API_KEY="..."` |
|
|
56
|
+
| **Local / Self-Hosted** | Ollama (`llama3.2`, `mistral`, `qwen2.5`), vLLM | `--base-url http://localhost:11434/v1` |
|
|
57
|
+
|
|
58
|
+
## 🌟 Key Highlights
|
|
59
|
+
|
|
60
|
+
- **Multi-Provider Flexibility**: Switch between Gemini, Claude, GPT, DeepSeek, or local Ollama with a single flag.
|
|
61
|
+
- **Recursive Search & Discovery**: Explores multi-hop questions by identifying knowledge gaps after each crawl iteration.
|
|
62
|
+
- **Zero-Config Search**: Built-in DuckDuckGo search integration requiring zero additional API keys.
|
|
63
|
+
- **Publication Deliverables**: Exports structured Markdown, JSON intelligence dossiers, and self-contained dark-mode interactive HTML reports.
|
|
64
|
+
|
|
65
|
+
## 🚀 Quick Start
|
|
66
|
+
|
|
67
|
+
### 1. Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
git clone https://github.com/shubhransh-gupta/deep-dossier.git
|
|
71
|
+
cd deep-dossier
|
|
72
|
+
pip install -e .
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### 2. Set Your Preferred API Key
|
|
76
|
+
|
|
77
|
+
Choose any provider:
|
|
78
|
+
```bash
|
|
79
|
+
# Option A: Google Gemini
|
|
80
|
+
export GEMINI_API_KEY="AIzaSy..."
|
|
81
|
+
|
|
82
|
+
# Option B: Anthropic Claude
|
|
83
|
+
export ANTHROPIC_API_KEY="sk-ant-..."
|
|
84
|
+
|
|
85
|
+
# Option C: OpenAI
|
|
86
|
+
export OPENAI_API_KEY="sk-proj-..."
|
|
87
|
+
|
|
88
|
+
# Option D: DeepSeek
|
|
89
|
+
export DEEPSEEK_API_KEY="sk-..."
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### 3. Run a Research Mission
|
|
93
|
+
|
|
94
|
+
Using default/auto-detected provider:
|
|
95
|
+
```bash
|
|
96
|
+
deep-dossier "Analyze battery degradation patterns of LFP vs NMC cells in commercial EVs"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Explicitly choose provider and model:
|
|
100
|
+
```bash
|
|
101
|
+
# Using Anthropic Claude 3.5 Sonnet
|
|
102
|
+
deep-dossier "Solid-state battery commercial readiness" --provider anthropic --model claude-3-5-sonnet-20241022
|
|
103
|
+
|
|
104
|
+
# Using OpenAI GPT-4o
|
|
105
|
+
deep-dossier "Cross-border payment rail comparison" --provider openai --model gpt-4o
|
|
106
|
+
|
|
107
|
+
# Using Local Ollama
|
|
108
|
+
deep-dossier "Local AI agent architectures" --base-url http://localhost:11434/v1 --model llama3.2
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## ⚙️ Options
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
deep-dossier [TOPIC] [OPTIONS]
|
|
115
|
+
|
|
116
|
+
Options:
|
|
117
|
+
--provider, -p TEXT LLM provider: gemini, openai, anthropic, deepseek, nvidia, ollama, openrouter
|
|
118
|
+
--model, -m TEXT Specific model name
|
|
119
|
+
--base-url TEXT Custom API base URL (for Ollama, vLLM, or custom proxies)
|
|
120
|
+
--depth, -d INTEGER Recursive search depth (default: 2)
|
|
121
|
+
--breadth, -b INTEGER Number of sources to retrieve per search query (default: 4)
|
|
122
|
+
--output, -o PATH Output directory for generated dossiers (default: ./research_outputs)
|
|
123
|
+
--api-key, -k TEXT Explicit API key override
|
|
124
|
+
--no-html Disable HTML report generation
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## 📦 Project Structure
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
deep-dossier/
|
|
131
|
+
├── deep_dossier/
|
|
132
|
+
│ ├── agent.py # Core autonomous orchestrator and synthesis engine
|
|
133
|
+
│ ├── llm.py # Universal multi-provider LLM interface (Gemini/Claude/GPT/Ollama)
|
|
134
|
+
│ ├── crawler.py # Concurrent HTML & document content extractor
|
|
135
|
+
│ ├── search.py # Search engine interface
|
|
136
|
+
│ ├── report.py # Markdown and interactive HTML renderer
|
|
137
|
+
│ ├── config.py # Configuration and environment loader
|
|
138
|
+
│ └── cli.py # Rich terminal interface
|
|
139
|
+
└── pyproject.toml # Build metadata and entry points
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## 📄 License
|
|
143
|
+
|
|
144
|
+
MIT License. Copyright (c) 2026 Shubhransh Gupta.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
deep_dossier/__init__.py,sha256=C0mlOTi4jYCw59wnPfHCSqfEQcR7Vo5jkvFJ3QbGozk,275
|
|
2
|
+
deep_dossier/agent.py,sha256=P05foZXGMnZX3mr4FPCoc4vpbAjC5BOHar4wUOzEYIQ,9480
|
|
3
|
+
deep_dossier/cli.py,sha256=8Q8-1IWzRh2dVCdAdG0AJC2H9yqh6mC__jtmQWu1hBg,4805
|
|
4
|
+
deep_dossier/config.py,sha256=T1MlJxtriOgiHeq2xBf-jFJQ6rpZEw5Onuu5P2oaXKU,3054
|
|
5
|
+
deep_dossier/crawler.py,sha256=9Qejw_Sd-4uxFQtTyRL-9XHgDEK5cBlnlEnYTbNOHRU,5212
|
|
6
|
+
deep_dossier/llm.py,sha256=3riyTdETThp8zit6OWlaFxBY7d6y9gCQ41xW9XOWHnU,7305
|
|
7
|
+
deep_dossier/report.py,sha256=C-OOYK7O0X5SQ-oJZgHhYK3shCkiQcHvOhwrfBwN23Q,8814
|
|
8
|
+
deep_dossier/search.py,sha256=WSSBHTCH_vLiq05lGryaCdNNH2SH1UFVBt4NtR54QiA,3550
|
|
9
|
+
deep_dossier-0.1.0.dist-info/METADATA,sha256=ySuN05IyYyaUBEqdkyV4Z2io_tnCkWLklAZbLdk2fdY,6133
|
|
10
|
+
deep_dossier-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
11
|
+
deep_dossier-0.1.0.dist-info/entry_points.txt,sha256=fMwjyvybhDrkWdWAM7B8ChZXoesSHGguqxmR7XXcJ2c,55
|
|
12
|
+
deep_dossier-0.1.0.dist-info/top_level.txt,sha256=yMpgleYZ0W_lpONVB6pmznhTtS3yURuBALaTVevzag0,13
|
|
13
|
+
deep_dossier-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
deep_dossier
|