local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- local_deep_research/__init__.py +23 -22
- local_deep_research/__main__.py +16 -0
- local_deep_research/advanced_search_system/__init__.py +7 -0
- local_deep_research/advanced_search_system/filters/__init__.py +8 -0
- local_deep_research/advanced_search_system/filters/base_filter.py +38 -0
- local_deep_research/advanced_search_system/filters/cross_engine_filter.py +200 -0
- local_deep_research/advanced_search_system/findings/base_findings.py +81 -0
- local_deep_research/advanced_search_system/findings/repository.py +452 -0
- local_deep_research/advanced_search_system/knowledge/__init__.py +1 -0
- local_deep_research/advanced_search_system/knowledge/base_knowledge.py +151 -0
- local_deep_research/advanced_search_system/knowledge/standard_knowledge.py +159 -0
- local_deep_research/advanced_search_system/questions/__init__.py +1 -0
- local_deep_research/advanced_search_system/questions/base_question.py +64 -0
- local_deep_research/advanced_search_system/questions/decomposition_question.py +445 -0
- local_deep_research/advanced_search_system/questions/standard_question.py +119 -0
- local_deep_research/advanced_search_system/repositories/__init__.py +7 -0
- local_deep_research/advanced_search_system/strategies/__init__.py +1 -0
- local_deep_research/advanced_search_system/strategies/base_strategy.py +118 -0
- local_deep_research/advanced_search_system/strategies/iterdrag_strategy.py +450 -0
- local_deep_research/advanced_search_system/strategies/parallel_search_strategy.py +312 -0
- local_deep_research/advanced_search_system/strategies/rapid_search_strategy.py +270 -0
- local_deep_research/advanced_search_system/strategies/standard_strategy.py +300 -0
- local_deep_research/advanced_search_system/tools/__init__.py +1 -0
- local_deep_research/advanced_search_system/tools/base_tool.py +100 -0
- local_deep_research/advanced_search_system/tools/knowledge_tools/__init__.py +1 -0
- local_deep_research/advanced_search_system/tools/question_tools/__init__.py +1 -0
- local_deep_research/advanced_search_system/tools/search_tools/__init__.py +1 -0
- local_deep_research/api/__init__.py +5 -5
- local_deep_research/api/research_functions.py +154 -160
- local_deep_research/app.py +8 -0
- local_deep_research/citation_handler.py +25 -16
- local_deep_research/{config.py → config/config_files.py} +102 -110
- local_deep_research/config/llm_config.py +472 -0
- local_deep_research/config/search_config.py +77 -0
- local_deep_research/defaults/__init__.py +10 -5
- local_deep_research/defaults/main.toml +2 -2
- local_deep_research/defaults/search_engines.toml +60 -34
- local_deep_research/main.py +121 -19
- local_deep_research/migrate_db.py +147 -0
- local_deep_research/report_generator.py +87 -45
- local_deep_research/search_system.py +153 -283
- local_deep_research/setup_data_dir.py +35 -0
- local_deep_research/test_migration.py +178 -0
- local_deep_research/utilities/__init__.py +0 -0
- local_deep_research/utilities/db_utils.py +49 -0
- local_deep_research/{utilties → utilities}/enums.py +2 -2
- local_deep_research/{utilties → utilities}/llm_utils.py +63 -29
- local_deep_research/utilities/search_utilities.py +242 -0
- local_deep_research/{utilties → utilities}/setup_utils.py +4 -2
- local_deep_research/web/__init__.py +0 -1
- local_deep_research/web/app.py +86 -1709
- local_deep_research/web/app_factory.py +289 -0
- local_deep_research/web/database/README.md +70 -0
- local_deep_research/web/database/migrate_to_ldr_db.py +289 -0
- local_deep_research/web/database/migrations.py +447 -0
- local_deep_research/web/database/models.py +117 -0
- local_deep_research/web/database/schema_upgrade.py +107 -0
- local_deep_research/web/models/database.py +294 -0
- local_deep_research/web/models/settings.py +94 -0
- local_deep_research/web/routes/api_routes.py +559 -0
- local_deep_research/web/routes/history_routes.py +354 -0
- local_deep_research/web/routes/research_routes.py +715 -0
- local_deep_research/web/routes/settings_routes.py +1583 -0
- local_deep_research/web/services/research_service.py +947 -0
- local_deep_research/web/services/resource_service.py +149 -0
- local_deep_research/web/services/settings_manager.py +669 -0
- local_deep_research/web/services/settings_service.py +187 -0
- local_deep_research/web/services/socket_service.py +210 -0
- local_deep_research/web/static/css/custom_dropdown.css +277 -0
- local_deep_research/web/static/css/settings.css +1223 -0
- local_deep_research/web/static/css/styles.css +525 -48
- local_deep_research/web/static/js/components/custom_dropdown.js +428 -0
- local_deep_research/web/static/js/components/detail.js +348 -0
- local_deep_research/web/static/js/components/fallback/formatting.js +122 -0
- local_deep_research/web/static/js/components/fallback/ui.js +215 -0
- local_deep_research/web/static/js/components/history.js +487 -0
- local_deep_research/web/static/js/components/logpanel.js +949 -0
- local_deep_research/web/static/js/components/progress.js +1107 -0
- local_deep_research/web/static/js/components/research.js +1865 -0
- local_deep_research/web/static/js/components/results.js +766 -0
- local_deep_research/web/static/js/components/settings.js +3981 -0
- local_deep_research/web/static/js/components/settings_sync.js +106 -0
- local_deep_research/web/static/js/main.js +226 -0
- local_deep_research/web/static/js/services/api.js +253 -0
- local_deep_research/web/static/js/services/audio.js +31 -0
- local_deep_research/web/static/js/services/formatting.js +119 -0
- local_deep_research/web/static/js/services/pdf.js +622 -0
- local_deep_research/web/static/js/services/socket.js +882 -0
- local_deep_research/web/static/js/services/ui.js +546 -0
- local_deep_research/web/templates/base.html +72 -0
- local_deep_research/web/templates/components/custom_dropdown.html +47 -0
- local_deep_research/web/templates/components/log_panel.html +32 -0
- local_deep_research/web/templates/components/mobile_nav.html +22 -0
- local_deep_research/web/templates/components/settings_form.html +299 -0
- local_deep_research/web/templates/components/sidebar.html +21 -0
- local_deep_research/web/templates/pages/details.html +73 -0
- local_deep_research/web/templates/pages/history.html +51 -0
- local_deep_research/web/templates/pages/progress.html +57 -0
- local_deep_research/web/templates/pages/research.html +139 -0
- local_deep_research/web/templates/pages/results.html +59 -0
- local_deep_research/web/templates/settings_dashboard.html +78 -192
- local_deep_research/web/utils/__init__.py +0 -0
- local_deep_research/web/utils/formatters.py +76 -0
- local_deep_research/web_search_engines/engines/full_search.py +18 -16
- local_deep_research/web_search_engines/engines/meta_search_engine.py +182 -131
- local_deep_research/web_search_engines/engines/search_engine_arxiv.py +224 -139
- local_deep_research/web_search_engines/engines/search_engine_brave.py +88 -71
- local_deep_research/web_search_engines/engines/search_engine_ddg.py +48 -39
- local_deep_research/web_search_engines/engines/search_engine_github.py +415 -204
- local_deep_research/web_search_engines/engines/search_engine_google_pse.py +123 -90
- local_deep_research/web_search_engines/engines/search_engine_guardian.py +210 -157
- local_deep_research/web_search_engines/engines/search_engine_local.py +532 -369
- local_deep_research/web_search_engines/engines/search_engine_local_all.py +42 -36
- local_deep_research/web_search_engines/engines/search_engine_pubmed.py +358 -266
- local_deep_research/web_search_engines/engines/search_engine_searxng.py +212 -160
- local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py +213 -170
- local_deep_research/web_search_engines/engines/search_engine_serpapi.py +84 -68
- local_deep_research/web_search_engines/engines/search_engine_wayback.py +186 -154
- local_deep_research/web_search_engines/engines/search_engine_wikipedia.py +115 -77
- local_deep_research/web_search_engines/search_engine_base.py +174 -99
- local_deep_research/web_search_engines/search_engine_factory.py +192 -102
- local_deep_research/web_search_engines/search_engines_config.py +22 -15
- {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/METADATA +177 -97
- local_deep_research-0.2.2.dist-info/RECORD +135 -0
- {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/WHEEL +1 -2
- {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/entry_points.txt +3 -0
- local_deep_research/defaults/llm_config.py +0 -338
- local_deep_research/utilties/search_utilities.py +0 -114
- local_deep_research/web/static/js/app.js +0 -3763
- local_deep_research/web/templates/api_keys_config.html +0 -82
- local_deep_research/web/templates/collections_config.html +0 -90
- local_deep_research/web/templates/index.html +0 -348
- local_deep_research/web/templates/llm_config.html +0 -120
- local_deep_research/web/templates/main_config.html +0 -89
- local_deep_research/web/templates/search_engines_config.html +0 -154
- local_deep_research/web/templates/settings.html +0 -519
- local_deep_research-0.1.26.dist-info/RECORD +0 -61
- local_deep_research-0.1.26.dist-info/top_level.txt +0 -1
- /local_deep_research/{utilties → config}/__init__.py +0 -0
- {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,16 +1,42 @@
|
|
1
|
-
|
2
|
-
from
|
3
|
-
|
4
|
-
from
|
1
|
+
import importlib
|
2
|
+
from typing import Dict, List
|
3
|
+
|
4
|
+
from langchain_core.language_models import BaseChatModel
|
5
|
+
|
6
|
+
# Fix circular import by importing directly from source modules
|
7
|
+
from .config.llm_config import get_llm
|
5
8
|
from .search_system import AdvancedSearchSystem
|
6
|
-
from
|
7
|
-
|
8
|
-
|
9
|
+
from .utilities import search_utilities
|
10
|
+
|
11
|
+
|
12
|
+
def get_report_generator(search_system=None):
|
13
|
+
"""Return an instance of the report generator with default settings.
|
14
|
+
|
15
|
+
Args:
|
16
|
+
search_system: Optional existing AdvancedSearchSystem to use
|
17
|
+
"""
|
18
|
+
return IntegratedReportGenerator(search_system=search_system)
|
19
|
+
|
9
20
|
|
10
21
|
class IntegratedReportGenerator:
|
11
|
-
def __init__(
|
12
|
-
self
|
13
|
-
|
22
|
+
def __init__(
|
23
|
+
self,
|
24
|
+
searches_per_section: int = 2,
|
25
|
+
search_system=None,
|
26
|
+
llm: BaseChatModel | None = None,
|
27
|
+
):
|
28
|
+
"""
|
29
|
+
Args:
|
30
|
+
searches_per_section: Number of searches to perform for each
|
31
|
+
section in the report.
|
32
|
+
search_system: Custom search system to use, otherwise just uses
|
33
|
+
the default.
|
34
|
+
llm: Custom LLM to use, otherwise just uses the default.
|
35
|
+
|
36
|
+
"""
|
37
|
+
self.model = llm or get_llm()
|
38
|
+
# Use provided search_system or create a new one
|
39
|
+
self.search_system = search_system or AdvancedSearchSystem(llm=self.model)
|
14
40
|
self.searches_per_section = (
|
15
41
|
searches_per_section # Control search depth per section
|
16
42
|
)
|
@@ -22,17 +48,16 @@ class IntegratedReportGenerator:
|
|
22
48
|
structure = self._determine_report_structure(initial_findings, query)
|
23
49
|
|
24
50
|
# Step 2: Research and generate content for each section in one step
|
25
|
-
sections = self._research_and_generate_sections(
|
51
|
+
sections = self._research_and_generate_sections(
|
52
|
+
initial_findings, structure, query
|
53
|
+
)
|
26
54
|
|
27
55
|
# Step 3: Format final report
|
28
56
|
report = self._format_final_report(sections, structure, query)
|
29
57
|
|
30
58
|
return report
|
31
59
|
|
32
|
-
def _determine_report_structure(
|
33
|
-
self, findings: Dict, query: str
|
34
|
-
) -> List[Dict]:
|
35
|
-
|
60
|
+
def _determine_report_structure(self, findings: Dict, query: str) -> List[Dict]:
|
36
61
|
"""Analyze content and determine optimal report structure."""
|
37
62
|
combined_content = findings["current_knowledge"]
|
38
63
|
prompt = f"""
|
@@ -92,44 +117,51 @@ class IntegratedReportGenerator:
|
|
92
117
|
) -> Dict[str, str]:
|
93
118
|
"""Research and generate content for each section in one step."""
|
94
119
|
sections = {}
|
95
|
-
|
120
|
+
|
96
121
|
for section in structure:
|
97
122
|
print(f"Processing section: {section['name']}")
|
98
123
|
section_content = []
|
99
124
|
section_content.append(f"# {section['name']}\n")
|
100
|
-
|
125
|
+
|
101
126
|
# Process each subsection by directly researching it
|
102
127
|
for subsection in section["subsections"]:
|
103
128
|
# Add subsection header
|
104
129
|
section_content.append(f"## {subsection['name']}\n")
|
105
130
|
section_content.append(f"_{subsection['purpose']}_\n\n")
|
106
|
-
|
131
|
+
|
107
132
|
# Generate a specific search query for this subsection
|
108
133
|
subsection_query = f"{query} {section['name']} {subsection['name']} {subsection['purpose']}"
|
109
|
-
|
110
|
-
print(
|
111
|
-
|
134
|
+
|
135
|
+
print(
|
136
|
+
f"Researching subsection: {subsection['name']} with query: {subsection_query}"
|
137
|
+
)
|
138
|
+
|
112
139
|
# Configure search system for focused search
|
113
140
|
original_max_iterations = self.search_system.max_iterations
|
114
141
|
self.search_system.max_iterations = 1 # Keep search focused
|
115
|
-
|
142
|
+
|
116
143
|
# Perform search for this subsection
|
117
144
|
subsection_results = self.search_system.analyze_topic(subsection_query)
|
118
|
-
|
145
|
+
|
119
146
|
# Restore original iterations setting
|
120
147
|
self.search_system.max_iterations = original_max_iterations
|
121
|
-
|
148
|
+
|
122
149
|
# Add the researched content for this subsection
|
123
|
-
if
|
150
|
+
if (
|
151
|
+
"current_knowledge" in subsection_results
|
152
|
+
and subsection_results["current_knowledge"]
|
153
|
+
):
|
124
154
|
section_content.append(subsection_results["current_knowledge"])
|
125
155
|
else:
|
126
|
-
section_content.append(
|
127
|
-
|
156
|
+
section_content.append(
|
157
|
+
"*Limited information was found for this subsection.*\n"
|
158
|
+
)
|
159
|
+
|
128
160
|
section_content.append("\n\n")
|
129
|
-
|
161
|
+
|
130
162
|
# Combine all content for this section
|
131
163
|
sections[section["name"]] = "\n".join(section_content)
|
132
|
-
|
164
|
+
|
133
165
|
return sections
|
134
166
|
|
135
167
|
def _generate_sections(
|
@@ -157,15 +189,21 @@ class IntegratedReportGenerator:
|
|
157
189
|
for i, section in enumerate(structure, 1):
|
158
190
|
toc.append(f"{i}. **{section['name']}**")
|
159
191
|
for j, subsection in enumerate(section["subsections"], 1):
|
160
|
-
toc.append(
|
192
|
+
toc.append(
|
193
|
+
f" {i}.{j} {subsection['name']} | _{subsection['purpose']}_"
|
194
|
+
)
|
161
195
|
|
162
196
|
# Combine TOC and sections
|
163
197
|
report_parts = ["\n".join(toc), ""]
|
164
|
-
|
198
|
+
|
165
199
|
# Add a summary of the research
|
166
200
|
report_parts.append("# Research Summary")
|
167
|
-
report_parts.append(
|
168
|
-
|
201
|
+
report_parts.append(
|
202
|
+
"This report was researched using an advanced search system."
|
203
|
+
)
|
204
|
+
report_parts.append(
|
205
|
+
"Research included targeted searches for each section and subsection."
|
206
|
+
)
|
169
207
|
report_parts.append("\n---\n")
|
170
208
|
|
171
209
|
# Add each section's content
|
@@ -173,30 +211,34 @@ class IntegratedReportGenerator:
|
|
173
211
|
if section["name"] in sections:
|
174
212
|
report_parts.append(sections[section["name"]])
|
175
213
|
report_parts.append("")
|
176
|
-
|
214
|
+
|
177
215
|
# Format links from search system
|
178
|
-
|
179
|
-
|
216
|
+
# Get utilities module dynamically to avoid circular imports
|
217
|
+
utilities = importlib.import_module("local_deep_research.utilities")
|
218
|
+
formatted_all_links = utilities.search_utilities.format_links_to_markdown(
|
219
|
+
all_links=self.search_system.all_links_of_system
|
220
|
+
)
|
221
|
+
|
180
222
|
# Create final report with all parts
|
181
223
|
final_report_content = "\n\n".join(report_parts)
|
182
|
-
final_report_content =
|
183
|
-
|
224
|
+
final_report_content = (
|
225
|
+
final_report_content + "\n\n## Sources\n\n" + formatted_all_links
|
226
|
+
)
|
227
|
+
|
184
228
|
# Create metadata dictionary
|
185
229
|
from datetime import datetime
|
230
|
+
|
186
231
|
metadata = {
|
187
232
|
"generated_at": datetime.utcnow().isoformat(),
|
188
233
|
"initial_sources": len(self.search_system.all_links_of_system),
|
189
234
|
"sections_researched": len(structure),
|
190
235
|
"searches_per_section": self.searches_per_section,
|
191
|
-
"query": query
|
236
|
+
"query": query,
|
192
237
|
}
|
193
|
-
|
238
|
+
|
194
239
|
# Return both content and metadata
|
195
|
-
return {
|
196
|
-
"content": final_report_content,
|
197
|
-
"metadata": metadata
|
198
|
-
}
|
240
|
+
return {"content": final_report_content, "metadata": metadata}
|
199
241
|
|
200
242
|
def _generate_error_report(self, query: str, error_msg: str) -> str:
|
201
243
|
error_report = f"=== ERROR REPORT ===\nQuery: {query}\nError: {error_msg}"
|
202
|
-
return error_report
|
244
|
+
return error_report
|