local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- local_deep_research/__version__.py +1 -1
- local_deep_research/advanced_search_system/filters/base_filter.py +2 -3
- local_deep_research/advanced_search_system/filters/cross_engine_filter.py +4 -5
- local_deep_research/advanced_search_system/filters/journal_reputation_filter.py +298 -0
- local_deep_research/advanced_search_system/findings/repository.py +0 -3
- local_deep_research/advanced_search_system/strategies/base_strategy.py +1 -2
- local_deep_research/advanced_search_system/strategies/iterdrag_strategy.py +14 -18
- local_deep_research/advanced_search_system/strategies/parallel_search_strategy.py +4 -8
- local_deep_research/advanced_search_system/strategies/rapid_search_strategy.py +5 -6
- local_deep_research/advanced_search_system/strategies/source_based_strategy.py +2 -2
- local_deep_research/advanced_search_system/strategies/standard_strategy.py +9 -7
- local_deep_research/api/benchmark_functions.py +288 -0
- local_deep_research/api/research_functions.py +8 -4
- local_deep_research/benchmarks/README.md +162 -0
- local_deep_research/benchmarks/__init__.py +51 -0
- local_deep_research/benchmarks/benchmark_functions.py +353 -0
- local_deep_research/benchmarks/cli/__init__.py +16 -0
- local_deep_research/benchmarks/cli/benchmark_commands.py +338 -0
- local_deep_research/benchmarks/cli.py +347 -0
- local_deep_research/benchmarks/comparison/__init__.py +12 -0
- local_deep_research/benchmarks/comparison/evaluator.py +768 -0
- local_deep_research/benchmarks/datasets/__init__.py +53 -0
- local_deep_research/benchmarks/datasets/base.py +295 -0
- local_deep_research/benchmarks/datasets/browsecomp.py +116 -0
- local_deep_research/benchmarks/datasets/custom_dataset_template.py +98 -0
- local_deep_research/benchmarks/datasets/simpleqa.py +74 -0
- local_deep_research/benchmarks/datasets/utils.py +116 -0
- local_deep_research/benchmarks/datasets.py +31 -0
- local_deep_research/benchmarks/efficiency/__init__.py +14 -0
- local_deep_research/benchmarks/efficiency/resource_monitor.py +367 -0
- local_deep_research/benchmarks/efficiency/speed_profiler.py +214 -0
- local_deep_research/benchmarks/evaluators/__init__.py +18 -0
- local_deep_research/benchmarks/evaluators/base.py +74 -0
- local_deep_research/benchmarks/evaluators/browsecomp.py +83 -0
- local_deep_research/benchmarks/evaluators/composite.py +121 -0
- local_deep_research/benchmarks/evaluators/simpleqa.py +271 -0
- local_deep_research/benchmarks/graders.py +410 -0
- local_deep_research/benchmarks/metrics/README.md +80 -0
- local_deep_research/benchmarks/metrics/__init__.py +24 -0
- local_deep_research/benchmarks/metrics/calculation.py +385 -0
- local_deep_research/benchmarks/metrics/reporting.py +155 -0
- local_deep_research/benchmarks/metrics/visualization.py +205 -0
- local_deep_research/benchmarks/metrics.py +11 -0
- local_deep_research/benchmarks/optimization/__init__.py +32 -0
- local_deep_research/benchmarks/optimization/api.py +274 -0
- local_deep_research/benchmarks/optimization/metrics.py +20 -0
- local_deep_research/benchmarks/optimization/optuna_optimizer.py +1163 -0
- local_deep_research/benchmarks/runners.py +434 -0
- local_deep_research/benchmarks/templates.py +65 -0
- local_deep_research/config/llm_config.py +26 -23
- local_deep_research/config/search_config.py +1 -5
- local_deep_research/defaults/default_settings.json +108 -7
- local_deep_research/search_system.py +16 -8
- local_deep_research/utilities/db_utils.py +3 -6
- local_deep_research/utilities/es_utils.py +441 -0
- local_deep_research/utilities/log_utils.py +36 -0
- local_deep_research/utilities/search_utilities.py +8 -9
- local_deep_research/web/app.py +7 -9
- local_deep_research/web/app_factory.py +9 -12
- local_deep_research/web/database/migrations.py +8 -5
- local_deep_research/web/database/models.py +20 -0
- local_deep_research/web/database/schema_upgrade.py +5 -8
- local_deep_research/web/models/database.py +15 -18
- local_deep_research/web/routes/benchmark_routes.py +427 -0
- local_deep_research/web/routes/research_routes.py +13 -17
- local_deep_research/web/routes/settings_routes.py +264 -67
- local_deep_research/web/services/research_service.py +47 -57
- local_deep_research/web/services/settings_manager.py +1 -4
- local_deep_research/web/services/settings_service.py +4 -6
- local_deep_research/web/static/css/styles.css +12 -0
- local_deep_research/web/static/js/components/logpanel.js +164 -155
- local_deep_research/web/static/js/components/research.js +44 -3
- local_deep_research/web/static/js/components/settings.js +27 -0
- local_deep_research/web/static/js/services/socket.js +47 -0
- local_deep_research/web_search_engines/default_search_engines.py +38 -0
- local_deep_research/web_search_engines/engines/meta_search_engine.py +100 -33
- local_deep_research/web_search_engines/engines/search_engine_arxiv.py +31 -17
- local_deep_research/web_search_engines/engines/search_engine_brave.py +8 -3
- local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py +343 -0
- local_deep_research/web_search_engines/engines/search_engine_google_pse.py +14 -6
- local_deep_research/web_search_engines/engines/search_engine_local.py +19 -23
- local_deep_research/web_search_engines/engines/search_engine_local_all.py +9 -12
- local_deep_research/web_search_engines/engines/search_engine_searxng.py +12 -17
- local_deep_research/web_search_engines/engines/search_engine_serpapi.py +8 -4
- local_deep_research/web_search_engines/search_engine_base.py +22 -5
- local_deep_research/web_search_engines/search_engine_factory.py +32 -11
- local_deep_research/web_search_engines/search_engines_config.py +14 -1
- {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/METADATA +10 -2
- {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/RECORD +92 -49
- {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/WHEEL +0 -0
- {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/entry_points.txt +0 -0
- {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,10 +1,11 @@
|
|
1
|
+
import hashlib
|
1
2
|
import json
|
2
|
-
import logging
|
3
3
|
import os
|
4
4
|
import threading
|
5
|
-
import traceback
|
6
5
|
from datetime import datetime
|
7
6
|
|
7
|
+
from loguru import logger
|
8
|
+
|
8
9
|
from ...config.llm_config import get_llm
|
9
10
|
from ...config.search_config import get_search
|
10
11
|
from ...report_generator import IntegratedReportGenerator
|
@@ -13,9 +14,6 @@ from ...utilities.search_utilities import extract_links_from_search_results
|
|
13
14
|
from ..models.database import add_log_to_db, calculate_duration, get_db_connection
|
14
15
|
from .socket_service import emit_to_subscribers
|
15
16
|
|
16
|
-
# Initialize logger
|
17
|
-
logger = logging.getLogger(__name__)
|
18
|
-
|
19
17
|
# Output directory for research results
|
20
18
|
OUTPUT_DIR = "research_outputs"
|
21
19
|
|
@@ -70,6 +68,26 @@ def start_research_process(
|
|
70
68
|
return thread
|
71
69
|
|
72
70
|
|
71
|
+
def _generate_report_path(query: str) -> str:
|
72
|
+
"""
|
73
|
+
Generates a path for a new report file based on the query.
|
74
|
+
|
75
|
+
Args:
|
76
|
+
query: The query used for the report.
|
77
|
+
|
78
|
+
Returns:
|
79
|
+
The path that it generated.
|
80
|
+
|
81
|
+
"""
|
82
|
+
# Generate a unique filename that does not contain
|
83
|
+
# non-alphanumeric characters.
|
84
|
+
query_hash = hashlib.md5(query.encode("utf-8")).hexdigest()[:10]
|
85
|
+
return os.path.join(
|
86
|
+
OUTPUT_DIR,
|
87
|
+
f"research_report_{query_hash}_{datetime.now().isoformat()}.md",
|
88
|
+
)
|
89
|
+
|
90
|
+
|
73
91
|
def run_research_process(
|
74
92
|
research_id, query, mode, active_research, termination_flags, **kwargs
|
75
93
|
):
|
@@ -239,8 +257,8 @@ def run_research_process(
|
|
239
257
|
event_data["log_entry"] = log_entry
|
240
258
|
|
241
259
|
emit_to_subscribers("research_progress", research_id, event_data)
|
242
|
-
except Exception
|
243
|
-
logger.
|
260
|
+
except Exception:
|
261
|
+
logger.exception("Socket emit error (non-critical)")
|
244
262
|
|
245
263
|
# Function to check termination during long-running operations
|
246
264
|
def check_termination():
|
@@ -275,14 +293,12 @@ def run_research_process(
|
|
275
293
|
model_provider,
|
276
294
|
model,
|
277
295
|
)
|
278
|
-
except Exception
|
279
|
-
logger.
|
280
|
-
"Error setting LLM provider=%s, model=%s
|
296
|
+
except Exception:
|
297
|
+
logger.exception(
|
298
|
+
"Error setting LLM provider=%s, model=%s",
|
281
299
|
model_provider,
|
282
300
|
model,
|
283
|
-
str(e),
|
284
301
|
)
|
285
|
-
logger.error(traceback.format_exc())
|
286
302
|
|
287
303
|
# Set the progress callback in the system
|
288
304
|
system = AdvancedSearchSystem(llm=use_llm)
|
@@ -302,10 +318,8 @@ def run_research_process(
|
|
302
318
|
)
|
303
319
|
|
304
320
|
logger.info("Successfully set search engine to: %s", search_engine)
|
305
|
-
except Exception
|
306
|
-
logger.
|
307
|
-
"Error setting search engine to %s: %s", search_engine, str(e)
|
308
|
-
)
|
321
|
+
except Exception:
|
322
|
+
logger.exception("Error setting search engine to %s", search_engine)
|
309
323
|
|
310
324
|
# Run the search
|
311
325
|
progress_callback("Starting research process", 5, {"phase": "init"})
|
@@ -358,10 +372,8 @@ def run_research_process(
|
|
358
372
|
if isinstance(
|
359
373
|
raw_formatted_findings, str
|
360
374
|
) and raw_formatted_findings.startswith("Error:"):
|
361
|
-
|
362
|
-
|
363
|
-
logger.warning(
|
364
|
-
f"Detected error in formatted findings: {raw_formatted_findings[:100]}... stack trace: {traceback.format_exc()}"
|
375
|
+
logger.exception(
|
376
|
+
f"Detected error in formatted findings: {raw_formatted_findings[:100]}..."
|
365
377
|
)
|
366
378
|
|
367
379
|
# Determine error type for better user feedback
|
@@ -502,9 +514,9 @@ def run_research_process(
|
|
502
514
|
search_results
|
503
515
|
)
|
504
516
|
all_links.extend(links)
|
505
|
-
except Exception
|
506
|
-
logger.
|
507
|
-
|
517
|
+
except Exception:
|
518
|
+
logger.exception(
|
519
|
+
"Error processing search results/links"
|
508
520
|
)
|
509
521
|
|
510
522
|
logger.info(
|
@@ -523,15 +535,7 @@ def run_research_process(
|
|
523
535
|
if not os.path.exists(OUTPUT_DIR):
|
524
536
|
os.makedirs(OUTPUT_DIR)
|
525
537
|
|
526
|
-
|
527
|
-
x for x in query if x.isalnum() or x in [" ", "-", "_"]
|
528
|
-
)[:50]
|
529
|
-
safe_query = safe_query.replace(" ", "_").lower()
|
530
|
-
report_path = os.path.join(
|
531
|
-
OUTPUT_DIR,
|
532
|
-
f"quick_summary_{safe_query}_"
|
533
|
-
f"{int(datetime.now().timestamp())}.md",
|
534
|
-
)
|
538
|
+
report_path = _generate_report_path(query)
|
535
539
|
|
536
540
|
# Send progress update for writing to file
|
537
541
|
progress_callback(
|
@@ -607,10 +611,7 @@ def run_research_process(
|
|
607
611
|
logger.info("Resources cleaned up for research_id: %s", research_id)
|
608
612
|
|
609
613
|
except Exception as inner_e:
|
610
|
-
logger.
|
611
|
-
"Error during quick summary generation: %s", str(inner_e)
|
612
|
-
)
|
613
|
-
logger.error(traceback.format_exc())
|
614
|
+
logger.exception("Error during quick summary generation")
|
614
615
|
raise Exception(f"Error generating quick summary: {str(inner_e)}")
|
615
616
|
else:
|
616
617
|
raise Exception(
|
@@ -637,14 +638,7 @@ def run_research_process(
|
|
637
638
|
if not os.path.exists(OUTPUT_DIR):
|
638
639
|
os.makedirs(OUTPUT_DIR)
|
639
640
|
|
640
|
-
|
641
|
-
x for x in query if x.isalnum() or x in [" ", "-", "_"]
|
642
|
-
)[:50]
|
643
|
-
safe_query = safe_query.replace(" ", "_").lower()
|
644
|
-
report_path = os.path.join(
|
645
|
-
OUTPUT_DIR,
|
646
|
-
f"detailed_report_{safe_query}_{int(datetime.now().timestamp())}.md",
|
647
|
-
)
|
641
|
+
report_path = _generate_report_path(query)
|
648
642
|
|
649
643
|
with open(report_path, "w", encoding="utf-8") as f:
|
650
644
|
f.write(final_report["content"])
|
@@ -694,10 +688,7 @@ def run_research_process(
|
|
694
688
|
except Exception as e:
|
695
689
|
# Handle error
|
696
690
|
error_message = f"Research failed: {str(e)}"
|
697
|
-
logger.
|
698
|
-
import traceback
|
699
|
-
|
700
|
-
logger.error("Exception occurred:" + str(traceback.print_exc()))
|
691
|
+
logger.exception(error_message)
|
701
692
|
|
702
693
|
try:
|
703
694
|
# Check for common Ollama error patterns in the exception and provide more user-friendly errors
|
@@ -783,12 +774,11 @@ def run_research_process(
|
|
783
774
|
research_id,
|
784
775
|
{"status": status, "error": message},
|
785
776
|
)
|
786
|
-
except Exception
|
787
|
-
logger.
|
777
|
+
except Exception:
|
778
|
+
logger.exception("Failed to emit error via socket")
|
788
779
|
|
789
|
-
except Exception
|
790
|
-
logger.
|
791
|
-
logger.error(traceback.format_exc())
|
780
|
+
except Exception:
|
781
|
+
logger.exception("Error in error handler")
|
792
782
|
|
793
783
|
# Clean up resources
|
794
784
|
cleanup_research_resources(research_id, active_research, termination_flags)
|
@@ -817,8 +807,8 @@ def cleanup_research_resources(research_id, active_research, termination_flags):
|
|
817
807
|
if result and result[0]:
|
818
808
|
current_status = result[0]
|
819
809
|
conn.close()
|
820
|
-
except Exception
|
821
|
-
logger.
|
810
|
+
except Exception:
|
811
|
+
logger.exception("Error retrieving research status during cleanup")
|
822
812
|
|
823
813
|
# Remove from active research
|
824
814
|
if research_id in active_research:
|
@@ -860,8 +850,8 @@ def cleanup_research_resources(research_id, active_research, termination_flags):
|
|
860
850
|
|
861
851
|
emit_to_subscribers("research_progress", research_id, final_message)
|
862
852
|
|
863
|
-
except Exception
|
864
|
-
logger.error("Error sending final cleanup message
|
853
|
+
except Exception:
|
854
|
+
logger.error("Error sending final cleanup message")
|
865
855
|
|
866
856
|
|
867
857
|
def handle_termination(research_id, active_research, termination_flags):
|
@@ -1,9 +1,9 @@
|
|
1
1
|
import importlib.resources as pkg_resources
|
2
2
|
import json
|
3
|
-
import logging
|
4
3
|
import os
|
5
4
|
from typing import Any, Dict, Optional, Union
|
6
5
|
|
6
|
+
from loguru import logger
|
7
7
|
from sqlalchemy import func
|
8
8
|
from sqlalchemy.exc import SQLAlchemyError
|
9
9
|
from sqlalchemy.orm import Session
|
@@ -19,9 +19,6 @@ from ..models.settings import (
|
|
19
19
|
SearchSetting,
|
20
20
|
)
|
21
21
|
|
22
|
-
# Setup logging
|
23
|
-
logger = logging.getLogger(__name__)
|
24
|
-
|
25
22
|
|
26
23
|
def check_env_setting(key: str) -> str | None:
|
27
24
|
"""
|
@@ -1,12 +1,10 @@
|
|
1
|
-
import logging
|
2
1
|
from typing import Any, Dict, Optional, Union
|
3
2
|
|
3
|
+
from loguru import logger
|
4
|
+
|
4
5
|
from ..database.models import Setting
|
5
6
|
from .settings_manager import SettingsManager
|
6
7
|
|
7
|
-
# Initialize logger
|
8
|
-
logger = logging.getLogger(__name__)
|
9
|
-
|
10
8
|
|
11
9
|
def get_settings_manager(db_session=None):
|
12
10
|
"""
|
@@ -110,8 +108,8 @@ def bulk_update_settings(
|
|
110
108
|
if commit and success and manager.db_session:
|
111
109
|
try:
|
112
110
|
manager.db_session.commit()
|
113
|
-
except Exception
|
114
|
-
logger.
|
111
|
+
except Exception:
|
112
|
+
logger.exception("Error committing bulk settings update")
|
115
113
|
manager.db_session.rollback()
|
116
114
|
success = False
|
117
115
|
|
@@ -1383,6 +1383,18 @@ textarea:focus, input[type="text"]:focus {
|
|
1383
1383
|
border-color: rgba(255, 193, 7, 0.2);
|
1384
1384
|
}
|
1385
1385
|
|
1386
|
+
/* Styling for search engine selection log entries */
|
1387
|
+
.console-log-entry[data-log-type="info"][data-engine-selected="true"] {
|
1388
|
+
background-color: rgba(64, 191, 255, 0.05);
|
1389
|
+
border-left: 2px solid var(--accent-tertiary);
|
1390
|
+
padding-left: calc(0.5rem - 2px);
|
1391
|
+
font-weight: 500;
|
1392
|
+
}
|
1393
|
+
|
1394
|
+
.console-log-entry[data-log-type="info"][data-engine-selected="true"] .log-message {
|
1395
|
+
color: var(--accent-tertiary);
|
1396
|
+
}
|
1397
|
+
|
1386
1398
|
/* Update existing error message styling if needed */
|
1387
1399
|
.error-message {
|
1388
1400
|
display: none;
|