local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. local_deep_research/__version__.py +1 -1
  2. local_deep_research/advanced_search_system/filters/base_filter.py +2 -3
  3. local_deep_research/advanced_search_system/filters/cross_engine_filter.py +4 -5
  4. local_deep_research/advanced_search_system/filters/journal_reputation_filter.py +298 -0
  5. local_deep_research/advanced_search_system/findings/repository.py +0 -3
  6. local_deep_research/advanced_search_system/strategies/base_strategy.py +1 -2
  7. local_deep_research/advanced_search_system/strategies/iterdrag_strategy.py +14 -18
  8. local_deep_research/advanced_search_system/strategies/parallel_search_strategy.py +4 -8
  9. local_deep_research/advanced_search_system/strategies/rapid_search_strategy.py +5 -6
  10. local_deep_research/advanced_search_system/strategies/source_based_strategy.py +2 -2
  11. local_deep_research/advanced_search_system/strategies/standard_strategy.py +9 -7
  12. local_deep_research/api/benchmark_functions.py +288 -0
  13. local_deep_research/api/research_functions.py +8 -4
  14. local_deep_research/benchmarks/README.md +162 -0
  15. local_deep_research/benchmarks/__init__.py +51 -0
  16. local_deep_research/benchmarks/benchmark_functions.py +353 -0
  17. local_deep_research/benchmarks/cli/__init__.py +16 -0
  18. local_deep_research/benchmarks/cli/benchmark_commands.py +338 -0
  19. local_deep_research/benchmarks/cli.py +347 -0
  20. local_deep_research/benchmarks/comparison/__init__.py +12 -0
  21. local_deep_research/benchmarks/comparison/evaluator.py +768 -0
  22. local_deep_research/benchmarks/datasets/__init__.py +53 -0
  23. local_deep_research/benchmarks/datasets/base.py +295 -0
  24. local_deep_research/benchmarks/datasets/browsecomp.py +116 -0
  25. local_deep_research/benchmarks/datasets/custom_dataset_template.py +98 -0
  26. local_deep_research/benchmarks/datasets/simpleqa.py +74 -0
  27. local_deep_research/benchmarks/datasets/utils.py +116 -0
  28. local_deep_research/benchmarks/datasets.py +31 -0
  29. local_deep_research/benchmarks/efficiency/__init__.py +14 -0
  30. local_deep_research/benchmarks/efficiency/resource_monitor.py +367 -0
  31. local_deep_research/benchmarks/efficiency/speed_profiler.py +214 -0
  32. local_deep_research/benchmarks/evaluators/__init__.py +18 -0
  33. local_deep_research/benchmarks/evaluators/base.py +74 -0
  34. local_deep_research/benchmarks/evaluators/browsecomp.py +83 -0
  35. local_deep_research/benchmarks/evaluators/composite.py +121 -0
  36. local_deep_research/benchmarks/evaluators/simpleqa.py +271 -0
  37. local_deep_research/benchmarks/graders.py +410 -0
  38. local_deep_research/benchmarks/metrics/README.md +80 -0
  39. local_deep_research/benchmarks/metrics/__init__.py +24 -0
  40. local_deep_research/benchmarks/metrics/calculation.py +385 -0
  41. local_deep_research/benchmarks/metrics/reporting.py +155 -0
  42. local_deep_research/benchmarks/metrics/visualization.py +205 -0
  43. local_deep_research/benchmarks/metrics.py +11 -0
  44. local_deep_research/benchmarks/optimization/__init__.py +32 -0
  45. local_deep_research/benchmarks/optimization/api.py +274 -0
  46. local_deep_research/benchmarks/optimization/metrics.py +20 -0
  47. local_deep_research/benchmarks/optimization/optuna_optimizer.py +1163 -0
  48. local_deep_research/benchmarks/runners.py +434 -0
  49. local_deep_research/benchmarks/templates.py +65 -0
  50. local_deep_research/config/llm_config.py +26 -23
  51. local_deep_research/config/search_config.py +1 -5
  52. local_deep_research/defaults/default_settings.json +108 -7
  53. local_deep_research/search_system.py +16 -8
  54. local_deep_research/utilities/db_utils.py +3 -6
  55. local_deep_research/utilities/es_utils.py +441 -0
  56. local_deep_research/utilities/log_utils.py +36 -0
  57. local_deep_research/utilities/search_utilities.py +8 -9
  58. local_deep_research/web/app.py +7 -9
  59. local_deep_research/web/app_factory.py +9 -12
  60. local_deep_research/web/database/migrations.py +8 -5
  61. local_deep_research/web/database/models.py +20 -0
  62. local_deep_research/web/database/schema_upgrade.py +5 -8
  63. local_deep_research/web/models/database.py +15 -18
  64. local_deep_research/web/routes/benchmark_routes.py +427 -0
  65. local_deep_research/web/routes/research_routes.py +13 -17
  66. local_deep_research/web/routes/settings_routes.py +264 -67
  67. local_deep_research/web/services/research_service.py +47 -57
  68. local_deep_research/web/services/settings_manager.py +1 -4
  69. local_deep_research/web/services/settings_service.py +4 -6
  70. local_deep_research/web/static/css/styles.css +12 -0
  71. local_deep_research/web/static/js/components/logpanel.js +164 -155
  72. local_deep_research/web/static/js/components/research.js +44 -3
  73. local_deep_research/web/static/js/components/settings.js +27 -0
  74. local_deep_research/web/static/js/services/socket.js +47 -0
  75. local_deep_research/web_search_engines/default_search_engines.py +38 -0
  76. local_deep_research/web_search_engines/engines/meta_search_engine.py +100 -33
  77. local_deep_research/web_search_engines/engines/search_engine_arxiv.py +31 -17
  78. local_deep_research/web_search_engines/engines/search_engine_brave.py +8 -3
  79. local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py +343 -0
  80. local_deep_research/web_search_engines/engines/search_engine_google_pse.py +14 -6
  81. local_deep_research/web_search_engines/engines/search_engine_local.py +19 -23
  82. local_deep_research/web_search_engines/engines/search_engine_local_all.py +9 -12
  83. local_deep_research/web_search_engines/engines/search_engine_searxng.py +12 -17
  84. local_deep_research/web_search_engines/engines/search_engine_serpapi.py +8 -4
  85. local_deep_research/web_search_engines/search_engine_base.py +22 -5
  86. local_deep_research/web_search_engines/search_engine_factory.py +32 -11
  87. local_deep_research/web_search_engines/search_engines_config.py +14 -1
  88. {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/METADATA +10 -2
  89. {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/RECORD +92 -49
  90. {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/WHEEL +0 -0
  91. {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/entry_points.txt +0 -0
  92. {local_deep_research-0.3.12.dist-info → local_deep_research-0.4.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,10 +1,11 @@
1
+ import hashlib
1
2
  import json
2
- import logging
3
3
  import os
4
4
  import threading
5
- import traceback
6
5
  from datetime import datetime
7
6
 
7
+ from loguru import logger
8
+
8
9
  from ...config.llm_config import get_llm
9
10
  from ...config.search_config import get_search
10
11
  from ...report_generator import IntegratedReportGenerator
@@ -13,9 +14,6 @@ from ...utilities.search_utilities import extract_links_from_search_results
13
14
  from ..models.database import add_log_to_db, calculate_duration, get_db_connection
14
15
  from .socket_service import emit_to_subscribers
15
16
 
16
- # Initialize logger
17
- logger = logging.getLogger(__name__)
18
-
19
17
  # Output directory for research results
20
18
  OUTPUT_DIR = "research_outputs"
21
19
 
@@ -70,6 +68,26 @@ def start_research_process(
70
68
  return thread
71
69
 
72
70
 
71
+ def _generate_report_path(query: str) -> str:
72
+ """
73
+ Generates a path for a new report file based on the query.
74
+
75
+ Args:
76
+ query: The query used for the report.
77
+
78
+ Returns:
79
+ The path that it generated.
80
+
81
+ """
82
+ # Generate a unique filename that does not contain
83
+ # non-alphanumeric characters.
84
+ query_hash = hashlib.md5(query.encode("utf-8")).hexdigest()[:10]
85
+ return os.path.join(
86
+ OUTPUT_DIR,
87
+ f"research_report_{query_hash}_{datetime.now().isoformat()}.md",
88
+ )
89
+
90
+
73
91
  def run_research_process(
74
92
  research_id, query, mode, active_research, termination_flags, **kwargs
75
93
  ):
@@ -239,8 +257,8 @@ def run_research_process(
239
257
  event_data["log_entry"] = log_entry
240
258
 
241
259
  emit_to_subscribers("research_progress", research_id, event_data)
242
- except Exception as e:
243
- logger.error(f"Socket emit error (non-critical): {str(e)}")
260
+ except Exception:
261
+ logger.exception("Socket emit error (non-critical)")
244
262
 
245
263
  # Function to check termination during long-running operations
246
264
  def check_termination():
@@ -275,14 +293,12 @@ def run_research_process(
275
293
  model_provider,
276
294
  model,
277
295
  )
278
- except Exception as e:
279
- logger.error(
280
- "Error setting LLM provider=%s, model=%s: %s",
296
+ except Exception:
297
+ logger.exception(
298
+ "Error setting LLM provider=%s, model=%s",
281
299
  model_provider,
282
300
  model,
283
- str(e),
284
301
  )
285
- logger.error(traceback.format_exc())
286
302
 
287
303
  # Set the progress callback in the system
288
304
  system = AdvancedSearchSystem(llm=use_llm)
@@ -302,10 +318,8 @@ def run_research_process(
302
318
  )
303
319
 
304
320
  logger.info("Successfully set search engine to: %s", search_engine)
305
- except Exception as e:
306
- logger.error(
307
- "Error setting search engine to %s: %s", search_engine, str(e)
308
- )
321
+ except Exception:
322
+ logger.exception("Error setting search engine to %s", search_engine)
309
323
 
310
324
  # Run the search
311
325
  progress_callback("Starting research process", 5, {"phase": "init"})
@@ -358,10 +372,8 @@ def run_research_process(
358
372
  if isinstance(
359
373
  raw_formatted_findings, str
360
374
  ) and raw_formatted_findings.startswith("Error:"):
361
- import traceback
362
-
363
- logger.warning(
364
- f"Detected error in formatted findings: {raw_formatted_findings[:100]}... stack trace: {traceback.format_exc()}"
375
+ logger.exception(
376
+ f"Detected error in formatted findings: {raw_formatted_findings[:100]}..."
365
377
  )
366
378
 
367
379
  # Determine error type for better user feedback
@@ -502,9 +514,9 @@ def run_research_process(
502
514
  search_results
503
515
  )
504
516
  all_links.extend(links)
505
- except Exception as link_err:
506
- logger.error(
507
- f"Error processing search results/links: {link_err}"
517
+ except Exception:
518
+ logger.exception(
519
+ "Error processing search results/links"
508
520
  )
509
521
 
510
522
  logger.info(
@@ -523,15 +535,7 @@ def run_research_process(
523
535
  if not os.path.exists(OUTPUT_DIR):
524
536
  os.makedirs(OUTPUT_DIR)
525
537
 
526
- safe_query = "".join(
527
- x for x in query if x.isalnum() or x in [" ", "-", "_"]
528
- )[:50]
529
- safe_query = safe_query.replace(" ", "_").lower()
530
- report_path = os.path.join(
531
- OUTPUT_DIR,
532
- f"quick_summary_{safe_query}_"
533
- f"{int(datetime.now().timestamp())}.md",
534
- )
538
+ report_path = _generate_report_path(query)
535
539
 
536
540
  # Send progress update for writing to file
537
541
  progress_callback(
@@ -607,10 +611,7 @@ def run_research_process(
607
611
  logger.info("Resources cleaned up for research_id: %s", research_id)
608
612
 
609
613
  except Exception as inner_e:
610
- logger.error(
611
- "Error during quick summary generation: %s", str(inner_e)
612
- )
613
- logger.error(traceback.format_exc())
614
+ logger.exception("Error during quick summary generation")
614
615
  raise Exception(f"Error generating quick summary: {str(inner_e)}")
615
616
  else:
616
617
  raise Exception(
@@ -637,14 +638,7 @@ def run_research_process(
637
638
  if not os.path.exists(OUTPUT_DIR):
638
639
  os.makedirs(OUTPUT_DIR)
639
640
 
640
- safe_query = "".join(
641
- x for x in query if x.isalnum() or x in [" ", "-", "_"]
642
- )[:50]
643
- safe_query = safe_query.replace(" ", "_").lower()
644
- report_path = os.path.join(
645
- OUTPUT_DIR,
646
- f"detailed_report_{safe_query}_{int(datetime.now().timestamp())}.md",
647
- )
641
+ report_path = _generate_report_path(query)
648
642
 
649
643
  with open(report_path, "w", encoding="utf-8") as f:
650
644
  f.write(final_report["content"])
@@ -694,10 +688,7 @@ def run_research_process(
694
688
  except Exception as e:
695
689
  # Handle error
696
690
  error_message = f"Research failed: {str(e)}"
697
- logger.error(error_message)
698
- import traceback
699
-
700
- logger.error("Exception occurred:" + str(traceback.print_exc()))
691
+ logger.exception(error_message)
701
692
 
702
693
  try:
703
694
  # Check for common Ollama error patterns in the exception and provide more user-friendly errors
@@ -783,12 +774,11 @@ def run_research_process(
783
774
  research_id,
784
775
  {"status": status, "error": message},
785
776
  )
786
- except Exception as socket_error:
787
- logger.error(f"Failed to emit error via socket: {str(socket_error)}")
777
+ except Exception:
778
+ logger.exception("Failed to emit error via socket")
788
779
 
789
- except Exception as inner_e:
790
- logger.error(f"Error in error handler: {str(inner_e)}")
791
- logger.error(traceback.format_exc())
780
+ except Exception:
781
+ logger.exception("Error in error handler")
792
782
 
793
783
  # Clean up resources
794
784
  cleanup_research_resources(research_id, active_research, termination_flags)
@@ -817,8 +807,8 @@ def cleanup_research_resources(research_id, active_research, termination_flags):
817
807
  if result and result[0]:
818
808
  current_status = result[0]
819
809
  conn.close()
820
- except Exception as e:
821
- logger.error("Error retrieving research status during cleanup: %s", e)
810
+ except Exception:
811
+ logger.exception("Error retrieving research status during cleanup")
822
812
 
823
813
  # Remove from active research
824
814
  if research_id in active_research:
@@ -860,8 +850,8 @@ def cleanup_research_resources(research_id, active_research, termination_flags):
860
850
 
861
851
  emit_to_subscribers("research_progress", research_id, final_message)
862
852
 
863
- except Exception as e:
864
- logger.error("Error sending final cleanup message: %s", e)
853
+ except Exception:
854
+ logger.error("Error sending final cleanup message")
865
855
 
866
856
 
867
857
  def handle_termination(research_id, active_research, termination_flags):
@@ -1,9 +1,9 @@
1
1
  import importlib.resources as pkg_resources
2
2
  import json
3
- import logging
4
3
  import os
5
4
  from typing import Any, Dict, Optional, Union
6
5
 
6
+ from loguru import logger
7
7
  from sqlalchemy import func
8
8
  from sqlalchemy.exc import SQLAlchemyError
9
9
  from sqlalchemy.orm import Session
@@ -19,9 +19,6 @@ from ..models.settings import (
19
19
  SearchSetting,
20
20
  )
21
21
 
22
- # Setup logging
23
- logger = logging.getLogger(__name__)
24
-
25
22
 
26
23
  def check_env_setting(key: str) -> str | None:
27
24
  """
@@ -1,12 +1,10 @@
1
- import logging
2
1
  from typing import Any, Dict, Optional, Union
3
2
 
3
+ from loguru import logger
4
+
4
5
  from ..database.models import Setting
5
6
  from .settings_manager import SettingsManager
6
7
 
7
- # Initialize logger
8
- logger = logging.getLogger(__name__)
9
-
10
8
 
11
9
  def get_settings_manager(db_session=None):
12
10
  """
@@ -110,8 +108,8 @@ def bulk_update_settings(
110
108
  if commit and success and manager.db_session:
111
109
  try:
112
110
  manager.db_session.commit()
113
- except Exception as e:
114
- logger.error(f"Error committing bulk settings update: {e}")
111
+ except Exception:
112
+ logger.exception("Error committing bulk settings update")
115
113
  manager.db_session.rollback()
116
114
  success = False
117
115
 
@@ -1383,6 +1383,18 @@ textarea:focus, input[type="text"]:focus {
1383
1383
  border-color: rgba(255, 193, 7, 0.2);
1384
1384
  }
1385
1385
 
1386
+ /* Styling for search engine selection log entries */
1387
+ .console-log-entry[data-log-type="info"][data-engine-selected="true"] {
1388
+ background-color: rgba(64, 191, 255, 0.05);
1389
+ border-left: 2px solid var(--accent-tertiary);
1390
+ padding-left: calc(0.5rem - 2px);
1391
+ font-weight: 500;
1392
+ }
1393
+
1394
+ .console-log-entry[data-log-type="info"][data-engine-selected="true"] .log-message {
1395
+ color: var(--accent-tertiary);
1396
+ }
1397
+
1386
1398
  /* Update existing error message styling if needed */
1387
1399
  .error-message {
1388
1400
  display: none;