local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/defaults/search_engines.toml

@@ -2,12 +2,12 @@
 # This file defines all available search engines and their properties
 
 [wikipedia]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_wikipedia"
+module_path = ".engines.search_engine_wikipedia"
 class_name = "WikipediaSearchEngine"
 requires_api_key = false
 reliability = 0.95
 strengths = [
-    "factual information", "general knowledge", "definitions", 
+    "factual information", "general knowledge", "definitions",
     "historical facts", "biographies", "overview information"
 ]
 weaknesses = ["recent events", "specialized academic topics", "product comparisons"]
@@ -17,12 +17,12 @@ max_results = 20
 include_content = true
 
 [arxiv]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_arxiv"
+module_path = ".engines.search_engine_arxiv"
 class_name = "ArXivSearchEngine"
 requires_api_key = false
 reliability = 0.9
 strengths = [
-    "scientific papers", "academic research", "physics", "computer science", 
+    "scientific papers", "academic research", "physics", "computer science",
     "mathematics", "statistics", "machine learning", "preprints"
 ]
 weaknesses = ["non-academic topics", "consumer products", "news", "general information"]
@@ -33,17 +33,17 @@ sort_by = "relevance"
 sort_order = "descending"
 
 [pubmed]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_pubmed"
+module_path = ".engines.search_engine_pubmed"
 class_name = "PubMedSearchEngine"
 requires_api_key = false
 api_key_env = "NCBI_API_KEY"
 reliability = 0.98
 strengths = [
-    "biomedical literature", "medical research", "clinical studies", 
+    "biomedical literature", "medical research", "clinical studies",
     "life sciences", "health information", "scientific papers"
 ]
 weaknesses = [
-    "non-medical topics", "very recent papers may be missing", 
+    "non-medical topics", "very recent papers may be missing",
     "limited to published research"
 ]
 requires_llm = true
@@ -57,12 +57,12 @@ days_limit = 0
 optimize_queries = true
 
 [github]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_github"
+module_path = ".engines.search_engine_github"
 class_name = "GitHubSearchEngine"
 requires_api_key = false
 reliability = 0.99
 strengths = [
-    "code repositories", "software documentation", "open source projects", 
+    "code repositories", "software documentation", "open source projects",
     "programming issues", "developer information", "technical documentation"
 ]
 weaknesses = ["non-technical content", "content outside GitHub", "rate limits without API key"]
@@ -75,13 +75,13 @@ include_readme = true
 include_issues = false
 
 [serpapi]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_serpapi"
+module_path = ".engines.search_engine_serpapi"
 class_name = "SerpAPISearchEngine"
 requires_api_key = true
 api_key_env = "SERP_API_KEY"
 reliability = 0.6
 strengths = [
-    "comprehensive web search", "product information", "reviews", 
+    "comprehensive web search", "product information", "reviews",
     "recent content", "news", "broad coverage"
 ]
 weaknesses = ["requires API key with usage limits", "not specialized for academic content"]
@@ -95,23 +95,49 @@ time_period = "y"
 safe_search = true
 search_language = "English"
 
+[searxng]
+module_path = ".engines.search_engine_searxng"
+class_name = "SearXNGSearchEngine"
+requires_api_key = false
+api_key_env = "SEARXNG_INSTANCE"
+reliability = 0.9
+strengths = [
+    "privacy-focused", "metasearch engine", "self-hosted",
+    "no tracking", "configurable", "multiple engines in one"
+]
+weaknesses = [
+    "requires self-hosting", "depends on other search engines",
+    "may be rate limited by underlying engines"
+]
+supports_full_search = true
+full_search_module = ".engines.full_search"
+full_search_class = "FullSearchResults"
+
+[searxng.default_params]
+max_results = 15
+categories = ["general"]
+language = "en"
+safe_search = 1
+delay_between_requests = 0.0
+include_full_content = true
+
 [google_pse]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_google_pse"
+module_path = ".engines.search_engine_google_pse"
 class_name = "GooglePSESearchEngine"
 requires_api_key = true
 api_key_env = "GOOGLE_PSE_API_KEY"
 reliability = 0.9
 strengths = [
-    "custom search scope", "high-quality results", "domain-specific search", 
+    "custom search scope", "high-quality results", "domain-specific search",
     "configurable search experience", "control over search index"
 ]
 weaknesses = [
-    "requires API key with usage limits", 
+    "requires API key with usage limits",
     "limited to 10,000 queries/day on free tier",
     "requires search engine configuration in Google Control Panel"
 ]
 supports_full_search = true
-full_search_module = "local_deep_research.web_search_engines.engines.full_search"
+full_search_module = ".engines.full_search"
 full_search_class = "FullSearchResults"
 
 [google_pse.default_params]
@@ -120,13 +146,13 @@ safe_search = true
 search_language = "English"
 
 [brave]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_brave"
+module_path = ".engines.search_engine_brave"
 class_name = "BraveSearchEngine"
 requires_api_key = true
 api_key_env = "BRAVE_API_KEY"
 reliability = 0.7
 strengths = [
-    "privacy-focused web search", "product information", "reviews", 
+    "privacy-focused web search", "product information", "reviews",
     "recent content", "news", "broad coverage"
 ]
 weaknesses = ["requires API key with usage limits", "smaller index than Google"]
@@ -141,16 +167,16 @@ safe_search = true
 search_language = "English"
 
 [wayback]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_wayback"
+module_path = ".engines.search_engine_wayback"
 class_name = "WaybackSearchEngine"
 requires_api_key = false
 reliability = 0.5
 strengths = [
-    "historical web content", "archived websites", "content verification", 
+    "historical web content", "archived websites", "content verification",
     "deleted or changed web pages", "website evolution tracking"
 ]
 weaknesses = [
-    "limited to previously archived content", "may miss recent changes", 
+    "limited to previously archived content", "may miss recent changes",
     "archiving quality varies"
 ]
 supports_full_search = true
@@ -162,12 +188,12 @@ closest_only = false
 language = "English"
 
 [auto]
-module_path = "local_deep_research.web_search_engines.engines.meta_search_engine"
+module_path = ".engines.meta_search_engine"
 class_name = "MetaSearchEngine"
 requires_api_key = false
 reliability = 0.85
 strengths = [
-    "intelligent engine selection", "adaptable to query type", 
+    "intelligent engine selection", "adaptable to query type",
     "fallback capabilities"
 ]
 weaknesses = ["slightly slower due to LLM analysis"]
@@ -178,7 +204,7 @@ use_api_key_services = true
 max_engines_to_try = 3
 
 [local_all]
-module_path = "local_deep_research.web_search_engines.engines.search_engine_local_all"
+module_path = ".engines.search_engine_local_all"
 class_name = "LocalAllSearchEngine"
 requires_api_key = false
 reliability = 0.85
@@ -187,16 +213,16 @@ weaknesses = ["may return too many results", "requires indexing"]
 requires_llm = true
 
 #[semantic_scholar]
-#module_path = "local_deep_research.web_search_engines.engines.search_engine_semantic_scholar"
+#module_path = ".engines.search_engine_semantic_scholar"
 #class_name = "SemanticScholarSearchEngine"
 #requires_api_key = false
 #api_key_env = "S2_API_KEY"
 #reliability = 0.87
 #strengths = [
-#    "comprehensive scientific literature", 
-#    "extensive citation network", 
-#    "AI-generated summaries (TLDRs)", 
-#    "academic paper metadata", 
+#    "comprehensive scientific literature",
+#    "extensive citation network",
+#    "AI-generated summaries (TLDRs)",
+#    "academic paper metadata",
 #    "cross-disciplinary coverage",
 #    "200M+ papers across all fields",
 #    "usable without API key"
@@ -232,34 +258,34 @@ DEFAULT_SEARCH_ENGINE = "wikipedia"
 # Uncomment and modify these templates as needed
 
 # [duckduckgo]
-# module_path = "local_deep_research.web_search_engines.engines.search_engine_ddg"
+# module_path = ".engines.search_engine_ddg"
 # class_name = "DuckDuckGoSearchEngine"
 # requires_api_key = false
 # reliability = 0.4
 # strengths = [
-#     "web search", "product information", "reviews", "recent information", 
+#     "web search", "product information", "reviews", "recent information",
 #     "news", "general queries", "broad coverage"
 # ]
 # weaknesses = ["inconsistent due to rate limits", "not specialized for academic content"]
 # supports_full_search = true
 # full_search_module = "local_deep_research.web_search_engines.engines.full_search"
 # full_search_class = "FullSearchResults"
-# 
+#
 # [duckduckgo.default_params]
 # region = "us"
 # safe_search = true
 
 # [guardian]
-# module_path = "local_deep_research.web_search_engines.engines.search_engine_guardian"
+# module_path = ".engines.search_engine_guardian"
 # class_name = "GuardianSearchEngine"
 # requires_api_key = true
 # api_key_env = "GUARDIAN_API_KEY"
 # reliability = 0.5
 # strengths = [
-#     "news articles", "current events", "opinion pieces", "journalism", 
+#     "news articles", "current events", "opinion pieces", "journalism",
 #     "UK and global news", "political analysis"
 # ]
 # weaknesses = ["primarily focused on news", "limited historical content pre-1999"]
-# 
+#
 # [guardian.default_params]
 # order_by = "relevance"

local_deep_research/main.py

@@ -1,6 +1,11 @@
-from .search_system import AdvancedSearchSystem
+import logging
+import sys
 from typing import Dict
-from .config import settings
+
+from . import get_advanced_search_system, get_report_generator
+from .config.config_files import settings
+from .utilities.db_utils import get_db_setting
+
 
 def print_report(report: Dict):
     """Print and save the report in a readable format"""
@@ -11,8 +16,6 @@ def print_report(report: Dict):
     # Print content
     print(report["content"])
 
-
-
     # Save to file in markdown format
     with open("report.md", "w", encoding="utf-8") as markdown_file:
         # Write content
@@ -24,31 +27,40 @@ def print_report(report: Dict):
 
         markdown_file.write(f"- Query: {report['metadata']['query']}\n")
 
-    print(f"\nReport has been saved to report.md")
+    print("\nReport has been saved to report.md")
 
 
-from .report_generator import IntegratedReportGenerator
+# Create the report generator lazily to avoid circular imports
+def get_report_generator_instance():
+    return get_report_generator()
 
-report_generator = IntegratedReportGenerator()
 
+# report_generator = IntegratedReportGenerator()
+report_generator = None  # Will be initialized when needed
 
 
 def main():
-    import os
     import logging
-    from .utilties.setup_utils import setup_user_directories
-    
+
+    from .utilities.setup_utils import setup_user_directories
+
     # Configure logging
     logging.basicConfig(level=logging.INFO)
     logger = logging.getLogger(__name__)
-    logger.info(f"Starting with settings: iterations={settings.search.iterations}, "
-                f"questions_per_iteration={settings.search.questions_per_iteration}")
-        
+    search_iterations = get_db_setting("search.iterations", settings.search.iterations)
+    questions_per_iteration = get_db_setting(
+        "search.questions_per_iteration", settings.search.questions_per_iteration
+    )
+    logger.info(
+        f"Starting with settings: iterations={search_iterations}, "
+        f"questions_per_iteration={questions_per_iteration}"
+    )
+
     # Explicitly run setup
     logger.info("Initializing configuration...")
     setup_user_directories()
-    
-    system = AdvancedSearchSystem()
+
+    system = get_advanced_search_system()
 
     print("Welcome to the Advanced Research System")
     print("Type 'quit' to exit")
@@ -97,9 +109,12 @@ def main():
 
             else:
                 # Full Report
-                final_report = report_generator.generate_report(
-                    results, query
-                )
+                # Initialize report_generator if not already done
+                global report_generator
+                if report_generator is None:
+                    report_generator = get_report_generator()
+
+                final_report = report_generator.generate_report(results, query)
                 print("\n=== RESEARCH REPORT ===")
                 print_report(final_report)
 
@@ -109,5 +124,92 @@ def main():
         else:
             print("Research failed. Please try again.")
 
+
+# Add command for database migration
 if __name__ == "__main__":
-    main()
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Local Deep Research")
+    parser.add_argument("--web", action="store_true", help="Start the web server")
+    parser.add_argument(
+        "--migrate-db", action="store_true", help="Migrate legacy databases to ldr.db"
+    )
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+    parser.add_argument(
+        "--test-migration",
+        action="store_true",
+        help="Test migration by checking database contents",
+    )
+    parser.add_argument(
+        "--schema-upgrade",
+        action="store_true",
+        help="Run schema upgrades on the database (e.g., remove redundant tables)",
+    )
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    if args.migrate_db:
+        try:
+            # First ensure data directory exists
+            from src.local_deep_research.setup_data_dir import setup_data_dir
+
+            setup_data_dir()
+
+            # Then run the migration
+            from src.local_deep_research.web.database.migrate_to_ldr_db import (
+                migrate_to_ldr_db,
+            )
+
+            print("Starting database migration...")
+            success = migrate_to_ldr_db()
+            if success:
+                print("Database migration completed successfully")
+                sys.exit(0)
+            else:
+                print("Database migration failed")
+                sys.exit(1)
+        except Exception as e:
+            print(f"Error running database migration: {e}")
+            sys.exit(1)
+
+    if args.test_migration:
+        try:
+            from src.local_deep_research.test_migration import main as test_main
+
+            sys.exit(test_main())
+        except Exception as e:
+            print(f"Error running migration test: {e}")
+            sys.exit(1)
+
+    if args.schema_upgrade:
+        try:
+            from src.local_deep_research.web.database.schema_upgrade import (
+                run_schema_upgrades,
+            )
+
+            print("Running database schema upgrades...")
+            success = run_schema_upgrades()
+            if success:
+                print("Schema upgrades completed successfully")
+                sys.exit(0)
+            else:
+                print("Schema upgrades failed")
+                sys.exit(1)
+        except Exception as e:
+            print(f"Error running schema upgrades: {e}")
+            sys.exit(1)
+
+    if args.web:
+        from src.local_deep_research.web.app import main as web_main
+
+        web_main()
+    else:
+        # Default to web if no command specified
+        from src.local_deep_research.web.app import main as web_main
+
+        web_main()

local_deep_research/migrate_db.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+"""
+Database migration script for Local Deep Research.
+Migrates data from legacy databases (deep_research.db and research_history.db) to the new unified database (ldr.db).
+"""
+
+import argparse
+import logging
+import os
+import sys
+
+try:
+    from local_deep_research.web.database.migrate_to_ldr_db import (
+        migrate_to_ldr_db,
+    )
+    from local_deep_research.web.models.database import (
+        DB_PATH,
+        LEGACY_DEEP_RESEARCH_DB,
+        LEGACY_RESEARCH_HISTORY_DB,
+    )
+except ImportError:
+    # If that fails, try with the relative path.
+    from .web.database.migrate_to_ldr_db import migrate_to_ldr_db
+    from .web.models.database import (
+        DB_PATH,
+        LEGACY_DEEP_RESEARCH_DB,
+        LEGACY_RESEARCH_HISTORY_DB,
+    )
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger("migrate_db")
+
+# Add proper paths for import
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(os.path.dirname(current_dir)))
+
+
+def main():
+    """Main migration function that parses arguments and runs the migration"""
+    parser = argparse.ArgumentParser(
+        description="Local Deep Research Database Migration"
+    )
+    parser.add_argument(
+        "--backup",
+        action="store_true",
+        help="Create backup of existing databases before migration",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force migration even if target database exists",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Only check what would be migrated, don't perform actual migration",
+    )
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    print("=" * 80)
+    print("LOCAL DEEP RESEARCH DATABASE MIGRATION")
+    print("=" * 80)
+
+    try:
+        # First try the normal import
+        print(f"Target database will be created at: {DB_PATH}")
+
+        # Check if migration is needed
+        if os.path.exists(DB_PATH) and not args.force:
+            print(f"Target database already exists at: {DB_PATH}")
+            if (
+                input(
+                    "Do you want to continue anyway? This may overwrite data. (y/n): "
+                ).lower()
+                != "y"
+            ):
+                print("Migration aborted.")
+                return 1
+
+        # Check if source databases exist
+        deep_research_exists = os.path.exists(LEGACY_DEEP_RESEARCH_DB)
+        research_history_exists = os.path.exists(LEGACY_RESEARCH_HISTORY_DB)
+
+        if not deep_research_exists and not research_history_exists:
+            print("No legacy databases found. Nothing to migrate.")
+            return 0
+
+        print("Found legacy databases:")
+        if deep_research_exists:
+            print(f"  - {LEGACY_DEEP_RESEARCH_DB}")
+        if research_history_exists:
+            print(f"  - {LEGACY_RESEARCH_HISTORY_DB}")
+
+        # Create backups if requested
+        if args.backup:
+            if deep_research_exists:
+                backup_path = f"{LEGACY_DEEP_RESEARCH_DB}.bak"
+                import shutil
+
+                shutil.copy2(LEGACY_DEEP_RESEARCH_DB, backup_path)
+                print(f"Created backup: {backup_path}")
+
+            if research_history_exists:
+                backup_path = f"{LEGACY_RESEARCH_HISTORY_DB}.bak"
+                import shutil
+
+                shutil.copy2(LEGACY_RESEARCH_HISTORY_DB, backup_path)
+                print(f"Created backup: {backup_path}")
+
+        # Run migration or dry run
+        if args.dry_run:
+            print("\nDRY RUN - No changes will be made.")
+            print(f"Would migrate data to: {DB_PATH}")
+            return 0
+        else:
+            print(f"\nStarting migration to: {DB_PATH}")
+
+            success = migrate_to_ldr_db()
+
+            if success:
+                print("\nMigration completed successfully.")
+                print(
+                    "You can now start the application with the new unified database."
+                )
+                return 0
+            else:
+                print("\nMigration failed. Check the logs for details.")
+                return 1
+
+    except Exception as e:
+        logger.error(f"Migration error: {e}", exc_info=True)
+        print(f"Error during migration: {e}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())