local-deep-research 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

local_deep_research/search_system.py

@@ -18,7 +18,6 @@ from .advanced_search_system.strategies.source_based_strategy import (
 )
 from .advanced_search_system.strategies.standard_strategy import StandardSearchStrategy
 from .citation_handler import CitationHandler
-from .config.config_files import settings
 from .config.llm_config import get_llm
 from .config.search_config import get_search
 from .utilities.db_utils import get_db_setting
@@ -58,11 +57,12 @@ class AdvancedSearchSystem:
         self.search = search
         if search is None:
             self.search = get_search(llm_instance=self.model)
-        self.max_iterations = get_db_setting(
-            "search.iterations", settings.search.iterations
-        )
+
+        # Get iterations setting
+        self.max_iterations = get_db_setting("search.iterations", 1)
+
         self.questions_per_iteration = get_db_setting(
-            "search.questions_per_iteration", settings.search.questions_per_iteration
+            "search.questions_per_iteration", 3
         )
 
         # Log the strategy name that's being used
@@ -74,11 +74,19 @@ class AdvancedSearchSystem:
         self.citation_handler = CitationHandler(self.model)
         self.question_generator = StandardQuestionGenerator(self.model)
         self.findings_repository = FindingsRepository(self.model)
+        # For backward compatibility
+        self.questions_by_iteration = list()
+        self.progress_callback = lambda _1, _2, _3: None
+        self.all_links_of_system = list()
 
         # Initialize strategy based on name
         if strategy_name.lower() == "iterdrag":
             logger.info("Creating IterDRAGStrategy instance")
-            self.strategy = IterDRAGStrategy(model=self.model, search=self.search)
+            self.strategy = IterDRAGStrategy(
+                model=self.model,
+                search=self.search,
+                all_links_of_system=self.all_links_of_system,
+            )
         elif strategy_name.lower() == "source-based":
             logger.info("Creating SourceBasedSearchStrategy instance")
             self.strategy = SourceBasedSearchStrategy(
@@ -86,6 +94,7 @@ class AdvancedSearchSystem:
                 search=self.search,
                 include_text_content=include_text_content,
                 use_cross_engine_filter=use_cross_engine_filter,
+                all_links_of_system=self.all_links_of_system,
             )
         elif strategy_name.lower() == "parallel":
             logger.info("Creating ParallelSearchStrategy instance")
@@ -94,22 +103,26 @@ class AdvancedSearchSystem:
                 search=self.search,
                 include_text_content=include_text_content,
                 use_cross_engine_filter=use_cross_engine_filter,
+                all_links_of_system=self.all_links_of_system,
             )
         elif strategy_name.lower() == "rapid":
             logger.info("Creating RapidSearchStrategy instance")
-            self.strategy = RapidSearchStrategy(model=self.model, search=self.search)
+            self.strategy = RapidSearchStrategy(
+                model=self.model,
+                search=self.search,
+                all_links_of_system=self.all_links_of_system,
+            )
         else:
             logger.info("Creating StandardSearchStrategy instance")
-            self.strategy = StandardSearchStrategy(model=self.model, search=self.search)
+            self.strategy = StandardSearchStrategy(
+                model=self.model,
+                search=self.search,
+                all_links_of_system=self.all_links_of_system,
+            )
 
         # Log the actual strategy class
         logger.info(f"Created strategy of type: {type(self.strategy).__name__}")
 
-        # For backward compatibility
-        self.questions_by_iteration = {}
-        self.progress_callback = lambda _1, _2, _3: None
-        self.all_links_of_system = list()
-
         # Configure the strategy with our attributes
         if hasattr(self, "progress_callback") and self.progress_callback:
             self.strategy.set_progress_callback(self.progress_callback)
@@ -163,25 +176,15 @@ class AdvancedSearchSystem:
         result = self.strategy.analyze_topic(query)
 
         # Update our attributes for backward compatibility
-        if hasattr(self.strategy, "questions_by_iteration"):
-            self.questions_by_iteration = self.strategy.questions_by_iteration
-            # Send progress message with search info
-            self.progress_callback(
-                f"Processed questions: {self.strategy.questions_by_iteration}",
-                2,  # Low percentage to show this as an early step
-                {
-                    "phase": "setup",
-                    "search_info": {
-                        "questions_by_iteration": len(
-                            self.strategy.questions_by_iteration
-                        )
-                    },
-                },
-            )
-        if hasattr(self.strategy, "all_links_of_system"):
-            self.all_links_of_system = self.strategy.all_links_of_system
+
+        self.questions_by_iteration = self.strategy.questions_by_iteration.copy()
+        # Send progress message with search info
+
+        # if hasattr(self.strategy, "all_links_of_system"):
+        self.all_links_of_system.extend(self.strategy.all_links_of_system)
 
         # Include the search system instance for access to citations
         result["search_system"] = self
-
+        result["all_links_of_system"] = self.all_links_of_system
+        result["questions_by_iteration"] = self.questions_by_iteration
         return result

local_deep_research/utilities/db_utils.py

@@ -1,11 +1,12 @@
 import logging
 import os
 from functools import cache
+from typing import Any, Dict
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session, sessionmaker
 
-from ..web.services.settings_manager import SettingsManager
+from ..web.services.settings_manager import SettingsManager, check_env_setting
 
 logger = logging.getLogger(__name__)
 
@@ -36,8 +37,24 @@ def get_settings_manager() -> SettingsManager:
     return SettingsManager(db_session=get_db_session())
 
 
-def get_db_setting(key, default_value=None):
-    """Get a setting from the database with fallback to default value"""
+def get_db_setting(
+    key: str, default_value: Any | None = None, check_env: bool = True
+) -> str | Dict[str, Any] | None:
+    """
+    Get a setting from the database with fallback to default value
+
+    Args:
+        key: The setting key.
+        default_value: If the setting is not found, it will return this instead.
+        check_env: If true, it will check the corresponding environment
+            variable before checking the DB and return that if it is set.
+
+    """
+    if check_env:
+        env_value = check_env_setting(key)
+        if env_value is not None:
+            return env_value
+
     try:
         # Get settings manager which handles database access
         value = get_settings_manager().get_setting(key)
@@ -46,4 +63,6 @@ def get_db_setting(key, default_value=None):
             return value
     except Exception as e:
         logger.error(f"Error getting setting {key} from database: {e}")
+
+    logger.warning(f"Could not find setting '{key}' in the database.")
     return default_value

local_deep_research/utilities/search_utilities.py

@@ -43,21 +43,22 @@ def extract_links_from_search_results(search_results: List[Dict]) -> List[Dict]:
     return links
 
 
-def format_links(links: List[Dict]) -> str:
-    formatted_links = format_links_to_markdown(links)
-    return formatted_links
-
-
 def format_links_to_markdown(all_links: List[Dict]) -> str:
     formatted_text = ""
+    logger.info(f"Formatting {len(all_links)} links to markdown...")
+
     if all_links:
 
         # Group links by URL and collect all their indices
         url_to_indices = {}
         for link in all_links:
             url = link.get("url")
+            if url is None:
+                url = link.get("link")
             index = link.get("index", "")
+            # logger.info(f"URL \n {str(url)} ")
             if url:
+
                 if url not in url_to_indices:
                     url_to_indices[url] = []
                 url_to_indices[url].append(index)
@@ -66,6 +67,8 @@ def format_links_to_markdown(all_links: List[Dict]) -> str:
         seen_urls = set()  # Initialize the set here
         for link in all_links:
             url = link.get("url")
+            if url is None:
+                url = link.get("link")
             title = link.get("title", "Untitled")
             if url and url not in seen_urls:
                 # Get all indices for this URL
@@ -211,7 +214,7 @@ def format_findings(
                     links = extract_links_from_search_results(search_results)
                     if links:
                         formatted_text += "### SOURCES USED IN THIS SECTION:\n"
-                        formatted_text += format_links(links) + "\n\n"
+                        formatted_text += format_links_to_markdown(links) + "\n\n"
                 except Exception as link_err:
                     logger.error(
                         f"Error processing search results/links for finding {idx}: {link_err}"
@@ -238,5 +241,5 @@ def print_search_results(search_results):
     formatted_text = ""
     links = extract_links_from_search_results(search_results)
     if links:
-        formatted_text = format_links(links=links)
+        formatted_text = format_links_to_markdown(links=links)
     logger.info(formatted_text)

local_deep_research/web/app.py

@@ -2,7 +2,6 @@ import logging
 import os
 import sys
 
-from ..config.config_files import settings
 from ..setup_data_dir import setup_data_dir
 from ..utilities.db_utils import get_db_setting
 from .app_factory import create_app
@@ -92,28 +91,9 @@ def main():
                 print("Please run migration manually.")
 
     # Get web server settings with defaults
-    port = get_db_setting("web.port", settings.web.port)
-    host = get_db_setting("web.host", settings.web.host)
-    debug = get_db_setting("web.debug", settings.web.debug)
-
-    # Check for OpenAI availability but don't import it unless necessary
-    try:
-        api_key = os.environ.get("OPENAI_API_KEY")
-        if api_key:
-            try:
-                # Only try to import if we have an API key
-                import openai
-
-                openai.api_key = api_key
-                logger.info("OpenAI integration is available")
-            except ImportError:
-                logger.info("OpenAI package not installed, integration disabled")
-        else:
-            logger.info(
-                "OPENAI_API_KEY not found in environment variables, OpenAI integration disabled"
-            )
-    except Exception as e:
-        logger.error(f"Error checking OpenAI availability: {e}")
+    port = get_db_setting("web.port", 5000)
+    host = get_db_setting("web.host", "0.0.0.0")
+    debug = get_db_setting("web.debug", True)
 
     logger.info(f"Starting web server on {host}:{port} (debug: {debug})")
     socketio.run(app, debug=debug, host=host, port=port, allow_unsafe_werkzeug=True)

local_deep_research/web/app_factory.py

@@ -165,29 +165,6 @@ def register_blueprints(app):
         api_bp, url_prefix="/research/api"
     )  # Register API blueprint with prefix
 
-    # Configure settings paths
-    # Import config inside the function to avoid circular dependencies
-    def configure_settings_routes():
-        try:
-            from ..config.config_files import SEARCH_ENGINES_FILE, get_config_dir
-            from .routes.settings_routes import set_config_paths
-
-            CONFIG_DIR = get_config_dir() / "config"
-            MAIN_CONFIG_FILE = CONFIG_DIR / "settings.toml"
-            LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
-
-            set_config_paths(
-                CONFIG_DIR,
-                SEARCH_ENGINES_FILE,
-                MAIN_CONFIG_FILE,
-                LOCAL_COLLECTIONS_FILE,
-            )
-        except Exception as e:
-            logger.error(f"Error configuring settings routes: {e}")
-
-    # Call this after all blueprints are registered
-    configure_settings_routes()
-
     # Add root route redirect
     @app.route("/")
     def root_index():
@@ -260,7 +237,7 @@ def create_database(app):
     from sqlalchemy import create_engine
     from sqlalchemy.orm import scoped_session, sessionmaker
 
-    from .database.migrations import run_migrations, setup_predefined_settings
+    from .database.migrations import run_migrations
     from .database.models import Base
 
     # Configure SQLite to use URI mode, which allows for relative file paths
@@ -281,7 +258,6 @@ def create_database(app):
 
     # Run migrations and setup predefined settings
     run_migrations(engine, app.db_session)
-    setup_predefined_settings(app.db_session)
 
     # Add teardown context
     @app.teardown_appcontext