local-deep-research 0.1.25__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/__init__.py

@@ -1,33 +1,34 @@
 """
-Local Deep Research - AI-powered research assistant
-
-A powerful AI research system with iterative analysis capabilities
-and multiple search engines integration.
+Local Deep Research - A tool for conducting deep research using AI.
 """
 
 __version__ = "0.1.0"
+__author__ = "Your Name"
+__description__ = "A tool for conducting deep research using AI"
+
+from .config.llm_config import get_llm
+from .config.search_config import get_search
+from .report_generator import get_report_generator
+
+
+def get_advanced_search_system(strategy_name: str = "iterdrag"):
+    """
+    Get an instance of the advanced search system.
+
+    Args:
+        strategy_name: The name of the search strategy to use ("standard" or "iterdrag")
 
-# Initialize configuration on module import
-from .utilties.setup_utils import setup_user_directories
+    Returns:
+        AdvancedSearchSystem: An instance of the advanced search system
+    """
+    from .search_system import AdvancedSearchSystem
 
-# Import main components
-from .search_system import AdvancedSearchSystem
-from .report_generator import IntegratedReportGenerator
-from .config import get_llm, get_search
+    return AdvancedSearchSystem(strategy_name=strategy_name)
 
-# Import API functions
-from .api import quick_summary, generate_report, analyze_documents
-from .api import get_available_search_engines, get_available_collections
 
-# Export it
 __all__ = [
-    "AdvancedSearchSystem", 
-    "IntegratedReportGenerator",
     "get_llm",
     "get_search",
-    "quick_summary",
-    "generate_report",
-    "analyze_documents",
-    "get_available_search_engines",
-    "get_available_collections"
-]
+    "get_report_generator",
+    "get_advanced_search_system",
+]

local_deep_research/__main__.py

@@ -0,0 +1,16 @@
+"""
+Main entry point when running the package with `python -m local_deep_research`.
+This avoids circular imports by directly importing the main function after
+the package is fully loaded.
+"""
+
+
+def main():
+    # Only import main after the whole package has been initialized
+    from .main import main as main_func
+
+    main_func()
+
+
+if __name__ == "__main__":
+    main()

local_deep_research/advanced_search_system/__init__.py

@@ -0,0 +1,7 @@
+# Search System Package
+# AdvancedSearchSystem is now in search_system.py
+
+# We cannot directly import AdvancedSearchSystem here due to circular imports
+# The web code should import from the search_system module instead
+
+__all__ = []

local_deep_research/advanced_search_system/filters/__init__.py

@@ -0,0 +1,8 @@
+# src/local_deep_research/advanced_search_system/filters/__init__.py
+"""
+Filters for search results.
+"""
+
+from .cross_engine_filter import CrossEngineFilter
+
+__all__ = ["CrossEngineFilter"]

local_deep_research/advanced_search_system/filters/base_filter.py

@@ -0,0 +1,38 @@
+# src/local_deep_research/advanced_search_system/filters/base_filter.py
+"""
+Base class for search result filters.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+class BaseFilter(ABC):
+    """Abstract base class for all search result filters."""
+
+    def __init__(self, model=None):
+        """
+        Initialize the filter.
+
+        Args:
+            model: The language model to use for relevance assessments
+        """
+        self.model = model
+
+    @abstractmethod
+    def filter_results(self, results: List[Dict], query: str, **kwargs) -> List[Dict]:
+        """
+        Filter search results by relevance to the query.
+
+        Args:
+            results: List of search result dictionaries
+            query: The original search query
+            **kwargs: Additional filter-specific parameters
+
+        Returns:
+            Filtered list of search results
+        """
+        pass

local_deep_research/advanced_search_system/filters/cross_engine_filter.py

@@ -0,0 +1,200 @@
+"""
+Cross-engine search result filter implementation.
+"""
+
+import json
+import logging
+from typing import Dict, List
+
+from ...utilities.search_utilities import remove_think_tags
+from .base_filter import BaseFilter
+
+logger = logging.getLogger(__name__)
+
+
+class CrossEngineFilter(BaseFilter):
+    """Filter that ranks and filters results from multiple search engines."""
+
+    def __init__(
+        self, model, max_results=20, default_reorder=True, default_reindex=True
+    ):
+        """
+        Initialize the cross-engine filter.
+
+        Args:
+            model: Language model to use for relevance assessment
+            max_results: Maximum number of results to keep after filtering
+            default_reorder: Default setting for reordering results by relevance
+            default_reindex: Default setting for reindexing results after filtering
+        """
+        super().__init__(model)
+        self.max_results = max_results
+        self.default_reorder = default_reorder
+        self.default_reindex = default_reindex
+
+    def filter_results(
+        self,
+        results: List[Dict],
+        query: str,
+        reorder=None,
+        reindex=None,
+        start_index=0,
+        **kwargs,
+    ) -> List[Dict]:
+        """
+        Filter and rank search results from multiple engines by relevance.
+
+        Args:
+            results: Combined list of search results from all engines
+            query: The original search query
+            reorder: Whether to reorder results by relevance (default: use instance default)
+            reindex: Whether to update result indices after filtering (default: use instance default)
+            start_index: Starting index for the results (used for continuous indexing)
+            **kwargs: Additional parameters
+
+        Returns:
+            Filtered list of search results
+        """
+        # Use instance defaults if not specified
+        if reorder is None:
+            reorder = self.default_reorder
+        if reindex is None:
+            reindex = self.default_reindex
+
+        if not self.model or len(results) <= 10:  # Don't filter if few results
+            # Even if not filtering, update indices if requested
+            if reindex:
+                for i, result in enumerate(
+                    results[: min(self.max_results, len(results))]
+                ):
+                    result["index"] = str(i + start_index + 1)
+            return results[: min(self.max_results, len(results))]
+
+        # Create context for LLM
+        preview_context = []
+        for i, result in enumerate(results):
+            title = result.get("title", "Untitled").strip()
+            snippet = result.get("snippet", "").strip()
+            engine = result.get("engine", "Unknown engine")
+
+            # Clean up snippet if too long
+            if len(snippet) > 200:
+                snippet = snippet[:200] + "..."
+
+            preview_context.append(
+                f"[{i}] Engine: {engine} | Title: {title}\nSnippet: {snippet}"
+            )
+
+        # Set a reasonable limit on context length
+        max_context_items = min(30, len(preview_context))
+        context = "\n\n".join(preview_context[:max_context_items])
+
+        prompt = f"""You are a search result filter. Your task is to rank search results from multiple engines by relevance to a query.
+
+Query: "{query}"
+
+Search Results:
+{context}
+
+Return the search results as a JSON array of indices, ranked from most to least relevant to the query.
+Only include indices of results that are actually relevant to the query.
+For example: [3, 0, 7, 1]
+
+If no results seem relevant to the query, return an empty array: []"""
+
+        try:
+            # Get LLM's evaluation
+            response = self.model.invoke(prompt)
+
+            # Extract response text
+            if hasattr(response, "content"):
+                response_text = remove_think_tags(response.content)
+            else:
+                response_text = remove_think_tags(str(response))
+
+            # Clean up response
+            response_text = response_text.strip()
+
+            # Find JSON array in response
+            start_idx = response_text.find("[")
+            end_idx = response_text.rfind("]")
+
+            if start_idx >= 0 and end_idx > start_idx:
+                array_text = response_text[start_idx : end_idx + 1]
+                ranked_indices = json.loads(array_text)
+
+                # If not reordering, just filter based on the indices
+                if not reorder:
+                    # Just keep the results that were deemed relevant
+                    filtered_results = []
+                    for idx in sorted(
+                        ranked_indices
+                    ):  # Sort to maintain original order
+                        if idx < len(results):
+                            filtered_results.append(results[idx])
+
+                    # Limit results if needed
+                    final_results = filtered_results[
+                        : min(self.max_results, len(filtered_results))
+                    ]
+
+                    # Reindex if requested
+                    if reindex:
+                        for i, result in enumerate(final_results):
+                            result["index"] = str(i + start_index + 1)
+
+                    logger.info(
+                        f"Cross-engine filtering kept {len(final_results)} out of {len(results)} results without reordering"
+                    )
+                    return final_results
+
+                # Create ranked results list (reordering)
+                ranked_results = []
+                for idx in ranked_indices:
+                    if idx < len(results):
+                        ranked_results.append(results[idx])
+
+                # If filtering removed everything, return top results
+                if not ranked_results and results:
+                    logger.info(
+                        "Cross-engine filtering removed all results, returning top 10 originals instead"
+                    )
+                    top_results = results[: min(10, len(results))]
+                    # Update indices if requested
+                    if reindex:
+                        for i, result in enumerate(top_results):
+                            result["index"] = str(i + start_index + 1)
+                    return top_results
+
+                # Limit results if needed
+                max_filtered = min(self.max_results, len(ranked_results))
+                final_results = ranked_results[:max_filtered]
+
+                # Update indices if requested
+                if reindex:
+                    for i, result in enumerate(final_results):
+                        result["index"] = str(i + start_index + 1)
+
+                logger.info(
+                    f"Cross-engine filtering kept {len(final_results)} out of {len(results)} results with reordering={reorder}, reindex={reindex}"
+                )
+                return final_results
+            else:
+                logger.info(
+                    "Could not find JSON array in response, returning original results"
+                )
+                top_results = results[: min(self.max_results, len(results))]
+                # Update indices if requested
+                if reindex:
+                    for i, result in enumerate(top_results):
+                        result["index"] = str(i + start_index + 1)
+                return top_results
+
+        except Exception as e:
+            logger.error(f"Cross-engine filtering error: {e}")
+            top_results = results[: min(self.max_results, len(results))]
+            # Update indices if requested
+            if reindex:
+                for i, result in enumerate(top_results):
+                    result["index"] = str(i + start_index + 1)
+            return top_results

local_deep_research/advanced_search_system/findings/base_findings.py

@@ -0,0 +1,81 @@
+"""
+Base class for all findings repositories.
+Defines the common interface and shared functionality for different findings management approaches.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+from langchain_core.language_models import BaseLLM
+
+logger = logging.getLogger(__name__)
+
+
+class BaseFindingsRepository(ABC):
+    """Abstract base class for all findings repositories."""
+
+    def __init__(self, model: BaseLLM):
+        """
+        Initialize the findings repository.
+
+        Args:
+            model: The language model to use for findings operations
+        """
+        self.model = model
+        self.findings: Dict[str, List[str]] = {}
+
+    @abstractmethod
+    def add_finding(self, query: str, finding: Dict | str) -> None:
+        """
+        Add a finding to the repository.
+
+        Args:
+            query: The query associated with the finding
+            finding: The finding to add
+        """
+        pass
+
+    @abstractmethod
+    def get_findings(self, query: str) -> List[str]:
+        """
+        Get findings for a query.
+
+        Args:
+            query: The query to get findings for
+
+        Returns:
+            List[str]: List of findings for the query
+        """
+        pass
+
+    @abstractmethod
+    def clear_findings(self, query: str) -> None:
+        """
+        Clear findings for a query.
+
+        Args:
+            query: The query to clear findings for
+        """
+        pass
+
+    @abstractmethod
+    def synthesize_findings(
+        self,
+        query: str,
+        sub_queries: List[str],
+        findings: List[str],
+        accumulated_knowledge: str,
+    ) -> str:
+        """
+        Synthesize findings from sub-queries into a final answer.
+
+        Args:
+            query: The original query
+            sub_queries: List of sub-queries
+            findings: List of findings for each sub-query
+            accumulated_knowledge: Accumulated knowledge from previous findings
+        Returns:
+            str: Synthesized final answer
+        """
+        pass