local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/advanced_search_system/strategies/standard_strategy.py

@@ -0,0 +1,300 @@
+import json
+import logging
+from typing import Dict
+
+from ...citation_handler import CitationHandler
+from ...config.config_files import settings
+from ...config.llm_config import get_llm
+from ...config.search_config import get_search
+from ...utilities.db_utils import get_db_setting
+from ...utilities.enums import KnowledgeAccumulationApproach
+from ...utilities.search_utilities import extract_links_from_search_results
+from ..findings.repository import FindingsRepository
+from ..knowledge.standard_knowledge import StandardKnowledge
+from ..questions.standard_question import StandardQuestionGenerator
+from .base_strategy import BaseSearchStrategy
+
+logger = logging.getLogger(__name__)
+
+
+class StandardSearchStrategy(BaseSearchStrategy):
+    """Standard iterative search strategy that generates follow-up questions."""
+
+    def __init__(self, search=None, model=None, citation_handler=None):
+        """Initialize with optional dependency injection for testing."""
+        self.search = search or get_search()
+        self.model = model or get_llm()
+        self.max_iterations = int(get_db_setting("search.iterations"))
+        self.questions_per_iteration = int(
+            get_db_setting("search.questions_per_iteration")
+        )
+        self.context_limit = int(
+            get_db_setting("general.knowledge_accumulation_context_limit")
+        )
+        self.questions_by_iteration = {}
+
+        # Use provided citation_handler or create one
+        self.citation_handler = citation_handler or CitationHandler(self.model)
+
+        # Initialize specialized components
+        self.question_generator = StandardQuestionGenerator(self.model)
+        self.knowledge_generator = StandardKnowledge(self.model)
+        self.findings_repository = FindingsRepository(self.model)
+
+        # Initialize other attributes
+        self.progress_callback = None
+        self.all_links_of_system = list()
+
+    def _update_progress(
+        self, message: str, progress_percent: int = None, metadata: dict = None
+    ) -> None:
+        """Send a progress update via the callback if available."""
+        if self.progress_callback:
+            self.progress_callback(message, progress_percent, metadata or {})
+
+    def analyze_topic(self, query: str) -> Dict:
+        """Standard implementation of the topic analysis process."""
+        logger.info(f"Starting research on topic: {query}")
+
+        findings = []
+        current_knowledge = ""
+        iteration = 0
+        total_iterations = self.max_iterations
+        section_links = list()
+
+        self._update_progress(
+            "Initializing research system",
+            5,
+            {"phase": "init", "iterations_planned": total_iterations},
+        )
+
+        # Check if search engine is available
+        if self.search is None:
+            error_msg = (
+                "Error: No search engine available. Please check your configuration."
+            )
+            self._update_progress(
+                error_msg,
+                100,
+                {
+                    "phase": "error",
+                    "error": "No search engine available",
+                    "status": "failed",
+                },
+            )
+            return {
+                "findings": [],
+                "iterations": 0,
+                "questions": {},
+                "formatted_findings": "Error: Unable to conduct research without a search engine.",
+                "current_knowledge": "",
+                "error": error_msg,
+            }
+
+        while iteration < self.max_iterations:
+            iteration_progress_base = (iteration / total_iterations) * 100
+            self._update_progress(
+                f"Starting iteration {iteration + 1} of {total_iterations}",
+                int(iteration_progress_base),
+                {"phase": "iteration_start", "iteration": iteration + 1},
+            )
+
+            # Generate questions for this iteration using the question generator
+            # Prepare context for question generation
+            context = f"""Current Query: {query}
+Current Knowledge: {current_knowledge}
+Previous Questions: {json.dumps(self.questions_by_iteration, indent=2)}
+Iteration: {iteration + 1} of {total_iterations}"""
+
+            # Call question generator with updated interface
+            questions = self.question_generator.generate_questions(
+                query=query, context=context
+            )
+
+            self.questions_by_iteration[iteration] = questions
+            logger.info(f"Generated questions: {questions}")
+
+            question_count = len(questions)
+            knowledge_accumulation = get_db_setting(
+                "general.knowledge_accumulation",
+                settings.general.knowledge_accumulation,
+            )
+            for q_idx, question in enumerate(questions):
+                question_progress_base = iteration_progress_base + (
+                    ((q_idx + 1) / question_count) * (100 / total_iterations) * 0.5
+                )
+
+                self._update_progress(
+                    f"Searching for: {question}",
+                    int(question_progress_base),
+                    {
+                        "phase": "search",
+                        "iteration": iteration + 1,
+                        "question_index": q_idx + 1,
+                    },
+                )
+
+                try:
+                    if self.search is None:
+                        self._update_progress(
+                            f"Search engine unavailable, skipping search for: {question}",
+                            int(question_progress_base + 2),
+                            {
+                                "phase": "search_error",
+                                "error": "No search engine available",
+                            },
+                        )
+                        search_results = []
+                    else:
+                        search_results = self.search.run(question)
+                except Exception as e:
+                    error_msg = f"Error during search: {str(e)}"
+                    logger.error(f"SEARCH ERROR: {error_msg}")
+                    self._handle_search_error(error_msg, question_progress_base + 10)
+                    search_results = []
+
+                if search_results is None:
+                    self._update_progress(
+                        f"No search results found for question: {question}",
+                        int(question_progress_base + 2),
+                        {"phase": "search_complete", "result_count": 0},
+                    )
+                    search_results = []  # Initialize to empty list instead of None
+                    continue
+
+                self._update_progress(
+                    f"Found {len(search_results)} results for question: {question}",
+                    int(question_progress_base + 2),
+                    {"phase": "search_complete", "result_count": len(search_results)},
+                )
+
+                logger.info(f"len search: {len(search_results)}")
+
+                if len(search_results) == 0:
+                    continue
+
+                self._update_progress(
+                    f"Analyzing results for: {question}",
+                    int(question_progress_base + 5),
+                    {"phase": "analysis"},
+                )
+
+                try:
+                    result = self.citation_handler.analyze_followup(
+                        question,
+                        search_results,
+                        current_knowledge,
+                        nr_of_links=len(self.all_links_of_system),
+                    )
+                    links = extract_links_from_search_results(search_results)
+                    self.all_links_of_system.extend(links)
+                    section_links.extend(links)
+
+                    if result is not None:
+                        results_with_links = str(result["content"])
+                        findings.append(
+                            {
+                                "phase": f"Follow-up {iteration}.{questions.index(question) + 1}",
+                                "content": results_with_links,
+                                "question": question,
+                                "search_results": search_results,
+                                "documents": result["documents"],
+                            }
+                        )
+
+                        if knowledge_accumulation != str(
+                            KnowledgeAccumulationApproach.NO_KNOWLEDGE.value
+                        ):
+                            current_knowledge = (
+                                current_knowledge
+                                + "\n\n\n New: \n"
+                                + results_with_links
+                            )
+
+                        if knowledge_accumulation == str(
+                            KnowledgeAccumulationApproach.QUESTION.value
+                        ):
+                            logger.info("Compressing knowledge")
+                            self._update_progress(
+                                f"Compress Knowledge for: {question}",
+                                int(question_progress_base + 0),
+                                {"phase": "analysis"},
+                            )
+                            current_knowledge = (
+                                self.knowledge_generator.compress_knowledge(
+                                    current_knowledge, query, section_links
+                                )
+                            )
+
+                        self._update_progress(
+                            f"Analysis complete for question: {question}",
+                            int(question_progress_base + 10),
+                            {"phase": "analysis_complete"},
+                        )
+                except Exception as e:
+                    error_msg = f"Error analyzing results: {str(e)}"
+                    logger.info(f"ANALYSIS ERROR: {error_msg}")
+                    self._handle_search_error(error_msg, question_progress_base + 10)
+
+            iteration += 1
+
+            self._update_progress(
+                f"Compressing knowledge after iteration {iteration}",
+                int((iteration / total_iterations) * 100 - 5),
+                {"phase": "knowledge_compression"},
+            )
+
+            if knowledge_accumulation == KnowledgeAccumulationApproach.ITERATION.value:
+                try:
+                    logger.info("ITERATION - Compressing Knowledge")
+                    current_knowledge = self.knowledge_generator.compress_knowledge(
+                        current_knowledge, query, section_links
+                    )
+                    logger.info("FINISHED ITERATION - Compressing Knowledge")
+                except Exception as e:
+                    error_msg = f"Error compressing knowledge: {str(e)}"
+                    logger.info(f"COMPRESSION ERROR: {error_msg}")
+                    self._handle_search_error(
+                        error_msg, int((iteration / total_iterations) * 100 - 3)
+                    )
+
+            self._update_progress(
+                f"Iteration {iteration} complete",
+                int((iteration / total_iterations) * 100),
+                {"phase": "iteration_complete", "iteration": iteration},
+            )
+
+            # Extract content from findings for synthesis
+            finding_contents = [f["content"] for f in findings if "content" in f]
+
+            # First synthesize findings to get coherent content
+            synthesized_content = self.findings_repository.synthesize_findings(
+                query,
+                finding_contents,
+                findings,  # Pass the full findings list with search results
+                accumulated_knowledge=current_knowledge,
+                old_formatting=False,  # Don't format here, just synthesize content
+            )
+
+            # Transfer questions to the repository
+            self.findings_repository.set_questions_by_iteration(
+                self.questions_by_iteration
+            )
+
+            # Now format the findings with search questions and sources
+            formatted_findings = self.findings_repository.format_findings_to_text(
+                findings, synthesized_content
+            )
+
+            # Add the synthesized content to the repository
+            self.findings_repository.add_finding(query, synthesized_content)
+
+        self._update_progress("Research complete", 95, {"phase": "complete"})
+
+        return {
+            "findings": findings,
+            "iterations": iteration,
+            "questions": self.questions_by_iteration,
+            "formatted_findings": formatted_findings,
+            "current_knowledge": current_knowledge,
+        }

local_deep_research/advanced_search_system/tools/__init__.py

@@ -0,0 +1 @@
+# Search System Tools Package

local_deep_research/advanced_search_system/tools/base_tool.py

@@ -0,0 +1,100 @@
+"""
+Base class for all agent-compatible tools.
+Defines the common interface and shared functionality for different tools.
+"""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+logger = logging.getLogger(__name__)
+
+
+class BaseTool(ABC):
+    """Abstract base class for all agent-compatible tools."""
+
+    def __init__(self, name: str, description: str):
+        """
+        Initialize the tool.
+
+        Args:
+            name: The name of the tool
+            description: A description of what the tool does
+        """
+        self.name = name
+        self.description = description
+        self.parameters: Dict[str, Dict[str, Any]] = {}
+
+    @abstractmethod
+    def execute(self, **kwargs) -> Any:
+        """
+        Execute the tool with the given parameters.
+
+        Args:
+            **kwargs: Tool-specific parameters
+
+        Returns:
+            Any: The result of the tool execution
+        """
+        pass
+
+    def get_schema(self) -> Dict[str, Any]:
+        """
+        Get the JSON schema for the tool's parameters.
+
+        Returns:
+            Dict[str, Any]: The JSON schema
+        """
+        return {
+            "name": self.name,
+            "description": self.description,
+            "parameters": self.parameters,
+        }
+
+    def validate_parameters(self, **kwargs) -> bool:
+        """
+        Validate the provided parameters against the tool's schema.
+
+        Args:
+            **kwargs: Parameters to validate
+
+        Returns:
+            bool: True if parameters are valid, False otherwise
+        """
+        for param_name, param_schema in self.parameters.items():
+            if param_name not in kwargs:
+                if param_schema.get("required", False):
+                    logger.error(f"Missing required parameter: {param_name}")
+                    return False
+                continue
+
+            param_value = kwargs[param_name]
+            param_type = param_schema.get("type")
+
+            if param_type and not isinstance(param_value, eval(param_type)):
+                logger.error(f"Invalid type for parameter {param_name}")
+                return False
+
+            if "enum" in param_schema and param_value not in param_schema["enum"]:
+                logger.error(f"Invalid value for parameter {param_name}")
+                return False
+
+        return True
+
+    def _log_execution(self, **kwargs) -> None:
+        """
+        Log tool execution details.
+
+        Args:
+            **kwargs: Parameters used in execution
+        """
+        logger.info(f"Executing tool {self.name} with parameters: {kwargs}")
+
+    def _log_result(self, result: Any) -> None:
+        """
+        Log tool execution result.
+
+        Args:
+            result: The result of the tool execution
+        """
+        logger.info(f"Tool {self.name} execution completed with result: {result}")

local_deep_research/advanced_search_system/tools/knowledge_tools/__init__.py

@@ -0,0 +1 @@
+# Search System Tools Package

local_deep_research/advanced_search_system/tools/question_tools/__init__.py

@@ -0,0 +1 @@
+# Search System Tools Package

local_deep_research/advanced_search_system/tools/search_tools/__init__.py

@@ -0,0 +1 @@
+# Search System Tools Package

local_deep_research/api/__init__.py

@@ -4,11 +4,11 @@ API module for programmatic access to Local Deep Research functionality.
 """
 
 from .research_functions import (
-    quick_summary, 
-    generate_report, 
     analyze_documents,
+    generate_report,
+    get_available_collections,
     get_available_search_engines,
-    get_available_collections
+    quick_summary,
 )
 
 __all__ = [
@@ -16,5 +16,5 @@ __all__ = [
     "generate_report",
     "analyze_documents",
     "get_available_search_engines",
-    "get_available_collections"
-]
+    "get_available_collections",
+]