local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/advanced_search_system/candidate_exploration/adaptive_explorer.py

@@ -0,0 +1,329 @@
+"""
+Adaptive candidate explorer implementation.
+
+This explorer adapts its search strategy based on the success of different
+approaches and the quality of candidates found.
+"""
+
+import time
+from collections import defaultdict
+from typing import List, Optional
+
+from loguru import logger
+
+from ..candidates.base_candidate import Candidate
+from ..constraints.base_constraint import Constraint
+from .base_explorer import (
+    BaseCandidateExplorer,
+    ExplorationResult,
+    ExplorationStrategy,
+)
+
+
+class AdaptiveExplorer(BaseCandidateExplorer):
+    """
+    Adaptive candidate explorer that learns from search results.
+
+    This explorer:
+    1. Tries different search strategies
+    2. Tracks which strategies work best
+    3. Adapts future searches based on success rates
+    4. Focuses effort on the most productive approaches
+    """
+
+    def __init__(
+        self,
+        *args,
+        initial_strategies: List[str] = None,
+        adaptation_threshold: int = 5,  # Adapt after this many searches
+        **kwargs,
+    ):
+        """
+        Initialize adaptive explorer.
+
+        Args:
+            initial_strategies: Starting search strategies to try
+            adaptation_threshold: Number of searches before adapting
+        """
+        super().__init__(*args, **kwargs)
+
+        self.initial_strategies = initial_strategies or [
+            "direct_search",
+            "synonym_expansion",
+            "category_exploration",
+            "related_terms",
+        ]
+
+        self.adaptation_threshold = adaptation_threshold
+
+        # Track strategy performance
+        self.strategy_stats = defaultdict(
+            lambda: {"attempts": 0, "candidates_found": 0, "quality_sum": 0.0}
+        )
+        self.current_strategy = self.initial_strategies[0]
+
+    def explore(
+        self,
+        initial_query: str,
+        constraints: Optional[List[Constraint]] = None,
+        entity_type: Optional[str] = None,
+    ) -> ExplorationResult:
+        """Explore candidates using adaptive strategy."""
+        start_time = time.time()
+        logger.info(f"Starting adaptive exploration for: {initial_query}")
+
+        all_candidates = []
+        exploration_paths = []
+        total_searched = 0
+
+        # Track current strategy performance
+        search_count = 0
+
+        while self._should_continue_exploration(
+            start_time, len(all_candidates)
+        ):
+            # Choose strategy based on current performance
+            strategy = self._choose_strategy(search_count)
+
+            # Generate query using chosen strategy
+            query = self._generate_query_with_strategy(
+                initial_query, strategy, all_candidates, constraints
+            )
+
+            if not query or query.lower() in self.explored_queries:
+                # Try next strategy or stop
+                if not self._try_next_strategy():
+                    break
+                continue
+
+            # Execute search
+            logger.info(
+                f"Using strategy '{strategy}' for query: {query[:50]}..."
+            )
+            results = self._execute_search(query)
+            candidates = self._extract_candidates_from_results(
+                results, entity_type
+            )
+
+            # Track strategy performance
+            self._update_strategy_stats(strategy, candidates)
+
+            # Add results
+            all_candidates.extend(candidates)
+            total_searched += 1
+            search_count += 1
+
+            exploration_paths.append(
+                f"{strategy}: {query} -> {len(candidates)} candidates"
+            )
+
+            # Adapt strategy if threshold reached
+            if search_count >= self.adaptation_threshold:
+                self._adapt_strategy()
+                search_count = 0
+
+        # Process final results
+        unique_candidates = self._deduplicate_candidates(all_candidates)
+        ranked_candidates = self._rank_candidates_by_relevance(
+            unique_candidates, initial_query
+        )
+        final_candidates = ranked_candidates[: self.max_candidates]
+
+        elapsed_time = time.time() - start_time
+        logger.info(
+            f"Adaptive exploration completed: {len(final_candidates)} candidates in {elapsed_time:.1f}s"
+        )
+
+        return ExplorationResult(
+            candidates=final_candidates,
+            total_searched=total_searched,
+            unique_candidates=len(unique_candidates),
+            exploration_paths=exploration_paths,
+            metadata={
+                "strategy": "adaptive",
+                "strategy_stats": dict(self.strategy_stats),
+                "final_strategy": self.current_strategy,
+                "entity_type": entity_type,
+            },
+            elapsed_time=elapsed_time,
+            strategy_used=ExplorationStrategy.ADAPTIVE,
+        )
+
+    def generate_exploration_queries(
+        self,
+        base_query: str,
+        found_candidates: List[Candidate],
+        constraints: Optional[List[Constraint]] = None,
+    ) -> List[str]:
+        """Generate queries using adaptive approach."""
+        queries = []
+
+        # Generate queries using best performing strategies
+        top_strategies = self._get_top_strategies(3)
+
+        for strategy in top_strategies:
+            query = self._generate_query_with_strategy(
+                base_query, strategy, found_candidates, constraints
+            )
+            if query:
+                queries.append(query)
+
+        return queries
+
+    def _choose_strategy(self, search_count: int) -> str:
+        """Choose the best strategy based on current performance."""
+        if search_count < self.adaptation_threshold:
+            # Use current strategy during initial phase
+            return self.current_strategy
+
+        # Choose best performing strategy
+        best_strategies = self._get_top_strategies(1)
+        return best_strategies[0] if best_strategies else self.current_strategy
+
+    def _get_top_strategies(self, n: int) -> List[str]:
+        """Get top N performing strategies."""
+        if not self.strategy_stats:
+            return self.initial_strategies[:n]
+
+        # Sort by candidates found per attempt
+        sorted_strategies = sorted(
+            self.strategy_stats.items(),
+            key=lambda x: x[1]["candidates_found"] / max(x[1]["attempts"], 1),
+            reverse=True,
+        )
+
+        return [strategy for strategy, _ in sorted_strategies[:n]]
+
+    def _generate_query_with_strategy(
+        self,
+        base_query: str,
+        strategy: str,
+        found_candidates: List[Candidate],
+        constraints: Optional[List[Constraint]] = None,
+    ) -> Optional[str]:
+        """Generate a query using specific strategy."""
+        try:
+            if strategy == "direct_search":
+                return self._direct_search_query(base_query)
+            elif strategy == "synonym_expansion":
+                return self._synonym_expansion_query(base_query)
+            elif strategy == "category_exploration":
+                return self._category_exploration_query(
+                    base_query, found_candidates
+                )
+            elif strategy == "related_terms":
+                return self._related_terms_query(base_query, found_candidates)
+            elif strategy == "constraint_focused" and constraints:
+                return self._constraint_focused_query(base_query, constraints)
+            else:
+                return self._direct_search_query(base_query)
+
+        except Exception as e:
+            logger.error(
+                f"Error generating query with strategy {strategy}: {e}"
+            )
+            return None
+
+    def _direct_search_query(self, base_query: str) -> str:
+        """Generate direct search variation."""
+        variations = [
+            f'"{base_query}" examples',
+            f"{base_query} list",
+            f"{base_query} instances",
+            f"types of {base_query}",
+        ]
+
+        # Choose variation not yet explored
+        for variation in variations:
+            if variation.lower() not in self.explored_queries:
+                return variation
+
+        return base_query
+
+    def _synonym_expansion_query(self, base_query: str) -> Optional[str]:
+        """Generate query with synonym expansion."""
+        prompt = f"""
+Generate a search query that means the same as "{base_query}" but uses different words.
+Focus on synonyms and alternative terminology.
+
+Query:
+"""
+
+        try:
+            response = self.model.invoke(prompt).content.strip()
+            return response if response != base_query else None
+        except:
+            return None
+
+    def _category_exploration_query(
+        self, base_query: str, found_candidates: List[Candidate]
+    ) -> Optional[str]:
+        """Generate query exploring categories of found candidates."""
+        if not found_candidates:
+            return f"categories of {base_query}"
+
+        sample_names = [c.name for c in found_candidates[:3]]
+        return f"similar to {', '.join(sample_names)}"
+
+    def _related_terms_query(
+        self, base_query: str, found_candidates: List[Candidate]
+    ) -> Optional[str]:
+        """Generate query using related terms."""
+        prompt = f"""
+Given the search topic "{base_query}", suggest a related search term that would find similar but different examples.
+
+Related search term:
+"""
+
+        try:
+            response = self.model.invoke(prompt).content.strip()
+            return response if response != base_query else None
+        except:
+            return None
+
+    def _constraint_focused_query(
+        self, base_query: str, constraints: List[Constraint]
+    ) -> Optional[str]:
+        """Generate query focused on a specific constraint."""
+        if not constraints:
+            return None
+
+        # Pick least explored constraint
+        constraint = constraints[0]  # Simple selection
+        return f"{base_query} {constraint.value}"
+
+    def _update_strategy_stats(
+        self, strategy: str, candidates: List[Candidate]
+    ):
+        """Update performance statistics for a strategy."""
+        self.strategy_stats[strategy]["attempts"] += 1
+        self.strategy_stats[strategy]["candidates_found"] += len(candidates)
+
+        # Simple quality assessment (could be more sophisticated)
+        quality = len(candidates) * 0.1  # Basic quality based on quantity
+        self.strategy_stats[strategy]["quality_sum"] += quality
+
+    def _adapt_strategy(self):
+        """Adapt current strategy based on performance."""
+        best_strategies = self._get_top_strategies(1)
+        if best_strategies and best_strategies[0] != self.current_strategy:
+            old_strategy = self.current_strategy
+            self.current_strategy = best_strategies[0]
+            logger.info(
+                f"Adapted strategy from '{old_strategy}' to '{self.current_strategy}'"
+            )
+
+    def _try_next_strategy(self) -> bool:
+        """Try the next available strategy."""
+        current_index = (
+            self.initial_strategies.index(self.current_strategy)
+            if self.current_strategy in self.initial_strategies
+            else 0
+        )
+        next_index = (current_index + 1) % len(self.initial_strategies)
+
+        if next_index == 0:  # We've tried all strategies
+            return False
+
+        self.current_strategy = self.initial_strategies[next_index]
+        return True

local_deep_research/advanced_search_system/candidate_exploration/base_explorer.py

@@ -0,0 +1,341 @@
+"""
+Base candidate explorer for inheritance-based exploration system.
+
+This module provides the base interface and common functionality for
+candidate exploration implementations.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, List, Optional, Set
+
+from langchain_core.language_models import BaseChatModel
+from loguru import logger
+
+from ..candidates.base_candidate import Candidate
+from ..constraints.base_constraint import Constraint
+
+
+class ExplorationStrategy(Enum):
+    """Different exploration strategies."""
+
+    BREADTH_FIRST = "breadth_first"  # Explore widely first
+    DEPTH_FIRST = "depth_first"  # Deep dive into promising areas
+    CONSTRAINT_GUIDED = "constraint_guided"  # Let constraints guide exploration
+    DIVERSITY_FOCUSED = "diversity_focused"  # Maximize candidate diversity
+    ADAPTIVE = "adaptive"  # Adapt based on findings
+
+
+@dataclass
+class ExplorationResult:
+    """Result of candidate exploration."""
+
+    candidates: List[Candidate]
+    total_searched: int
+    unique_candidates: int
+    exploration_paths: List[str]
+    metadata: Dict
+    elapsed_time: float
+    strategy_used: ExplorationStrategy
+
+
+class BaseCandidateExplorer(ABC):
+    """
+    Base class for candidate exploration implementations.
+
+    This provides the common interface and shared functionality that
+    all candidate explorers should implement.
+    """
+
+    def __init__(
+        self,
+        model: BaseChatModel,
+        search_engine,
+        max_candidates: int = 50,
+        max_search_time: float = 60.0,
+        **kwargs,
+    ):
+        """
+        Initialize the base candidate explorer.
+
+        Args:
+            model: Language model for analysis
+            search_engine: Search engine for finding candidates
+            max_candidates: Maximum number of candidates to find
+            max_search_time: Maximum time to spend searching
+            **kwargs: Additional parameters for specific implementations
+        """
+        self.model = model
+        self.search_engine = search_engine
+        self.max_candidates = max_candidates
+        self.max_search_time = max_search_time
+
+        # Tracking
+        self.explored_queries: Set[str] = set()
+        self.found_candidates: Dict[str, Candidate] = {}
+
+    @abstractmethod
+    def explore(
+        self,
+        initial_query: str,
+        constraints: Optional[List[Constraint]] = None,
+        entity_type: Optional[str] = None,
+    ) -> ExplorationResult:
+        """
+        Explore and discover candidates.
+
+        Args:
+            initial_query: Starting query for exploration
+            constraints: Optional constraints to guide exploration
+            entity_type: Optional entity type to focus on
+
+        Returns:
+            ExplorationResult: Complete exploration results
+        """
+        pass
+
+    @abstractmethod
+    def generate_exploration_queries(
+        self,
+        base_query: str,
+        found_candidates: List[Candidate],
+        constraints: Optional[List[Constraint]] = None,
+    ) -> List[str]:
+        """
+        Generate new queries for continued exploration.
+
+        Args:
+            base_query: Original base query
+            found_candidates: Candidates found so far
+            constraints: Optional constraints to consider
+
+        Returns:
+            List[str]: New queries to explore
+        """
+        pass
+
+    def _execute_search(self, query: str) -> Dict:
+        """Execute a search query."""
+        try:
+            # Mark query as explored
+            self.explored_queries.add(query.lower())
+
+            # Execute search
+            results = self.search_engine.run(query)
+
+            # Handle different result formats
+            if isinstance(results, list):
+                # If results is a list, wrap it in the expected format
+                formatted_results = {"results": results, "query": query}
+                logger.info(
+                    f"Search '{query[:50]}...' returned {len(results)} results"
+                )
+                return formatted_results
+            elif isinstance(results, dict):
+                # If results is already a dict, use it as is
+                result_count = len(results.get("results", []))
+                logger.info(
+                    f"Search '{query[:50]}...' returned {result_count} results"
+                )
+                return results
+            else:
+                # Unknown format, return empty
+                logger.warning(f"Unknown search result format: {type(results)}")
+                return {"results": [], "query": query}
+
+        except Exception as e:
+            logger.error(f"Error executing search '{query}': {e}")
+            return {"results": []}
+
+    def _extract_candidates_from_results(
+        self,
+        results: Dict,
+        original_query: str = None,
+        entity_type: Optional[str] = None,
+    ) -> List[Candidate]:
+        """Generate answer candidates directly from search results using LLM."""
+        candidates = []
+
+        # Collect all search result content
+        all_content = []
+        for result in results.get("results", []):
+            title = result.get("title", "")
+            snippet = result.get("snippet", "")
+            if title or snippet:
+                all_content.append(f"Title: {title}\nContent: {snippet}")
+
+        if not all_content or not original_query:
+            return candidates
+
+        # Generate answer candidates using LLM
+        answer_candidates = self._generate_answer_candidates(
+            original_query,
+            "\n\n".join(all_content[:10]),  # Limit to first 10 results
+        )
+
+        for answer in answer_candidates:
+            if answer and answer not in self.found_candidates:
+                candidate = Candidate(
+                    name=answer,
+                    metadata={
+                        "source": "llm_answer_generation",
+                        "query": results.get("query", ""),
+                        "original_query": original_query,
+                        "result_count": len(results.get("results", [])),
+                    },
+                )
+                candidates.append(candidate)
+                self.found_candidates[answer] = candidate
+
+        return candidates
+
+    def _generate_answer_candidates(
+        self, question: str, search_content: str
+    ) -> List[str]:
+        """Generate multiple answer candidates from search results."""
+        prompt = f"""
+Question: {question}
+
+Based on these search results, provide 3-5 possible answers:
+
+{search_content}
+
+Give me multiple possible answers, one per line:
+"""
+
+        try:
+            response = self.model.invoke(prompt)
+            content = response.content.strip()
+
+            # Parse multiple answers
+            answers = []
+            for line in content.split("\n"):
+                line = line.strip()
+                if line:
+                    # Clean up common prefixes and formatting
+                    line = line.lstrip("•-*1234567890.").strip()
+                    if line and len(line) > 2:  # Skip very short answers
+                        answers.append(line)
+
+            return answers[:5]  # Limit to 5 candidates max
+
+        except Exception as e:
+            logger.error(f"Error generating answer candidates: {e}")
+            return []
+
+    def _extract_entity_names(
+        self, text: str, entity_type: Optional[str] = None
+    ) -> List[str]:
+        """Extract entity names from text using LLM."""
+        if not text.strip():
+            return []
+
+        prompt = f"""
+Extract specific entity names from this text.
+{"Focus on: " + entity_type if entity_type else "Extract any named entities."}
+
+Text: {text[:500]}
+
+Return only the names, one per line. Be selective - only include clear, specific names.
+Do not include:
+- Generic terms or categories
+- Adjectives or descriptions
+- Common words
+
+Names:
+"""
+
+        try:
+            response = self.model.invoke(prompt).content.strip()
+
+            # Parse response into names
+            names = []
+            for line in response.split("\n"):
+                name = line.strip()
+                if (
+                    name
+                    and len(name) > 2
+                    and not name.lower().startswith(("the ", "a ", "an "))
+                ):
+                    names.append(name)
+
+            return names[:5]  # Limit to top 5 per text
+
+        except Exception as e:
+            logger.error(f"Error extracting entity names: {e}")
+            return []
+
+    def _should_continue_exploration(
+        self, start_time: float, candidates_found: int
+    ) -> bool:
+        """Determine if exploration should continue."""
+        import time
+
+        elapsed = time.time() - start_time
+
+        # Stop if time limit reached
+        if elapsed > self.max_search_time:
+            logger.info(f"Time limit reached ({elapsed:.1f}s)")
+            return False
+
+        # Stop if candidate limit reached
+        if candidates_found >= self.max_candidates:
+            logger.info(f"Candidate limit reached ({candidates_found})")
+            return False
+
+        return True
+
+    def _deduplicate_candidates(
+        self, candidates: List[Candidate]
+    ) -> List[Candidate]:
+        """Remove duplicate candidates based on name similarity."""
+        unique_candidates = []
+        seen_names = set()
+
+        for candidate in candidates:
+            # Simple deduplication by exact name match
+            name_key = candidate.name.lower().strip()
+            if name_key not in seen_names:
+                seen_names.add(name_key)
+                unique_candidates.append(candidate)
+
+        return unique_candidates
+
+    def _rank_candidates_by_relevance(
+        self, candidates: List[Candidate], query: str
+    ) -> List[Candidate]:
+        """Rank candidates by relevance to original query."""
+        if not candidates:
+            return candidates
+
+        # Simple relevance scoring based on metadata
+        for candidate in candidates:
+            score = 0.0
+
+            # Score based on source query similarity
+            if "query" in candidate.metadata:
+                # Simple word overlap scoring
+                query_words = set(query.lower().split())
+                candidate_query_words = set(
+                    candidate.metadata["query"].lower().split()
+                )
+                overlap = len(query_words.intersection(candidate_query_words))
+                score += overlap * 0.1
+
+            # Score based on result title relevance
+            if "result_title" in candidate.metadata:
+                title_words = set(
+                    candidate.metadata["result_title"].lower().split()
+                )
+                overlap = len(query_words.intersection(title_words))
+                score += overlap * 0.2
+
+            candidate.relevance_score = score
+
+        # Sort by relevance
+        return sorted(
+            candidates,
+            key=lambda c: getattr(c, "relevance_score", 0.0),
+            reverse=True,
+        )