local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/advanced_search_system/strategies/modular_strategy.py

@@ -0,0 +1,1142 @@
+"""
+Modular strategy that demonstrates usage of the new constraint_checking and candidate_exploration modules.
+Enhanced with LLM-driven constraint processing, early rejection, and immediate evaluation.
+"""
+
+import asyncio
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple
+
+from loguru import logger
+
+from ...utilities.search_cache import get_search_cache, normalize_entity_query
+from ..candidate_exploration import (
+    AdaptiveExplorer,
+    ConstraintGuidedExplorer,
+    DiversityExplorer,
+    ParallelExplorer,
+)
+from ..constraint_checking import (
+    DualConfidenceChecker,
+    StrictChecker,
+    ThresholdChecker,
+)
+from ..constraints import ConstraintAnalyzer
+from ..questions import StandardQuestionGenerator
+from .base_strategy import BaseSearchStrategy
+
+
+@dataclass
+class CandidateConfidence:
+    """Track candidate confidence levels for early rejection"""
+
+    candidate: object
+    positive_confidence: float
+    negative_confidence: float
+    rejection_reason: Optional[str] = None
+    should_continue: bool = True
+
+
+class LLMConstraintProcessor:
+    """LLM-driven intelligent constraint processing"""
+
+    def __init__(self, model):
+        self.model = model
+
+    async def decompose_constraints_intelligently(self, constraints):
+        """Let LLM intelligently break down constraints into searchable elements"""
+        constraint_text = "\n".join([f"- {c.description}" for c in constraints])
+
+        prompt = f"""
+        I have these constraints from a search query:
+        {constraint_text}
+
+        Please intelligently decompose these constraints into atomic, searchable elements that can be combined in different ways.
+
+        For each constraint, provide:
+        1. **Atomic elements** - Break it into smallest meaningful parts
+        2. **Variations** - Different ways to express the same concept
+        3. **Granular specifics** - Specific values, years, numbers, etc.
+
+        Example for a time-based constraint:
+        - Atomic elements: Break down the main subject into searchable terms
+        - Time variations: Different ways to express time periods
+        - Granular specifics: Individual years, dates, or specific values mentioned
+
+        Return as valid JSON format:
+        {{
+            "constraint_1": {{
+                "atomic_elements": [...],
+                "variations": [...],
+                "granular_specifics": [...]
+            }},
+            "constraint_2": {{
+                "atomic_elements": [...],
+                "variations": [...],
+                "granular_specifics": [...]
+            }}
+        }}
+        """
+
+        response = await self.model.ainvoke(prompt)
+        return self._parse_decomposition(response.content)
+
+    async def generate_intelligent_combinations(
+        self, decomposed_constraints, existing_queries=None, original_query=None
+    ):
+        """LLM generates smart combinations of atomic elements"""
+
+        if existing_queries is None:
+            existing_queries = []
+
+        existing_queries_str = (
+            "\n".join([f"- {q}" for q in existing_queries])
+            if existing_queries
+            else "None yet"
+        )
+
+        # Store all queries we've used to avoid repeats
+        if existing_queries is None:
+            existing_queries = []
+
+        # Add the original query as first in our tracking
+        all_queries_used = (
+            [original_query] + existing_queries
+            if original_query
+            else existing_queries
+        )
+        existing_queries_str = (
+            "\n".join([f"- {q}" for q in all_queries_used])
+            if all_queries_used
+            else "None yet"
+        )
+
+        prompt = f"""
+        Create search query variations using TWO strategies:
+
+        ORIGINAL QUERY: "{original_query if original_query else "Not provided"}"
+
+        ALREADY USED QUERIES (DO NOT REPEAT):
+        {existing_queries_str}
+
+        **STRATEGY 1: QUERY REFORMULATION** (5-8 variations)
+        Keep ALL key information but rephrase the entire query:
+        - Change word order and sentence structure
+        - Use synonyms for key terms
+        - Convert questions to statements or keyword phrases
+        - Maintain all specific details (names, dates, numbers)
+
+        **STRATEGY 2: RANGE SPLITTING** (10-15 variations)
+        For any time periods, ranges, or multiple options, create separate specific searches:
+        - Split year ranges into individual years
+        - Split time periods into specific decades/years
+        - Split "between X and Y" into individual values
+        - Create one search per specific value in any range
+
+        **EXAMPLES:**
+        Original: "Who won Nobel Prize between 1960-1965?"
+        - Reformulations: "Nobel Prize winner 1960-1965", "Nobel laureate from 1960 to 1965"
+        - Range splits: "Nobel Prize winner 1960", "Nobel Prize winner 1961", "Nobel Prize winner 1962", "Nobel Prize winner 1963", "Nobel Prize winner 1964", "Nobel Prize winner 1965"
+
+        Generate 15-25 search queries total (reformulations + range splits).
+        Focus on maximum specificity through systematic coverage.
+
+        Return as a valid JSON list of search queries:
+        ["query1", "query2", "query3"]
+        """
+
+        response = await self.model.ainvoke(prompt)
+        return self._parse_combinations(response.content)
+
+    def _parse_decomposition(self, content):
+        """Parse LLM decomposition response"""
+        try:
+            start = content.find("{")
+            end = content.rfind("}") + 1
+            if start != -1 and end != -1:
+                json_str = content[start:end]
+                return json.loads(json_str)
+        except Exception as e:
+            logger.error(f"Failed to parse decomposition: {e}")
+
+        # If parsing fails, return empty dict - let the system handle gracefully
+        logger.warning(
+            "Failed to parse constraint decomposition, returning empty dict"
+        )
+        return {}
+
+    def _parse_combinations(self, content):
+        """Parse LLM combinations response"""
+        try:
+            start = content.find("[")
+            end = content.rfind("]") + 1
+            if start != -1 and end != -1:
+                json_str = content[start:end]
+                return json.loads(json_str)
+        except Exception as e:
+            logger.error(f"Failed to parse combinations: {e}")
+
+        # If parsing fails, return empty list - let the system handle gracefully
+        logger.warning("Failed to parse LLM combinations, returning empty list")
+        return []
+
+
+class EarlyRejectionManager:
+    """Manages early rejection and confidence tracking"""
+
+    def __init__(self, model, positive_threshold=0.6, negative_threshold=0.3):
+        self.model = model
+        self.positive_threshold = positive_threshold
+        self.negative_threshold = negative_threshold
+        self.rejected_candidates = set()
+
+    async def quick_confidence_check(self, candidate, constraints):
+        """Quick confidence assessment for early rejection"""
+
+        prompt = f"""
+        Quickly assess if this candidate matches the search criteria:
+
+        Candidate: {candidate.name}
+        Available info: {getattr(candidate, "metadata", {})}
+
+        Constraints to match:
+        {[c.description for c in constraints]}
+
+        Provide:
+        1. **Positive confidence** (0.0-1.0): How likely this candidate matches
+        2. **Negative confidence** (0.0-1.0): How likely this candidate does NOT match
+        3. **Quick reasoning**: Brief explanation
+
+        Return as JSON:
+        {{
+            "positive_confidence": 0.X,
+            "negative_confidence": 0.X,
+            "reasoning": "brief explanation"
+        }}
+        """
+
+        try:
+            response = await self.model.ainvoke(prompt)
+            return self._parse_confidence(response.content)
+        except Exception as e:
+            logger.error(f"Quick confidence check failed: {e}")
+            return {
+                "positive_confidence": 0.5,
+                "negative_confidence": 0.3,
+                "reasoning": "fallback",
+            }
+
+    def should_reject_early(self, confidence_result):
+        """Determine if candidate should be rejected early"""
+        # positive = confidence_result.get("positive_confidence", 0.5)  # Not currently used
+        negative = confidence_result.get("negative_confidence", 0.3)
+
+        # Only reject if we have strong negative evidence (not just lack of positive evidence)
+        if negative > 0.85:
+            return (
+                True,
+                f"High negative confidence ({negative:.2f})",
+            )
+
+        return False, None
+
+    def should_continue_search(self, all_candidates, high_confidence_count):
+        """Determine if we should continue searching"""
+        # Stop if we have enough high-confidence candidates
+        if high_confidence_count >= 5:
+            return False, "Found sufficient high-confidence candidates"
+
+        # Stop if we have many candidates but low quality
+        if len(all_candidates) > 50 and high_confidence_count == 0:
+            return False, "Too many low-quality candidates"
+
+        return True, None
+
+    def _parse_confidence(self, content):
+        """Parse confidence assessment"""
+        try:
+            start = content.find("{")
+            end = content.rfind("}") + 1
+            if start != -1 and end != -1:
+                json_str = content[start:end]
+                return json.loads(json_str)
+        except Exception as e:
+            logger.error(f"Failed to parse confidence: {e}")
+
+        return {
+            "positive_confidence": 0.5,
+            "negative_confidence": 0.3,
+            "reasoning": "parse_error",
+        }
+
+
+class ModularStrategy(BaseSearchStrategy):
+    """
+    A strategy that showcases the new modular architecture using:
+    - constraint_checking module for candidate evaluation
+    - candidate_exploration module for search execution
+    - constraints module for constraint analysis
+    - LLM-driven intelligent constraint processing
+    - Early rejection and confidence-based filtering
+    - Immediate candidate evaluation
+    """
+
+    def __init__(
+        self,
+        model,
+        search,
+        all_links_of_system=None,
+        constraint_checker_type: str = "dual_confidence",  # dual_confidence, strict, threshold
+        exploration_strategy: str = "adaptive",  # parallel, adaptive, constraint_guided, diversity
+        early_rejection: bool = True,  # Enable early rejection by default
+        early_stopping: bool = True,  # Enable early stopping by default
+        llm_constraint_processing: bool = True,  # Enable LLM-driven constraint processing by default
+        immediate_evaluation: bool = True,  # Enable immediate candidate evaluation by default
+        **kwargs,
+    ):
+        super().__init__(all_links_of_system=all_links_of_system)
+
+        self.model = model
+        self.search_engine = search
+        self.search_engines = getattr(search, "search_engines", [])
+
+        # Initialize constraint analyzer
+        self.constraint_analyzer = ConstraintAnalyzer(self.model)
+
+        # Initialize LLM constraint processor if enabled
+        self.llm_processor = (
+            LLMConstraintProcessor(self.model)
+            if llm_constraint_processing
+            else None
+        )
+
+        # Initialize early rejection manager if enabled
+        self.early_rejection_manager = (
+            EarlyRejectionManager(self.model) if early_rejection else None
+        )
+
+        # Initialize constraint checker based on type (default to dual confidence)
+        self.constraint_checker = self._create_constraint_checker(
+            constraint_checker_type
+        )
+
+        # Initialize candidate explorer based on strategy
+        self.candidate_explorer = self._create_candidate_explorer(
+            exploration_strategy
+        )
+
+        # Initialize question generator
+        self.question_generator = StandardQuestionGenerator(model=self.model)
+
+        # Strategy configuration
+        self.constraint_checker_type = constraint_checker_type
+        self.exploration_strategy = exploration_strategy
+        self.early_rejection = early_rejection
+        self.early_stopping = early_stopping
+        self.llm_constraint_processing = llm_constraint_processing
+        self.immediate_evaluation = immediate_evaluation
+
+        logger.info(
+            f"Initialized ModularStrategy with {constraint_checker_type} checker, {exploration_strategy} explorer, "
+            f"early_rejection={early_rejection}, early_stopping={early_stopping}, "
+            f"llm_processing={llm_constraint_processing}, immediate_eval={immediate_evaluation}"
+        )
+
+    def _create_constraint_checker(self, checker_type: str):
+        """Create the appropriate constraint checker."""
+        if checker_type == "dual_confidence":
+            return DualConfidenceChecker(
+                model=self.model,
+                evidence_gatherer=self._gather_evidence_for_constraint,
+                negative_threshold=0.75,
+                positive_threshold=0.2,
+                uncertainty_penalty=0.1,
+                negative_weight=1.5,
+            )
+        elif checker_type == "strict":
+            return StrictChecker(
+                model=self.model,
+                evidence_gatherer=self._gather_evidence_for_constraint,
+            )
+        elif checker_type == "threshold":
+            return ThresholdChecker(
+                model=self.model,
+                evidence_gatherer=self._gather_evidence_for_constraint,
+                acceptance_threshold=0.7,
+            )
+        else:
+            raise ValueError(f"Unknown constraint checker type: {checker_type}")
+
+    def _create_candidate_explorer(self, strategy_type: str):
+        """Create the appropriate candidate explorer."""
+        if strategy_type == "parallel":
+            return ParallelExplorer(
+                search_engine=self.search_engine,
+                model=self.model,
+                max_workers=4,
+            )
+        elif strategy_type == "adaptive":
+            return AdaptiveExplorer(
+                search_engine=self.search_engine,
+                model=self.model,
+                learning_rate=0.1,
+            )
+        elif strategy_type == "constraint_guided":
+            return ConstraintGuidedExplorer(
+                search_engine=self.search_engine, model=self.model
+            )
+        elif strategy_type == "diversity":
+            return DiversityExplorer(
+                search_engine=self.search_engine,
+                model=self.model,
+                diversity_factor=0.3,
+            )
+        else:
+            raise ValueError(f"Unknown exploration strategy: {strategy_type}")
+
+    async def search(
+        self,
+        query: str,
+        search_engines: List[str] = None,
+        progress_callback=None,
+        **kwargs,
+    ) -> Tuple[str, Dict]:
+        """
+        Execute the modular search strategy.
+        """
+        try:
+            logger.info(f"Starting enhanced modular search for: {query}")
+
+            # Phase 1: Extract base constraints
+            if progress_callback:
+                progress_callback(
+                    {
+                        "phase": "constraint_analysis",
+                        "progress": 5,
+                        "message": "Analyzing query constraints",
+                    }
+                )
+
+            base_constraints = self.constraint_analyzer.extract_constraints(
+                query
+            )
+            logger.info(f"Extracted {len(base_constraints)} base constraints")
+
+            # Phase 2: LLM constraint processing (if enabled)
+            all_search_queries = []
+            if self.llm_constraint_processing and self.llm_processor:
+                if progress_callback:
+                    progress_callback(
+                        {
+                            "phase": "llm_processing",
+                            "progress": 15,
+                            "message": "LLM processing constraints intelligently",
+                        }
+                    )
+
+                logger.info("🤖 LLM CONSTRAINT PROCESSING ACTIVATED")
+                # LLM decomposition and combination
+                decomposed = await self.llm_processor.decompose_constraints_intelligently(
+                    base_constraints
+                )
+
+                # Pass existing base constraint queries to avoid duplication
+                existing_queries = [c.description for c in base_constraints]
+                logger.info("📋 BASE CONSTRAINT QUERIES:")
+                for i, base_query in enumerate(existing_queries, 1):
+                    logger.info(f"   BASE-{i:02d}: {base_query}")
+
+                intelligent_combinations = (
+                    await self.llm_processor.generate_intelligent_combinations(
+                        decomposed, existing_queries, query
+                    )
+                )
+
+                logger.info("🧠 LLM-GENERATED INTELLIGENT QUERIES:")
+                logger.info("### START_LLM_QUERIES ###")  # Grep-friendly marker
+                for i, llm_query in enumerate(intelligent_combinations, 1):
+                    logger.info(f"   LLM-{i:02d}: {llm_query}")
+                logger.info("### END_LLM_QUERIES ###")  # Grep-friendly marker
+
+                # OPTIMIZATION: Start with original query, then use LLM-generated targeted queries
+                # This ensures we search for the exact question first, then explore variations
+                all_search_queries = (
+                    [query] + intelligent_combinations
+                )  # Original query first, then LLM combinations
+                logger.info(
+                    f"🎯 Using original query + {len(intelligent_combinations)} targeted LLM search combinations (skipping broad base constraints)"
+                )
+                logger.info(
+                    f"📊 Optimized search strategies: {len(all_search_queries)} (original + {len(intelligent_combinations)} LLM queries)"
+                )
+            else:
+                logger.warning(
+                    "⚠️  LLM constraint processing is DISABLED - falling back to basic searches"
+                )
+                # Fallback to basic constraint searches
+                all_search_queries = [c.description for c in base_constraints]
+
+            # Phase 3: Enhanced candidate exploration with immediate evaluation
+            if progress_callback:
+                progress_callback(
+                    {
+                        "phase": "candidate_exploration",
+                        "progress": 25,
+                        "message": f"🔍 Exploring with {len(all_search_queries)} enhanced search strategies",
+                    }
+                )
+
+            all_candidates = []
+            high_confidence_count = 0
+            search_progress = 30
+
+            # DECOUPLED APPROACH: Separate search execution from candidate evaluation
+            candidate_evaluation_queue = asyncio.Queue()
+            evaluation_results = []
+            rejected_candidates = []  # Store rejected candidates for potential recovery
+
+            # Execute searches in parallel batches with decoupled evaluation
+            batch_size = 8  # Optimized for parallel execution
+            logger.info(
+                f"🚀 Starting enhanced exploration with {len(all_search_queries)} search queries (8 concurrent, decoupled evaluation)"
+            )
+
+            # Start background candidate evaluation task
+            evaluation_task = asyncio.create_task(
+                self._background_candidate_evaluation(
+                    candidate_evaluation_queue,
+                    base_constraints,
+                    evaluation_results,
+                    query,
+                    rejected_candidates,
+                )
+            )
+
+            for i in range(0, len(all_search_queries), batch_size):
+                batch = all_search_queries[i : i + batch_size]
+
+                if progress_callback:
+                    progress_callback(
+                        {
+                            "phase": "search_batch",
+                            "progress": search_progress,
+                            "message": f"🔍 Executing search batch {i // batch_size + 1}",
+                        }
+                    )
+
+                logger.info(
+                    f"📦 Processing batch {i // batch_size + 1}: {batch}"
+                )
+
+                # Execute batch searches in parallel using ThreadPoolExecutor
+                batch_results = []
+                with ThreadPoolExecutor(max_workers=8) as executor:
+                    # Submit all searches in the batch concurrently
+                    future_to_query = {
+                        executor.submit(
+                            self.candidate_explorer._execute_search, query
+                        ): query
+                        for query in batch
+                    }
+
+                    # Collect results as they complete
+                    for future in as_completed(future_to_query):
+                        query = future_to_query[future]
+                        try:
+                            result = future.result()
+                            batch_results.append(result)
+                        except Exception as e:
+                            logger.error(
+                                f"❌ Parallel search failed for '{query[:30]}...': {e}"
+                            )
+                            batch_results.append(e)
+
+                        # CRITICAL: Yield control to allow background evaluation task to run
+                        await asyncio.sleep(0)
+
+                # Process batch results - QUEUE CANDIDATES FOR BACKGROUND EVALUATION
+                for j, result in enumerate(batch_results):
+                    if isinstance(result, Exception):
+                        logger.error(f"❌ Search failed: {batch[j]} - {result}")
+                        continue
+
+                    candidates = self.candidate_explorer._extract_candidates_from_results(
+                        result, original_query=query
+                    )
+
+                    logger.info(
+                        f"🎯 Found {len(candidates)} candidates from query: '{batch[j][:50]}...'"
+                    )
+
+                    # QUEUE CANDIDATES for background evaluation (non-blocking)
+                    for candidate in candidates:
+                        await candidate_evaluation_queue.put(candidate)
+
+                # Progress tracking without blocking on evaluation
+                total_candidates = sum(
+                    len(
+                        self.candidate_explorer._extract_candidates_from_results(
+                            result, original_query=query
+                        )
+                    )
+                    for result in batch_results
+                    if not isinstance(result, Exception)
+                )
+
+                logger.info(
+                    f"📦 Batch {i // batch_size + 1}: queued {total_candidates} candidates for evaluation"
+                )
+
+                # CRITICAL: Yield control after each batch to allow background evaluation
+                await asyncio.sleep(
+                    0.1
+                )  # Small delay to let background task process
+
+                search_progress = min(search_progress + 10, 75)
+
+            # Signal completion to background evaluation and wait for final results
+            await candidate_evaluation_queue.put(
+                None
+            )  # Sentinel to signal completion
+
+            # Wait for background evaluation to complete
+            try:
+                await asyncio.wait_for(
+                    evaluation_task, timeout=30.0
+                )  # 30s timeout
+            except asyncio.TimeoutError:
+                logger.warning(
+                    "⚠️ Background evaluation timed out, using partial results"
+                )
+                evaluation_task.cancel()
+
+            # Collect all evaluated candidates
+            all_candidates = [
+                result for result in evaluation_results if result is not None
+            ]
+
+            logger.info(
+                f"🏁 Search completed: {len(all_candidates)} total candidates, {high_confidence_count} high-confidence"
+            )
+
+            # Phase 4: Final candidate evaluation (if immediate evaluation was disabled)
+            evaluated_candidates = all_candidates
+            if not self.immediate_evaluation:
+                if progress_callback:
+                    progress_callback(
+                        {
+                            "phase": "candidate_evaluation",
+                            "progress": 80,
+                            "message": f"🔍 Evaluating {len(all_candidates)} candidates",
+                        }
+                    )
+
+                evaluated_candidates = []
+                for candidate in all_candidates[:20]:  # Limit to top 20
+                    try:
+                        result = self.constraint_checker.check_candidate(
+                            candidate, base_constraints
+                        )
+                        candidate.evaluation_results = result.detailed_results
+                        candidate.score = result.total_score
+                        candidate.should_reject = result.should_reject
+
+                        if not result.should_reject:
+                            evaluated_candidates.append(candidate)
+
+                    except Exception as e:
+                        logger.error(
+                            f"💥 Error evaluating candidate {candidate.name}: {e}"
+                        )
+                        continue
+
+            # Phase 5: Select best candidate
+            if progress_callback:
+                progress_callback(
+                    {
+                        "phase": "result_selection",
+                        "progress": 90,
+                        "message": "🏆 Selecting best result",
+                    }
+                )
+
+            if not evaluated_candidates:
+                # Check all candidates including rejected ones
+                all_scored_candidates = []
+
+                # Add all candidates with scores
+                for c in all_candidates:
+                    if hasattr(c, "score") and c.score > 0:
+                        all_scored_candidates.append(c)
+
+                # Add rejected candidates with scores
+                for c in rejected_candidates:
+                    if hasattr(c, "score") and c.score > 0:
+                        all_scored_candidates.append(c)
+
+                if all_scored_candidates:
+                    # Sort by score
+                    all_scored_candidates.sort(
+                        key=lambda x: x.score, reverse=True
+                    )
+                    best_candidate = all_scored_candidates[0]
+
+                    # Accept if score is above minimum threshold (20%)
+                    if best_candidate.score >= 0.20:
+                        logger.info(
+                            f"🎯 Accepting best available candidate (recovered from rejected): {best_candidate.name} with score {best_candidate.score:.2%}"
+                        )
+                        evaluated_candidates = [best_candidate]
+                    else:
+                        logger.warning(
+                            f"❌ Best candidate {best_candidate.name} has too low score: {best_candidate.score:.2%}"
+                        )
+
+                if not evaluated_candidates:
+                    logger.warning(
+                        "❌ No valid candidates found after evaluation"
+                    )
+                    return "No valid candidates found after evaluation", {
+                        "strategy": "enhanced_modular",
+                        "constraint_checker": self.constraint_checker_type,
+                        "exploration_strategy": self.exploration_strategy,
+                        "early_rejection": self.early_rejection,
+                        "llm_processing": self.llm_constraint_processing,
+                        "total_searches": len(all_search_queries),
+                        "candidates_found": len(all_candidates),
+                        "candidates_valid": 0,
+                        "high_confidence_count": high_confidence_count,
+                    }
+
+            # Sort by score and select best
+            evaluated_candidates.sort(
+                key=lambda x: getattr(x, "score", 0), reverse=True
+            )
+            best_candidate = evaluated_candidates[0]
+
+            logger.info(
+                f"🏆 Best candidate: {best_candidate.name} with score {getattr(best_candidate, 'score', 0):.2%}"
+            )
+
+            # Phase 6: Generate final answer
+            if progress_callback:
+                progress_callback(
+                    {
+                        "phase": "final_answer",
+                        "progress": 95,
+                        "message": "📝 Generating final answer",
+                    }
+                )
+
+            answer = await self._generate_final_answer(
+                query, best_candidate, base_constraints
+            )
+
+            # Search Query Analysis Summary for easy analysis
+            logger.info("=" * 80)
+            logger.info("🔍 SEARCH QUERY ANALYSIS SUMMARY")
+            logger.info("=" * 80)
+            logger.info(
+                f"📊 TOTAL QUERIES GENERATED: {len(all_search_queries)}"
+            )
+            logger.info(
+                f"📋 BASE CONSTRAINT QUERIES: {len(existing_queries) if 'existing_queries' in locals() else 0}"
+            )
+            logger.info(
+                f"🧠 LLM INTELLIGENT QUERIES: {len(intelligent_combinations) if 'intelligent_combinations' in locals() else 0}"
+            )
+
+            if (
+                "intelligent_combinations" in locals()
+                and intelligent_combinations
+            ):
+                logger.info("\n🎯 SAMPLE LLM-GENERATED QUERIES (first 10):")
+                for i, query in enumerate(intelligent_combinations[:10], 1):
+                    logger.info(f"   SAMPLE-{i:02d}: {query}")
+
+            logger.info("=" * 80)
+
+            metadata = {
+                "strategy": "enhanced_modular",
+                "constraint_checker": self.constraint_checker_type,
+                "exploration_strategy": self.exploration_strategy,
+                "early_rejection_enabled": self.early_rejection,
+                "early_stopping_enabled": self.early_stopping,
+                "llm_processing_enabled": self.llm_constraint_processing,
+                "immediate_evaluation_enabled": self.immediate_evaluation,
+                "total_searches_generated": len(all_search_queries),
+                "candidates_found": len(all_candidates),
+                "candidates_evaluated": len(evaluated_candidates),
+                "high_confidence_count": high_confidence_count,
+                "best_candidate": best_candidate.name,
+                "best_score": getattr(best_candidate, "score", 0),
+            }
+
+            return answer, metadata
+
+        except Exception as e:
+            logger.error(f"💥 Error in enhanced modular search: {e}")
+            import traceback
+
+            logger.error(f"🔍 Traceback: {traceback.format_exc()}")
+            return f"Search failed: {str(e)}", {"error": str(e)}
+
+    async def _generate_final_answer(
+        self, query: str, best_candidate, constraints
+    ) -> str:
+        """Generate the final answer using the best candidate."""
+
+        constraint_info = "\n".join(
+            [f"- {c.description} (weight: {c.weight})" for c in constraints]
+        )
+
+        evaluation_info = ""
+        if hasattr(best_candidate, "evaluation_results"):
+            evaluation_info = "\n".join(
+                [
+                    f"- {result.get('constraint', 'Unknown')}: {result.get('score', 0):.0%}"
+                    for result in best_candidate.evaluation_results
+                ]
+            )
+
+        prompt = f"""Based on the search results, provide a comprehensive answer to: {query}
+
+Best candidate found: {best_candidate.name}
+Score: {best_candidate.score:.0%}
+
+Constraints analyzed:
+{constraint_info}
+
+Constraint evaluation results:
+{evaluation_info}
+
+Evidence summary: {getattr(best_candidate, "summary", "No summary available")}
+
+Provide a clear, factual answer that addresses the original question and explains how the candidate satisfies the constraints."""
+
+        response = await self.model.ainvoke(prompt)
+        return response.content
+
+    def _gather_evidence_for_constraint(self, candidate, constraint):
+        """Gather evidence for a constraint using actual search with caching."""
+        # Check cache first
+        cache = get_search_cache()
+        cache_key = normalize_entity_query(candidate.name, constraint.value)
+
+        cached_results = cache.get(cache_key, "modular_strategy")
+        if cached_results is not None:
+            logger.debug(
+                f"Using cached evidence for {candidate.name} - {constraint.value[:30]}..."
+            )
+            return cached_results
+
+        try:
+            # Build search query intelligently based on constraint type
+            query_parts = []
+
+            # Add candidate name
+            query_parts.append(f'"{candidate.name}"')
+
+            # Parse constraint value for key terms
+            constraint_value = constraint.value
+
+            # Remove common prefixes
+            prefixes_to_remove = [
+                "The individual is associated with",
+                "The answer must be",
+                "The character must be",
+                "The entity must be",
+                "Must be",
+                "Should be",
+                "Is",
+            ]
+
+            for prefix in prefixes_to_remove:
+                if constraint_value.startswith(prefix):
+                    constraint_value = constraint_value[len(prefix) :].strip()
+                    break
+
+            # Handle different constraint types
+            if constraint.type.value == "TEMPORAL":
+                # For temporal constraints, extract years/dates and search specifically
+                import re
+
+                years = re.findall(r"\b(19\d{2}|20\d{2})\b", constraint_value)
+                decades = re.findall(
+                    r"\b(19\d{2}s|20\d{2}s)\b", constraint_value
+                )
+
+                if years:
+                    for year in years:
+                        query_parts.append(year)
+                elif decades:
+                    for decade in decades:
+                        query_parts.append(decade)
+                else:
+                    query_parts.append(constraint_value)
+
+            elif constraint.type.value == "PROPERTY":
+                # For properties, focus on the specific characteristic
+                query_parts.append(constraint_value)
+
+            elif constraint.type.value == "STATISTIC":
+                # For statistics, include numbers and comparisons
+                query_parts.append(constraint_value)
+
+            else:
+                # Default: use the constraint value as-is
+                query_parts.append(constraint_value)
+
+            # Build final query
+            query = " ".join(query_parts)
+            logger.debug(f"Evidence search query: {query}")
+
+            # Execute search using the appropriate method
+            results = None
+
+            # Try different search methods based on what's available
+            if hasattr(self.search_engine, "run"):
+                results = self.search_engine.run(query)
+            elif hasattr(self.search_engine, "search"):
+                results = self.search_engine.search(query)
+            elif callable(self.search_engine):
+                results = self.search_engine(query)
+            else:
+                logger.error(
+                    f"Search engine has no callable method: {type(self.search_engine)}"
+                )
+                return []
+
+            # Process results
+            evidence = []
+
+            # Handle different result formats
+            if results is None:
+                logger.warning("Search returned None")
+                return []
+
+            if isinstance(results, list):
+                result_list = results
+            elif isinstance(results, dict):
+                # Try common keys for results
+                result_list = (
+                    results.get("results")
+                    or results.get("items")
+                    or results.get("data")
+                    or []
+                )
+            else:
+                logger.warning(f"Unknown search result type: {type(results)}")
+                result_list = []
+
+            # Extract top evidence (limit to 5 for better quality)
+            for i, result in enumerate(result_list[:5]):
+                if isinstance(result, dict):
+                    # Extract text content
+                    text = (
+                        result.get("snippet")
+                        or result.get("content")
+                        or result.get("description")
+                        or result.get("text")
+                        or ""
+                    )
+
+                    # Extract source information
+                    source = (
+                        result.get("url")
+                        or result.get("link")
+                        or result.get("source")
+                        or f"search_result_{i + 1}"
+                    )
+
+                    # Extract title
+                    title = result.get("title", "")
+
+                    # Calculate confidence based on result position and content
+                    base_confidence = 0.8 - (i * 0.1)  # Decay by position
+
+                    # Boost confidence if key terms are present
+                    if candidate.name.lower() in text.lower():
+                        base_confidence += 0.1
+                    if any(
+                        term.lower() in text.lower()
+                        for term in constraint_value.split()
+                    ):
+                        base_confidence += 0.1
+
+                    confidence = min(base_confidence, 0.95)
+
+                    evidence.append(
+                        {
+                            "text": text[:500],  # Limit text length
+                            "source": source,
+                            "confidence": confidence,
+                            "title": title,
+                            "full_text": text,  # Keep full text for detailed analysis
+                        }
+                    )
+                else:
+                    # Handle string results
+                    evidence.append(
+                        {
+                            "text": str(result)[:500],
+                            "source": f"search_result_{i + 1}",
+                            "confidence": 0.6,
+                            "title": "",
+                        }
+                    )
+
+            logger.debug(
+                f"Gathered {len(evidence)} evidence items for {candidate.name} - {constraint.value[:50]}..."
+            )
+
+            # Cache the results for future use
+            cache.put(
+                cache_key, evidence, "modular_strategy", ttl=1800
+            )  # 30 minutes TTL
+
+            return evidence
+
+        except Exception as e:
+            logger.error(f"Error gathering evidence: {e}", exc_info=True)
+            # Return empty list instead of mock evidence
+            return []
+
+    async def _background_candidate_evaluation(
+        self,
+        queue,
+        constraints,
+        results,
+        original_query=None,
+        rejected_candidates=None,
+    ):
+        """Background task to evaluate candidates without blocking search progress."""
+        logger.info("🔄 Started background candidate evaluation")
+
+        # Use provided rejected_candidates list or create new one
+        if rejected_candidates is None:
+            rejected_candidates = []
+
+        while True:
+            try:
+                # Get candidate from queue
+                candidate = await queue.get()
+
+                # Check for completion sentinel
+                if candidate is None:
+                    logger.info("🏁 Background evaluation completed")
+                    break
+
+                # Evaluate candidate with LLM pre-screening
+                try:
+                    # Always do full constraint evaluation to get scores
+                    result = self.constraint_checker.check_candidate(
+                        candidate, constraints, original_query=original_query
+                    )
+                    candidate.evaluation_results = result.detailed_results
+                    candidate.score = result.total_score
+                    candidate.should_reject = result.should_reject
+
+                    # Now check early rejection AFTER we have a score
+                    if self.early_rejection_manager:
+                        confidence = await self.early_rejection_manager.quick_confidence_check(
+                            candidate, constraints
+                        )
+
+                        should_reject, reason = (
+                            self.early_rejection_manager.should_reject_early(
+                                confidence
+                            )
+                        )
+                        if (
+                            should_reject and candidate.score < 0.5
+                        ):  # Only early reject if score is also low
+                            logger.debug(
+                                f"⚡ Early rejected {candidate.name}: {reason} (score: {candidate.score:.2%})"
+                            )
+                            # Store the candidate anyway for potential best candidate recovery
+                            rejected_candidates.append(candidate)
+                            continue
+
+                    if not result.should_reject:
+                        results.append(candidate)
+                        logger.info(
+                            f"✅ Accepted: {candidate.name} (score: {result.total_score:.2%})"
+                        )
+
+                        # Check for excellent candidates
+                        if result.total_score > 0.9:
+                            logger.info(
+                                f"🏆 EXCELLENT: {candidate.name} with {result.total_score:.1%} score"
+                            )
+                    else:
+                        # Store rejected candidates with scores for potential recovery
+                        rejected_candidates.append(candidate)
+                        logger.debug(
+                            f"❌ Rejected: {candidate.name} (score: {candidate.score:.2%})"
+                        )
+
+                except Exception as e:
+                    logger.error(f"💥 Error evaluating {candidate.name}: {e}")
+
+            except Exception as e:
+                logger.error(f"💥 Background evaluation error: {e}")
+
+    def analyze_topic(self, query: str) -> Dict:
+        """
+        Analyze a topic using the modular strategy.
+
+        This is the main entry point that implements the BaseSearchStrategy interface.
+        """
+        try:
+            # Run the search asynchronously
+            import asyncio
+
+            # Create a new event loop if none exists or if the current loop is running
+            try:
+                loop = asyncio.get_event_loop()
+                if loop.is_running():
+                    # If we're already in an async context, run in a new thread
+                    import concurrent.futures
+
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future = executor.submit(
+                            lambda: asyncio.run(self.search(query))
+                        )
+                        answer, metadata = future.result()
+                else:
+                    # If not in async context, run directly
+                    answer, metadata = loop.run_until_complete(
+                        self.search(query)
+                    )
+            except RuntimeError:
+                # No event loop, create one
+                answer, metadata = asyncio.run(self.search(query))
+
+            return {
+                "findings": [{"content": answer}],
+                "iterations": 1,
+                "final_answer": answer,
+                "current_knowledge": answer,
+                "metadata": metadata,
+                "links": getattr(self, "all_links_of_system", []),
+                "questions_by_iteration": getattr(
+                    self, "questions_by_iteration", []
+                ),
+            }
+
+        except Exception as e:
+            logger.error(f"Error in analyze_topic: {e}")
+            import traceback
+
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            return {
+                "findings": [],
+                "iterations": 0,
+                "final_answer": f"Analysis failed: {str(e)}",
+                "metadata": {"error": str(e)},
+                "links": [],
+                "questions_by_iteration": [],
+            }