local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/advanced_search_system/strategies/constrained_search_strategy.py

@@ -0,0 +1,1348 @@
+"""
+Constrained search strategy that progressively narrows down candidates.
+
+This strategy mimics human problem-solving by:
+1. Starting with the most restrictive constraints
+2. Finding candidates that match those constraints
+3. Progressively checking additional constraints
+4. Narrowing down the candidate pool step by step
+"""
+
+from datetime import datetime
+from typing import Any, Dict, List
+
+from langchain_core.language_models import BaseChatModel
+from loguru import logger
+
+from ...utilities.search_utilities import remove_think_tags
+from ..candidates.base_candidate import Candidate
+from ..constraints.base_constraint import Constraint, ConstraintType
+from ..evidence.base_evidence import Evidence, EvidenceType
+from .evidence_based_strategy import EvidenceBasedStrategy
+
+
+class ConstrainedSearchStrategy(EvidenceBasedStrategy):
+    """
+    Strategy that progressively narrows down candidates using constraints.
+
+    Key approach:
+    1. Rank constraints by restrictiveness
+    2. Start with most restrictive constraint
+    3. Find candidates matching that constraint
+    4. Progressively filter using additional constraints
+    5. Gather evidence only for promising candidates
+    """
+
+    def __init__(
+        self,
+        model: BaseChatModel,
+        search: Any,
+        all_links_of_system: List[str],
+        max_iterations: int = 20,
+        confidence_threshold: float = 0.85,
+        candidate_limit: int = 100,  # Increased to get more candidates
+        evidence_threshold: float = 0.6,
+        max_search_iterations: int = 2,
+        questions_per_iteration: int = 3,
+        min_candidates_per_stage: int = 20,  # Need more candidates before filtering
+    ):
+        """Initialize the constrained search strategy."""
+        super().__init__(
+            model=model,
+            search=search,
+            all_links_of_system=all_links_of_system,
+            max_iterations=max_iterations,
+            confidence_threshold=confidence_threshold,
+            candidate_limit=candidate_limit,
+            evidence_threshold=evidence_threshold,
+            max_search_iterations=max_search_iterations,
+            questions_per_iteration=questions_per_iteration,
+        )
+
+        self.min_candidates_per_stage = min_candidates_per_stage
+        self.constraint_ranking: List[Constraint] = []
+        self.stage_candidates: Dict[int, List[Candidate]] = {}
+
+        # Enable direct search optimization for entity identification
+        # Note: parent class already sets this, but we ensure it's True
+        self.use_direct_search = True
+        logger.info(
+            f"ConstrainedSearchStrategy init: use_direct_search={self.use_direct_search}"
+        )
+
+    def analyze_topic(self, query: str) -> Dict:
+        """Analyze topic using progressive constraint narrowing."""
+        # Initialize
+        self.all_links_of_system.clear()
+        self.questions_by_iteration = []
+        self.findings = []
+        self.iteration = 0
+
+        if self.progress_callback:
+            self.progress_callback(
+                "Analyzing query to identify constraints and rank by restrictiveness",
+                2,
+                {"phase": "initialization", "strategy": "constrained_search"},
+            )
+
+        # Extract and rank constraints
+        self.constraints = self.constraint_analyzer.extract_constraints(query)
+        self.constraint_ranking = self._rank_constraints_by_restrictiveness()
+
+        if self.progress_callback:
+            ranking_summary = ", ".join(
+                [
+                    f"{i + 1}. {c.description[:30]}..."
+                    for i, c in enumerate(self.constraint_ranking[:3])
+                ]
+            )
+            self.progress_callback(
+                f"Found {len(self.constraints)} constraints. Order: {ranking_summary}",
+                5,
+                {
+                    "phase": "constraint_ranking",
+                    "constraint_count": len(self.constraints),
+                    "ranking": [
+                        (c.description, c.type.value)
+                        for c in self.constraint_ranking
+                    ],
+                },
+            )
+
+        # Add initial analysis finding
+        initial_finding = {
+            "phase": "Constraint Analysis",
+            "content": self._format_constraint_analysis(),
+            "timestamp": self._get_timestamp(),
+        }
+        self.findings.append(initial_finding)
+
+        # Progressive constraint search
+        self._progressive_constraint_search()
+
+        # Add search summary finding
+        search_finding = {
+            "phase": "Progressive Search Summary",
+            "content": self._format_search_summary(),
+            "timestamp": self._get_timestamp(),
+        }
+        self.findings.append(search_finding)
+
+        # Evidence gathering for final candidates
+        self._focused_evidence_gathering()
+
+        # Add evidence summary finding
+        evidence_finding = {
+            "phase": "Evidence Summary",
+            "content": self._format_evidence_summary(),
+            "timestamp": self._get_timestamp(),
+        }
+        self.findings.append(evidence_finding)
+
+        # Add comprehensive debug summary
+        debug_finding = {
+            "phase": "Debug Summary",
+            "content": self._format_debug_summary(),
+            "timestamp": self._get_timestamp(),
+            "metadata": {
+                "total_searches": (
+                    len(self.search_history)
+                    if hasattr(self, "search_history")
+                    else 0
+                ),
+                "final_candidates": len(self.candidates),
+                "constraints_used": len(self.constraints),
+                "stages_completed": len(self.stage_candidates),
+            },
+        }
+        self.findings.append(debug_finding)
+
+        # Final synthesis
+        return self._synthesize_final_answer(query)
+
+    def _rank_constraints_by_restrictiveness(self) -> List[Constraint]:
+        """Rank constraints from most to least restrictive."""
+        # Scoring system for restrictiveness
+        restrictiveness_scores = []
+
+        for constraint in self.constraints:
+            score = 0
+
+            # Type-based scoring
+            if constraint.type == ConstraintType.STATISTIC:
+                score += 10  # Numbers are usually very restrictive
+            elif constraint.type == ConstraintType.EVENT:
+                score += 8  # Events/time periods are restrictive
+            elif constraint.type == ConstraintType.LOCATION:
+                score += 6  # Locations are moderately restrictive
+            elif constraint.type == ConstraintType.PROPERTY:
+                score += 4  # Properties are less restrictive
+
+            # Specificity scoring
+            if constraint.value:
+                # Check for specific markers
+                if any(char.isdigit() for char in constraint.value):
+                    score += 5  # Contains numbers
+                if len(constraint.value.split()) > 3:
+                    score += 3  # Longer, more specific
+                if any(
+                    term in constraint.value.lower()
+                    for term in ["specific", "exact", "only", "must"]
+                ):
+                    score += 2  # Explicit specificity
+
+            restrictiveness_scores.append((constraint, score))
+
+        # Sort by score (highest first)
+        ranked = sorted(
+            restrictiveness_scores, key=lambda x: x[1], reverse=True
+        )
+        return [constraint for constraint, _ in ranked]
+
+    def _progressive_constraint_search(self):
+        """Progressively search using constraints from most to least restrictive."""
+        current_candidates = []
+
+        for stage, constraint in enumerate(self.constraint_ranking):
+            self.current_stage = stage  # Track current stage for logging
+            if self.progress_callback:
+                stage_desc = f"[{constraint.type.value}] {constraint.value} ({len(current_candidates)} candidates)"
+                self.progress_callback(
+                    f"Stage {stage + 1}/{len(self.constraint_ranking)}: {stage_desc}",
+                    10 + (stage * 15),
+                    {
+                        "phase": "progressive_search",
+                        "stage": stage + 1,
+                        "total_stages": len(self.constraint_ranking),
+                        "constraint": constraint.description,
+                        "constraint_type": constraint.type.value,
+                        "constraint_value": constraint.value,
+                        "current_candidates": len(current_candidates),
+                        "search_intent": f"Finding entities matching: {constraint.value}",
+                    },
+                )
+
+            if stage == 0:
+                # First stage - find initial candidates
+                current_candidates = self._search_with_single_constraint(
+                    constraint
+                )
+            else:
+                # Subsequent stages - filter existing candidates
+                current_candidates = self._filter_candidates_with_constraint(
+                    current_candidates, constraint
+                )
+
+            # Store stage results
+            self.stage_candidates[stage] = current_candidates.copy()
+
+            if self.progress_callback:
+                candidate_names = ", ".join(
+                    [c.name for c in current_candidates[:3]]
+                )
+                more = (
+                    f" (+{len(current_candidates) - 3})"
+                    if len(current_candidates) > 3
+                    else ""
+                )
+                change = len(current_candidates) - len(
+                    self.stage_candidates.get(stage - 1, [])
+                )
+                change_str = f" (Δ{change:+d})" if stage > 0 else ""
+
+                self.progress_callback(
+                    f"Stage {stage + 1} complete: {len(current_candidates)} candidates{change_str}. {candidate_names}{more}",
+                    None,
+                    {
+                        "phase": "stage_complete",
+                        "stage": stage + 1,
+                        "candidates_found": len(current_candidates),
+                        "candidates_delta": change if stage > 0 else 0,
+                        "sample": [c.name for c in current_candidates[:10]],
+                    },
+                )
+
+            # Add stage finding
+            stage_finding = {
+                "phase": f"Stage {stage + 1} - {constraint.type.value}",
+                "content": self._format_stage_results(
+                    stage, constraint, current_candidates
+                ),
+                "timestamp": self._get_timestamp(),
+            }
+            self.findings.append(stage_finding)
+
+            # Continue applying constraints unless we have very few candidates
+            if len(current_candidates) <= 3:
+                if self.progress_callback:
+                    self.progress_callback(
+                        f"Too few candidates ({len(current_candidates)}) - stopping constraint application",
+                        None,
+                        {
+                            "phase": "early_stop",
+                            "candidates_remaining": len(current_candidates),
+                        },
+                    )
+                break
+
+            # Stop if no candidates remain
+            if not current_candidates:
+                # Backtrack to previous stage if possible
+                if stage > 0:
+                    current_candidates = self.stage_candidates[stage - 1]
+                break
+
+        self.candidates = current_candidates[: self.candidate_limit]
+
+    def _search_with_single_constraint(
+        self, constraint: Constraint
+    ) -> List[Candidate]:
+        """Search for candidates using a single constraint."""
+        candidates = []
+
+        # Generate targeted queries for this constraint
+        queries = self._generate_constraint_specific_queries(constraint)
+
+        # Add more diverse query patterns
+        additional_queries = self._generate_additional_queries(constraint)
+        queries.extend(additional_queries)
+
+        # Diversify query execution order
+        import random
+
+        random.shuffle(queries)
+
+        for i, query in enumerate(queries[:20]):  # Increased query limit
+            if self.progress_callback:
+                # Show query and what we're looking for
+                self.progress_callback(
+                    f"Q{i + 1}/{min(20, len(queries))}: '{query}' | Found: {len(candidates)} candidates",
+                    None,
+                    {
+                        "phase": "constraint_search",
+                        "query": query,
+                        "query_index": i + 1,
+                        "total_queries": min(20, len(queries)),
+                        "constraint_type": constraint.type.value,
+                        "constraint_value": constraint.value,
+                        "candidates_so_far": len(candidates),
+                        "search_context": f"Stage {getattr(self, 'current_stage', 0) + 1}: {constraint.value}",
+                    },
+                )
+
+            results = self._execute_search(query)
+
+            # Validate search results before extraction
+            if self._validate_search_results(results, constraint):
+                extracted = self._extract_relevant_candidates(
+                    results, constraint
+                )
+                candidates.extend(extracted)
+
+                # Track stage information in search history
+                if self.search_history:
+                    self.search_history[-1]["stage"] = getattr(
+                        self, "current_stage", 0
+                    )
+                    self.search_history[-1]["results_count"] = len(extracted)
+                    self.search_history[-1]["results_preview"] = results.get(
+                        "current_knowledge", ""
+                    )[:200]
+            else:
+                logger.info(f"Skipping invalid results for query: {query}")
+
+            # Continue searching to build a comprehensive list
+            # Don't stop too early - we want diversity
+            if len(candidates) >= self.candidate_limit * 2:
+                break
+
+        return self._deduplicate_candidates(candidates)
+
+    def _generate_additional_queries(self, constraint: Constraint) -> List[str]:
+        """Generate additional diverse queries for better coverage."""
+        queries = []
+        base_value = constraint.value
+
+        # Add reference source queries
+        queries.extend(
+            [
+                f"reference {base_value}",
+                f"authoritative {base_value}",
+                f"official {base_value}",
+            ]
+        )
+
+        # Add structured data queries
+        if constraint.type == ConstraintType.STATISTIC:
+            queries.extend(
+                [
+                    f"spreadsheet {base_value}",
+                    f"dataset {base_value}",
+                    f"statistical analysis {base_value}",
+                    f"quantitative {base_value}",
+                ]
+            )
+        elif constraint.type == ConstraintType.PROPERTY:
+            queries.extend(
+                [
+                    f"characterized by {base_value}",
+                    f"known for {base_value}",
+                    f"featuring {base_value}",
+                ]
+            )
+        else:
+            # Generic comprehensive queries
+            queries.extend(
+                [
+                    f"exhaustive {base_value}",
+                    f"thorough {base_value}",
+                    f"detailed {base_value}",
+                ]
+            )
+
+        return queries
+
+    def _generate_constraint_specific_queries(
+        self, constraint: Constraint
+    ) -> List[str]:
+        """Generate queries specific to a constraint type."""
+        queries = []
+        base_value = constraint.value
+
+        # Add context from other constraints for more targeted searches
+        context_parts = []
+        if hasattr(self, "constraints") and self.constraints:
+            for other_constraint in self.constraints[
+                :2
+            ]:  # Use top 2 constraints for context
+                if other_constraint.id != constraint.id:
+                    context_parts.append(other_constraint.value)
+
+        # Base queries using the constraint description
+        if hasattr(constraint, "description") and constraint.description:
+            queries.append(constraint.description)
+            if context_parts:
+                queries.append(f"{constraint.description} {context_parts[0]}")
+
+        # Type-specific patterns
+        if constraint.type == ConstraintType.STATISTIC:
+            # Numeric constraints - look for quantitative information
+            queries.extend(
+                [
+                    f"list {base_value}",
+                    f"complete {base_value}",
+                    f"all {base_value}",
+                    f"comprehensive {base_value}",
+                    f"database {base_value}",
+                    f"statistics {base_value}",
+                    f"data {base_value}",
+                    f"comparison {base_value}",
+                ]
+            )
+
+        elif (
+            constraint.type == ConstraintType.EVENT
+            or hasattr(constraint.type, "value")
+            and constraint.type.value == "temporal"
+        ):
+            # Time-based constraints
+            queries.extend(
+                [
+                    f"during {base_value}",
+                    f"in {base_value}",
+                    f"list {base_value}",
+                    f"comprehensive {base_value}",
+                    f"all from {base_value}",
+                    f"complete list {base_value}",
+                    f"history {base_value}",
+                    f"timeline {base_value}",
+                ]
+            )
+
+        elif constraint.type == ConstraintType.PROPERTY:
+            # Property constraints - characteristics and attributes
+            queries.extend(
+                [
+                    f"with {base_value}",
+                    f"having {base_value}",
+                    f"characterized by {base_value}",
+                    f"examples {base_value}",
+                    f"instances {base_value}",
+                    f"who {base_value}",
+                    f"which {base_value}",
+                    f"known for {base_value}",
+                ]
+            )
+        else:
+            # Generic queries
+            queries.extend(
+                [
+                    f"{base_value}",
+                    f"list {base_value}",
+                    f"examples {base_value}",
+                    f"all {base_value}",
+                    f"complete {base_value}",
+                ]
+            )
+
+        # Add combined queries with other constraints
+        if context_parts:
+            queries.extend(
+                [
+                    f"{base_value} {context_parts[0]}",
+                    f"list {base_value} with {context_parts[0]}",
+                    f"{base_value} and {context_parts[0]}",
+                ]
+            )
+
+        return queries
+
+    def _filter_candidates_with_constraint(
+        self, candidates: List[Candidate], constraint: Constraint
+    ) -> List[Candidate]:
+        """Filter existing candidates using an additional constraint."""
+        filtered = []
+
+        for candidate in candidates:
+            # Check if candidate matches the constraint
+            query = f"{candidate.name} {constraint.value}"
+
+            results = self._execute_search(query)
+
+            # Quick evidence check
+            evidence = self._quick_evidence_check(
+                results, candidate, constraint
+            )
+
+            if evidence.confidence >= 0.5:  # Lower threshold for filtering
+                candidate.add_evidence(constraint.id, evidence)
+                filtered.append(candidate)
+
+        return filtered
+
+    def _extract_relevant_candidates(
+        self, results: Dict, constraint: Constraint
+    ) -> List[Candidate]:
+        """Extract candidates relevant to a specific constraint."""
+        content = results.get("current_knowledge", "")
+
+        # If no content, return empty list
+        if not content or "Error" in content or "No results found" in content:
+            logger.warning(
+                f"No valid content to extract candidates from for constraint: {constraint.description}"
+            )
+            return []
+
+        # Determine what type of entity we're looking for
+        entity_type = getattr(self, "entity_type", "entity")
+
+        # Use LLM to extract entities matching the constraint
+        prompt = f"""Analyze these search results and extract {entity_type} names that could satisfy this constraint:
+
+Constraint: {constraint.description}
+Type: {constraint.type.value}
+Value: {constraint.value}
+
+Search Results:
+{content}
+
+Your task:
+1. Understand what the constraint is asking for
+2. Identify mentions of specific {entity_type} names in the search results
+3. Extract only those names that could potentially satisfy the constraint
+4. Focus on proper nouns and specific names
+
+Important:
+- Extract actual {entity_type} names, not descriptions or categories
+- If the search results mention a specific {entity_type} that matches the constraint criteria, extract it
+- Be thorough - don't miss names that are mentioned in passing
+- Consider the context to determine if a name is relevant to the constraint
+
+Return one {entity_type} name per line. Only include names that could satisfy the constraint."""
+
+        try:
+            response = self.model.invoke(prompt)
+            extracted_text = remove_think_tags(response.content)
+
+            candidates = []
+            seen_names = set()  # Track unique names
+
+            for line in extracted_text.strip().split("\n"):
+                name = line.strip()
+                # Remove common list markers and clean up
+                name = name.lstrip("- •·*0123456789.").strip()
+
+                # Skip empty lines or very short names
+                if not name or len(name) <= 2:
+                    continue
+
+                # Normalize for deduplication
+                normalized_name = name.lower()
+                if normalized_name in seen_names:
+                    continue
+
+                # Exclude meta-commentary patterns
+                exclude_patterns = [
+                    "search result",
+                    "based on",
+                    "provided",
+                    "found",
+                    "does not",
+                    "doesn't",
+                    "cannot",
+                    "there are no",
+                    "according to",
+                    "mentions",
+                    "discusses",
+                    "shows that",
+                    "indicates",
+                    "suggests",
+                    "appears",
+                    "seems",
+                    "search",
+                    "constraint",
+                    "extract",
+                    "entity",
+                ]
+
+                # Check if it's meta-commentary
+                is_meta = any(
+                    pattern in name.lower() for pattern in exclude_patterns
+                )
+                is_too_long = (
+                    len(name.split()) > 10
+                )  # Very long strings are usually explanations
+                is_sentence = name.endswith(".") and len(name.split()) > 5
+
+                if not is_meta and not is_too_long and not is_sentence:
+                    # Accept various name formats
+                    if (
+                        name[0].isupper()  # Capitalized
+                        or any(c.isupper() for c in name)  # Has capitals
+                        or any(c.isdigit() for c in name)  # Contains numbers
+                        or any(
+                            c in name
+                            for c in ["-", "&", "/", ":", "(", ")", '"', "'"]
+                        )  # Special chars
+                        or len(name.split()) <= 6
+                    ):  # Reasonable length phrases
+                        candidate = Candidate(name=name)
+                        candidates.append(candidate)
+                        seen_names.add(normalized_name)
+
+            # Log extraction results for debugging
+            logger.info(
+                f"Extracted {len(candidates)} candidates for constraint: {constraint.description}"
+            )
+            if candidates:
+                logger.debug(
+                    f"Sample candidates: {[c.name for c in candidates[:5]]}"
+                )
+
+            return candidates[:50]  # Limit per search
+
+        except Exception as e:
+            logger.error(f"Error extracting candidates: {e}")
+            import traceback
+
+            logger.error(traceback.format_exc())
+            return []
+
+    def _quick_evidence_check(
+        self, results: Dict, candidate: Candidate, constraint: Constraint
+    ) -> Evidence:
+        """Quick evidence check for filtering with enhanced scoring."""
+        content = results.get("current_knowledge", "")
+        search_results = results.get("search_results", [])
+
+        # Initialize confidence components
+        name_presence = 0.0
+        constraint_presence = 0.0
+        co_occurrence = 0.0
+        context_quality = 0.0
+
+        candidate_lower = candidate.name.lower()
+        value_lower = constraint.value.lower()
+        content_lower = content.lower()
+
+        # Check candidate name presence
+        if candidate_lower in content_lower:
+            name_count = content_lower.count(candidate_lower)
+            name_presence = min(
+                0.3 + (name_count * 0.05), 0.4
+            )  # More occurrences = higher confidence
+
+        # Check constraint value presence
+        if value_lower in content_lower:
+            value_count = content_lower.count(value_lower)
+            constraint_presence = min(0.3 + (value_count * 0.05), 0.4)
+
+        # Check co-occurrence and proximity
+        if name_presence > 0 and constraint_presence > 0:
+            # Find all positions
+            name_positions = []
+            start = 0
+            while start < len(content_lower):
+                pos = content_lower.find(candidate_lower, start)
+                if pos == -1:
+                    break
+                name_positions.append(pos)
+                start = pos + 1
+
+            value_positions = []
+            start = 0
+            while start < len(content_lower):
+                pos = content_lower.find(value_lower, start)
+                if pos == -1:
+                    break
+                value_positions.append(pos)
+                start = pos + 1
+
+            # Calculate minimum distance
+            if name_positions and value_positions:
+                min_distance = min(
+                    abs(n - v) for n in name_positions for v in value_positions
+                )
+
+                if min_distance < 100:  # Very close proximity
+                    co_occurrence = 0.2
+                elif min_distance < 200:  # Close proximity
+                    co_occurrence = 0.15
+                elif min_distance < 500:  # Moderate proximity
+                    co_occurrence = 0.1
+                else:  # Same document
+                    co_occurrence = 0.05
+
+        # Check result quality
+        if search_results:
+            # Count how many results mention both candidate and constraint
+            relevant_results = 0
+            for result in search_results[:10]:
+                title = result.get("title", "").lower()
+                snippet = result.get("snippet", "").lower()
+
+                if (
+                    candidate_lower in title or candidate_lower in snippet
+                ) and (value_lower in title or value_lower in snippet):
+                    relevant_results += 1
+
+            context_quality = min(relevant_results * 0.05, 0.2)
+
+        # Calculate final confidence
+        confidence = (
+            name_presence
+            + constraint_presence
+            + co_occurrence
+            + context_quality
+        )
+
+        # Apply constraint type weight
+        if constraint.type == ConstraintType.STATISTIC:
+            confidence *= 1.1  # Numeric constraints need precise matching
+        elif constraint.type == ConstraintType.PROPERTY:
+            confidence *= 0.95  # Properties can be more flexible
+
+        return Evidence(
+            claim=f"Evidence for {candidate.name} matching {constraint.description}",
+            confidence=min(confidence, 1.0),
+            type=EvidenceType.INFERENCE,
+            source="quick_evidence_check",
+            metadata={
+                "name_presence": name_presence,
+                "constraint_presence": constraint_presence,
+                "co_occurrence": co_occurrence,
+                "context_quality": context_quality,
+            },
+        )
+
+    def _focused_evidence_gathering(self):
+        """Gather detailed evidence for the narrowed candidates."""
+        if self.progress_callback:
+            constraint_count = len(self.constraints)
+            evidence_needed = len(self.candidates) * constraint_count
+            self.progress_callback(
+                f"Verifying {len(self.candidates)} candidates against {constraint_count} constraints ({evidence_needed} checks)",
+                80,
+                {
+                    "phase": "evidence_gathering",
+                    "candidate_count": len(self.candidates),
+                    "constraint_count": constraint_count,
+                    "total_evidence_needed": evidence_needed,
+                },
+            )
+
+        for i, candidate in enumerate(self.candidates):
+            for j, constraint in enumerate(self.constraints):
+                # Skip if we already have evidence from filtering
+                if constraint.id in candidate.evidence:
+                    continue
+
+                # Detailed evidence search
+                query = f'"{candidate.name}" {constraint.value} verification'
+                results = self._execute_search(query)
+
+                evidence = self.evidence_evaluator.extract_evidence(
+                    results.get("current_knowledge", ""),
+                    candidate.name,
+                    constraint,
+                )
+
+                candidate.add_evidence(constraint.id, evidence)
+
+                if (
+                    self.progress_callback and i < 5
+                ):  # Report progress for top candidates
+                    conf_emoji = (
+                        "✓"
+                        if evidence.confidence >= self.evidence_threshold
+                        else "○"
+                    )
+                    self.progress_callback(
+                        f"{conf_emoji} {candidate.name} | {constraint.type.value}: {evidence.confidence:.0%}",
+                        None,
+                        {
+                            "phase": "evidence_detail",
+                            "candidate": candidate.name,
+                            "constraint": constraint.description,
+                            "constraint_type": constraint.type.value,
+                            "confidence": evidence.confidence,
+                            "evidence_type": evidence.type.value,
+                            "meets_threshold": evidence.confidence
+                            >= self.evidence_threshold,
+                        },
+                    )
+
+        # Final scoring
+        for candidate in self.candidates:
+            candidate.calculate_score(self.constraints)
+
+        # Sort by score
+        self.candidates.sort(key=lambda c: c.score, reverse=True)
+
+    def _deduplicate_candidates(
+        self, candidates: List[Candidate]
+    ) -> List[Candidate]:
+        """Remove duplicate candidates."""
+        seen = {}
+        unique = []
+
+        for candidate in candidates:
+            key = candidate.name.lower().strip()
+            if key not in seen:
+                seen[key] = candidate
+                unique.append(candidate)
+
+        return unique
+
+    def _format_constraint_analysis(self) -> str:
+        """Format initial constraint analysis."""
+        analysis = "**Query Constraint Analysis**\n\n"
+        analysis += f"Total constraints identified: {len(self.constraints)}\n\n"
+        analysis += "**Constraint Ranking (by restrictiveness):**\n"
+
+        for i, constraint in enumerate(self.constraint_ranking):
+            score = self._calculate_restrictiveness_score(constraint)
+            analysis += (
+                f"{i + 1}. [{constraint.type.value}] {constraint.description}\n"
+            )
+            analysis += f"   Restrictiveness score: {score}\n"
+            analysis += f"   Value: {constraint.value}\n\n"
+
+        return analysis
+
+    def _format_debug_summary(self) -> str:
+        """Format comprehensive debug summary."""
+        summary = "**Debug Summary**\n\n"
+
+        # Constraint analysis
+        summary += "**Constraint Processing:**\n"
+        for i, constraint in enumerate(self.constraint_ranking):
+            score = self._calculate_restrictiveness_score(constraint)
+            summary += f"{i + 1}. [{constraint.type.value}] {constraint.value} (score: {score})\n"
+
+        # Search progression
+        summary += "\n**Search Progression:**\n"
+        if hasattr(self, "stage_candidates"):
+            for stage, candidates in self.stage_candidates.items():
+                summary += f"Stage {stage + 1}: {len(candidates)} candidates\n"
+
+        # Evidence coverage
+        summary += "\n**Evidence Coverage:**\n"
+
+        for i, candidate in enumerate(self.candidates[:5]):
+            evidence_count = len(candidate.evidence)
+            satisfied = sum(
+                1
+                for c in self.constraints
+                if c.id in candidate.evidence
+                and candidate.evidence[c.id].confidence
+                >= self.evidence_threshold
+            )
+
+            summary += f"{i + 1}. {candidate.name}: {evidence_count} evidence, "
+            summary += f"{satisfied}/{len(self.constraints)} constraints\n"
+
+        # Search statistics
+        summary += "\n**Search Statistics:**\n"
+        total_discovered = (
+            sum(len(c) for c in self.stage_candidates.values())
+            if hasattr(self, "stage_candidates")
+            else 0
+        )
+        summary += f"Total candidates discovered: {total_discovered}\n"
+        summary += f"Final candidates: {len(self.candidates)}\n"
+        summary += f"Constraints: {len(self.constraints)}\n"
+
+        return summary
+
+    def _calculate_restrictiveness_score(self, constraint: Constraint) -> int:
+        """Calculate restrictiveness score for a constraint."""
+        score = 0
+
+        # Type-based scoring
+        if constraint.type == ConstraintType.STATISTIC:
+            score += 10
+        elif constraint.type == ConstraintType.EVENT:
+            score += 8
+        elif constraint.type == ConstraintType.LOCATION:
+            score += 6
+        elif constraint.type == ConstraintType.PROPERTY:
+            score += 4
+
+        # Specificity scoring
+        if constraint.value:
+            if any(char.isdigit() for char in constraint.value):
+                score += 5
+            if len(constraint.value.split()) > 3:
+                score += 3
+            if any(
+                term in constraint.value.lower()
+                for term in ["specific", "exact", "only", "must"]
+            ):
+                score += 2
+
+        return score
+
+    def _format_stage_results(
+        self, stage: int, constraint: Constraint, candidates: List[Candidate]
+    ) -> str:
+        """Format results for a search stage with detailed information."""
+        result = f"**Search Stage {stage + 1}**\n\n"
+        result += f"Constraint: {constraint.description}\n"
+        result += f"Type: {constraint.type.value}\n"
+        result += f"Search Value: {constraint.value}\n"
+        result += f"Candidates found: {len(candidates)}\n\n"
+
+        # Add search statistics
+        result += "**Search Statistics:**\n"
+        if hasattr(self, "search_history"):
+            stage_searches = [
+                s for s in self.search_history if s.get("stage", -1) == stage
+            ]
+            result += f"- Queries executed: {len(stage_searches)}\n"
+            result += f"- Total results analyzed: {getattr(self, f'stage_{stage}_results_count', 0)}\n"
+
+        result += f"- Candidates before filtering: {getattr(self, f'stage_{stage}_raw_candidates', len(candidates))}\n"
+        result += f"- Candidates after deduplication: {len(candidates)}\n\n"
+
+        if candidates:
+            result += "**Top Candidates:**\n"
+            # Group candidates to show variety
+            grouped = self._group_similar_candidates(candidates[:20])
+            for group_name, group_items in grouped.items():
+                result += f"\n{group_name} ({len(group_items)} items):\n"
+                for i, candidate in enumerate(group_items[:5]):
+                    result += f"  {i + 1}. {candidate.name}\n"
+                if len(group_items) > 5:
+                    result += f"  ... and {len(group_items) - 5} more\n"
+        else:
+            result += "No candidates found for this constraint.\n"
+
+        # Add sample search results for debugging
+        if hasattr(self, "search_history") and candidates:
+            result += "\n**Sample Search Results:**\n"
+            recent_searches = [
+                s
+                for s in self.search_history[-3:]
+                if s.get("stage", -1) == stage
+            ]
+            for search in recent_searches[:2]:
+                result += f"- Query: '{search.get('query', '')}'\n"
+                if "results_preview" in search:
+                    result += (
+                        f"  Preview: {search['results_preview'][:100]}...\n"
+                    )
+
+        return result
+
+    def _format_search_summary(self) -> str:
+        """Format progressive search summary."""
+        summary = "**Progressive Search Summary**\n\n"
+
+        # Show search progression
+        summary += "**Stage-by-Stage Filtering:**\n"
+        prev_count = 0
+
+        for stage, candidates in self.stage_candidates.items():
+            constraint = (
+                self.constraint_ranking[stage]
+                if stage < len(self.constraint_ranking)
+                else None
+            )
+            if constraint:
+                count = len(candidates)
+                change = count - prev_count if stage > 0 else count
+                change_str = f" ({change:+d})" if stage > 0 else ""
+
+                summary += f"\nStage {stage + 1} [{constraint.type.value}]: {constraint.value[:40]}\n"
+                summary += f"  Results: {count} candidates{change_str}\n"
+
+                if candidates:
+                    # Group candidates by type
+                    grouped = self._group_similar_candidates(candidates[:20])
+                    for group_name, group_items in grouped.items():
+                        summary += f"  {group_name}: {len(group_items)} items\n"
+                        for item in group_items[:3]:
+                            summary += f"    • {item.name}\n"
+                        if len(group_items) > 3:
+                            summary += (
+                                f"    ... and {len(group_items) - 3} more\n"
+                            )
+
+                prev_count = count
+
+        summary += (
+            f"\n**Final Result: {len(self.candidates)} candidates selected**\n"
+        )
+
+        return summary
+
+    def _format_evidence_summary(self) -> str:
+        """Format evidence gathering summary."""
+        summary = "**Evidence Gathering Summary**\n\n"
+
+        for i, candidate in enumerate(self.candidates[:5]):
+            summary += f"**{i + 1}. {candidate.name}**\n"
+
+            for constraint in self.constraints:
+                evidence = candidate.evidence.get(constraint.id)
+                if evidence:
+                    conf_str = f"{evidence.confidence:.0%}"
+                    summary += (
+                        f"  • {constraint.description[:40]}...: {conf_str}\n"
+                    )
+                else:
+                    summary += (
+                        f"  • {constraint.description[:40]}...: No evidence\n"
+                    )
+
+            summary += f"  Overall Score: {candidate.score:.2f}\n\n"
+
+        return summary
+
+    # Commented out to use the parent's optimized _execute_search method
+    '''def _execute_search(self, search_query: str) -> Dict:
+        """Execute a comprehensive search using source-based strategy for complex queries."""
+        if not hasattr(self, "search_history"):
+            self.search_history = []
+
+        self.search_history.append(
+            {
+                "query": search_query,
+                "timestamp": self._get_timestamp(),
+                "iteration": getattr(self, "iteration", 0),
+            }
+        )
+
+        # Debug: Check if search engine is available
+        if not hasattr(self, "search") or self.search is None:
+            logger.error(f"No search engine configured for query: {search_query}")
+            logger.error(f"Strategy attributes: {list(self.__dict__.keys())}")
+            return {"current_knowledge": "", "search_results": []}
+
+        try:
+            # Log that we're attempting to use source-based strategy
+            logger.info(f"Attempting source-based search for: {search_query}")
+
+            # For complex queries, use source-based strategy with multiple iterations
+            if hasattr(self, "source_strategy"):
+                source_strategy = self.source_strategy
+            else:
+                logger.info("Creating new SourceBasedSearchStrategy instance")
+                source_strategy = SourceBasedSearchStrategy(
+                    model=self.model,
+                    search=self.search,
+                    all_links_of_system=self.all_links_of_system,
+                    include_text_content=True,
+                    use_cross_engine_filter=False,  # We'll handle filtering ourselves
+                    use_atomic_facts=False,
+                )
+                source_strategy.max_iterations = (
+                    1  # More efficient with single iteration
+                )
+                source_strategy.questions_per_iteration = (
+                    9  # More questions for broader coverage
+                )
+
+            # Use source-based strategy for complex search
+            try:
+                # Set a simple progress callback if we have one
+                if self.progress_callback:
+
+                    def sub_callback(msg, prog, data):
+                        # Don't propagate all sub-progress updates
+                        if "phase" in data and data["phase"] in [
+                            "search_complete",
+                            "final_results",
+                        ]:
+                            self.progress_callback(f"Sub-search: {msg}", None, data)
+
+                    source_strategy.set_progress_callback(sub_callback)
+
+                logger.info("Executing source-based search...")
+                # Run the search
+                result = source_strategy.analyze_topic(search_query)
+
+                if (
+                    result
+                    and "current_knowledge" in result
+                    and "all_links_of_system" in result
+                ):
+                    search_results = result.get("all_links_of_system", [])
+
+                    # Extract the most relevant information from the findings
+                    knowledge_parts = []
+                    if "findings" in result:
+                        for finding in result["findings"]:
+                            if "content" in finding and finding["content"]:
+                                knowledge_parts.append(finding["content"])
+
+                    # Also include search results summaries
+                    for i, link in enumerate(search_results[:15]):  # More results
+                        if isinstance(link, dict):
+                            title = link.get("title", "")
+                            snippet = link.get("snippet", "")
+                            content = link.get("content", "")
+                            url = link.get("link", link.get("url", ""))
+
+                            if title or snippet:
+                                result_text = f"\nResult {i+1}: {title}"
+                                if url:
+                                    result_text += f"\nURL: {url}"
+                                if snippet:
+                                    result_text += f"\nSnippet: {snippet}"
+                                if content and content != snippet:
+                                    result_text += f"\nContent: {content[:500]}..."
+                                knowledge_parts.append(result_text)
+
+                    current_knowledge = "\n\n".join(knowledge_parts)
+
+                    return {
+                        "current_knowledge": current_knowledge,
+                        "search_results": search_results,
+                        "detailed_findings": result.get("findings", []),
+                    }
+                else:
+                    # Fallback to simple search
+                    logger.warning(
+                        "Source-based search returned empty results, falling back to simple search"
+                    )
+                    return self._simple_search(search_query)
+
+            except Exception as e:
+                logger.error(f"Source-based search failed with error: {e}")
+                logger.error(f"Error type: {type(e).__name__}")
+                import traceback
+
+                logger.error(f"Traceback: {traceback.format_exc()}")
+                logger.warning("Falling back to simple search")
+                return self._simple_search(search_query)
+
+        except Exception as e:
+            logger.error(f"Error during search for '{search_query}': {str(e)}")
+            return {
+                "current_knowledge": f"Error during search: {str(e)}",
+                "search_results": [],
+            }'''
+
+    def _simple_search(self, search_query: str) -> Dict:
+        """Fallback simple search using search engine directly."""
+        try:
+            # Use the search engine directly for simple queries
+            search_results = self.search.run(search_query)
+
+            if search_results and isinstance(search_results, list):
+                # Format search results into a knowledge string
+                content_parts = []
+
+                for i, result in enumerate(search_results[:15]):  # More results
+                    title = result.get("title", "Untitled")
+                    snippet = result.get("snippet", "")
+                    content = result.get("content", "")
+                    url = result.get("link", result.get("url", ""))
+
+                    content_parts.append(f"Result {i + 1}: {title}")
+                    if url:
+                        content_parts.append(f"URL: {url}")
+                    if snippet:
+                        content_parts.append(f"Snippet: {snippet}")
+                    if content and content != snippet:
+                        content_parts.append(
+                            f"Content preview: {content[:300]}..."
+                        )
+                    content_parts.append("")  # Empty line between results
+
+                current_knowledge = "\n".join(content_parts)
+
+                return {
+                    "current_knowledge": current_knowledge,
+                    "search_results": search_results,
+                }
+            else:
+                # Return empty knowledge if no results
+                return {
+                    "current_knowledge": f"No results found for query: {search_query}",
+                    "search_results": [],
+                }
+        except Exception as e:
+            logger.error(f"Simple search error: {e}")
+            return {
+                "current_knowledge": f"Search error: {str(e)}",
+                "search_results": [],
+            }
+
+    def _validate_search_results(
+        self, results: Dict, constraint: Constraint
+    ) -> bool:
+        """Validate that search results contain relevant information."""
+        if not results:
+            return False
+
+        content = results.get("current_knowledge", "")
+        search_results = results.get("search_results", [])
+
+        # Basic validation checks
+        if not content or len(content) < 50:  # Too short to be meaningful
+            logger.debug(f"Content too short: {len(content)} characters")
+            return False
+
+        if "Error" in content and len(content) < 100:
+            logger.debug(f"Error in results: {content[:100]}")
+            return False
+
+        if "No results found" in content:
+            logger.debug("No results found")
+            return False
+
+        # For stats/numeric constraints, check for related terms
+        if constraint.type == ConstraintType.STATISTIC:
+            # Look for related terms about TV shows, episodes, etc
+            relevant_terms = [
+                "tv",
+                "show",
+                "series",
+                "episode",
+                "season",
+                "program",
+                "character",
+                "fiction",
+            ]
+            content_lower = content.lower()
+
+            term_found = any(term in content_lower for term in relevant_terms)
+            if not term_found:
+                logger.debug(
+                    "No relevant TV/show terms found for statistic constraint"
+                )
+                return False
+        else:
+            # Check for relevance to constraint using key terms
+            constraint_terms = [
+                term
+                for term in constraint.value.lower().split()
+                if len(term) > 2
+                and term
+                not in ["the", "and", "with", "for", "had", "his", "her"]
+            ]
+            content_lower = content.lower()
+
+            # Count how many meaningful terms appear
+            if constraint_terms:
+                term_matches = sum(
+                    1 for term in constraint_terms if term in content_lower
+                )
+                relevance_ratio = term_matches / len(constraint_terms)
+
+                # Require at least one term match
+                if relevance_ratio < 0.2:
+                    logger.debug(
+                        f"Low relevance: {relevance_ratio:.0%} term matches"
+                    )
+                    return False
+
+        # Check search results quality
+        if search_results and isinstance(search_results, list):
+            valid_results = sum(
+                1
+                for r in search_results
+                if isinstance(r, dict) and (r.get("title") or r.get("snippet"))
+            )
+            if valid_results < 1:
+                logger.debug("No valid search results with title/snippet")
+                return False
+
+        return True
+
+    def _get_timestamp(self) -> str:
+        """Get current timestamp."""
+        return datetime.utcnow().isoformat()
+
+    def _group_similar_candidates(
+        self, candidates: List[Candidate]
+    ) -> Dict[str, List[Candidate]]:
+        """Group candidates by similar characteristics."""
+        grouped = {}
+
+        for candidate in candidates:
+            # Try to determine group type based on name patterns
+            name = candidate.name.lower()
+
+            if any(
+                keyword in name
+                for keyword in ["model", "llm", "gpt", "claude", "gemini"]
+            ):
+                group = "AI Models"
+            elif any(
+                keyword in name
+                for keyword in ["country", "nation", "republic", "kingdom"]
+            ):
+                group = "Countries"
+            elif any(
+                keyword in name for keyword in ["city", "town", "village"]
+            ):
+                group = "Cities"
+            elif any(
+                keyword in name for keyword in ["year", "century", "decade"]
+            ):
+                group = "Time Periods"
+            elif any(
+                keyword in name
+                for keyword in ["person", "mr", "ms", "dr", "prof"]
+            ):
+                group = "People"
+            elif any(c.isdigit() for c in name):
+                group = "Numeric Items"
+            else:
+                # Default grouping based on first word
+                first_word = (
+                    candidate.name.split()[0]
+                    if candidate.name.split()
+                    else "Other"
+                )
+                group = f"{first_word} Items"
+
+            if group not in grouped:
+                grouped[group] = []
+            grouped[group].append(candidate)
+
+        return grouped