local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/advanced_search_system/strategies/dual_confidence_with_rejection.py

@@ -0,0 +1,219 @@
+"""
+Enhanced dual confidence strategy with early rejection of candidates.
+"""
+
+from loguru import logger
+
+from ..constraint_checking import DualConfidenceChecker
+from .dual_confidence_strategy import DualConfidenceStrategy
+
+
+class DualConfidenceWithRejectionStrategy(DualConfidenceStrategy):
+    """
+    Enhanced dual confidence strategy that rejects candidates early when they have
+    high negative evidence for any constraint.
+    """
+
+    def __init__(
+        self,
+        *args,
+        rejection_threshold: float = 0.3,  # If negative > 30% and positive < threshold
+        positive_threshold: float = 0.2,  # Minimum positive needed to overcome negative
+        critical_constraint_rejection: float = 0.2,  # Even stricter for critical constraints
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.rejection_threshold = rejection_threshold
+        self.positive_threshold = positive_threshold
+        self.critical_constraint_rejection = critical_constraint_rejection
+
+        # Initialize constraint checker (using new inheritance-based system)
+        self.constraint_checker = DualConfidenceChecker(
+            model=self.model,
+            evidence_gatherer=self._gather_evidence_for_constraint,
+            negative_threshold=0.25,  # 25% negative evidence threshold
+            positive_threshold=0.4,  # 40% minimum positive evidence
+            uncertainty_penalty=self.uncertainty_penalty,
+            negative_weight=self.negative_weight,
+        )
+
+    def _evaluate_candidate_immediately(self, candidate) -> float:
+        """Enhanced evaluation with early rejection based on negative evidence."""
+        try:
+            logger.info(
+                f"Evaluating candidate: {candidate.name} with early rejection"
+            )
+
+            total_score = 0.0
+            constraint_scores = []
+            detailed_results = []
+
+            for i, constraint in enumerate(self.constraint_ranking):
+                # Gather evidence for this constraint
+                evidence = self._gather_evidence_for_constraint(
+                    candidate, constraint
+                )
+
+                if evidence:
+                    # Analyze evidence with dual confidence
+                    dual_evidence = [
+                        self._analyze_evidence_dual_confidence(e, constraint)
+                        for e in evidence
+                    ]
+
+                    # Calculate average scores
+                    avg_positive = sum(
+                        e.positive_confidence for e in dual_evidence
+                    ) / len(dual_evidence)
+                    avg_negative = sum(
+                        e.negative_confidence for e in dual_evidence
+                    ) / len(dual_evidence)
+                    avg_uncertainty = sum(
+                        e.uncertainty for e in dual_evidence
+                    ) / len(dual_evidence)
+
+                    # EARLY REJECTION LOGIC
+                    # Reject if negative evidence is above 25% - simplified approach
+                    if avg_negative > 0.25:
+                        logger.info(
+                            f"❌ EARLY REJECTION: {candidate.name} - Constraint '{constraint.value}' "
+                            f"has significant negative evidence ({avg_negative:.0%})"
+                        )
+                        return 0.0  # Immediate rejection
+
+                    # If high negative but also decent positive, continue but penalize
+                    if (
+                        avg_negative > self.rejection_threshold
+                        and avg_positive > self.positive_threshold
+                    ):
+                        logger.warning(
+                            f"⚠️ Mixed evidence for {candidate.name} - {constraint.value}: "
+                            f"+{avg_positive:.0%} -{avg_negative:.0%}"
+                        )
+
+                    # Calculate score using parent method
+                    score = self._evaluate_evidence(evidence, constraint)
+                    constraint_scores.append(score)
+
+                    detailed_results.append(
+                        {
+                            "constraint": constraint.value,
+                            "score": score,
+                            "positive": avg_positive,
+                            "negative": avg_negative,
+                            "uncertainty": avg_uncertainty,
+                            "weight": constraint.weight,
+                        }
+                    )
+
+                    # Visual feedback
+                    symbol = (
+                        "✓" if score >= 0.8 else "○" if score >= 0.5 else "✗"
+                    )
+                    logger.info(
+                        f"{symbol} {candidate.name} | {constraint.value}: {int(score * 100)}% "
+                        f"(+{int(avg_positive * 100)}% -{int(avg_negative * 100)}% ?{int(avg_uncertainty * 100)}%)"
+                    )
+
+                    # Skip remaining constraints if this one failed badly
+                    if score < 0.2 and constraint.weight > 0.5:
+                        logger.info(
+                            "⚠️ Skipping remaining constraints due to poor score on important constraint"
+                        )
+                        break
+                else:
+                    # No evidence - high uncertainty
+                    score = 0.5 - self.uncertainty_penalty
+                    constraint_scores.append(score)
+                    logger.info(
+                        f"? {candidate.name} | {constraint.value}: No evidence found"
+                    )
+
+            # Calculate weighted average
+            if constraint_scores:
+                weights = [
+                    c.weight
+                    for c in self.constraint_ranking[: len(constraint_scores)]
+                ]
+                total_score = sum(
+                    s * w for s, w in zip(constraint_scores, weights)
+                ) / sum(weights)
+
+            # Log detailed breakdown
+            logger.info(f"\nDetailed analysis for {candidate.name}:")
+            for result in detailed_results:
+                logger.info(
+                    f"  {result['constraint']}: {result['score']:.2%} "
+                    f"(+{result['positive']:.0%} -{result['negative']:.0%} ?{result['uncertainty']:.0%}) "
+                    f"[weight: {result['weight']:.1f}]"
+                )
+
+            logger.info(f"Final score for {candidate.name}: {total_score:.2%}")
+
+            # Store constraint evaluation results on the candidate object
+            candidate.evaluation_results = detailed_results
+            candidate.score = total_score
+
+            # Update tracking
+            with self.evaluation_lock:
+                self.evaluated_candidates[candidate.name] = total_score
+
+                if total_score > self.best_score:
+                    self.best_score = total_score
+                    self.best_candidate = candidate
+                    logger.info(
+                        f"New best: {candidate.name} with {total_score:.2%}"
+                    )
+
+                    # Check for early stop
+                    if total_score >= self.early_stop_threshold:
+                        logger.info(
+                            f"🎯 EARLY STOP: {candidate.name} reached {total_score:.2%}!"
+                        )
+                        self.found_answer.set()
+
+            return total_score
+
+        except Exception as e:
+            logger.error(f"Error evaluating {candidate.name}: {e}")
+            return 0.0
+
+    def _evaluate_candidate_with_constraint_checker(self, candidate) -> float:
+        """
+        Evaluate candidate using the new modular constraint checking system.
+
+        This method can be used as an alternative to the existing evaluation logic.
+        """
+        try:
+            # Use the constraint checker
+            result = self.constraint_checker.check_candidate(
+                candidate, self.constraint_ranking
+            )
+
+            # Store results on candidate
+            candidate.evaluation_results = result.detailed_results
+            candidate.score = result.total_score
+
+            # Update tracking
+            with self.evaluation_lock:
+                self.evaluated_candidates[candidate.name] = result.total_score
+
+                if result.total_score > self.best_score:
+                    self.best_score = result.total_score
+                    self.best_candidate = candidate
+                    logger.info(
+                        f"New best: {candidate.name} with {result.total_score:.2%}"
+                    )
+
+                    # Check for early stop
+                    if result.total_score >= self.early_stop_threshold:
+                        logger.info(
+                            f"🎯 EARLY STOP: {candidate.name} reached {result.total_score:.2%}!"
+                        )
+                        self.found_answer.set()
+
+            return result.total_score
+
+        except Exception as e:
+            logger.error(f"Error evaluating {candidate.name}: {e}")
+            return 0.0

local_deep_research/advanced_search_system/strategies/early_stop_constrained_strategy.py

@@ -0,0 +1,369 @@
+"""
+Early-stop constrained search strategy that evaluates candidates immediately
+and stops when finding a very high confidence match.
+"""
+
+import concurrent.futures
+import threading
+from typing import Dict, List
+
+from loguru import logger
+
+from ..candidates.base_candidate import Candidate
+from ..constraints.base_constraint import Constraint
+from .parallel_constrained_strategy import ParallelConstrainedStrategy
+
+
+class EarlyStopConstrainedStrategy(ParallelConstrainedStrategy):
+    """
+    Enhanced constrained strategy that:
+    1. Evaluates candidates as soon as they're found
+    2. Stops early when finding a very high confidence match (99%+)
+    3. Runs evaluation and search concurrently
+    """
+
+    def __init__(
+        self,
+        *args,
+        early_stop_threshold: float = 0.99,
+        concurrent_evaluation: bool = True,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.early_stop_threshold = early_stop_threshold
+        self.concurrent_evaluation = concurrent_evaluation
+
+        # Thread-safe tracking
+        self.found_answer = threading.Event()
+        self.best_candidate = None
+        self.best_score = 0.0
+        self.evaluation_lock = threading.Lock()
+
+        # Track candidates being evaluated
+        self.evaluating_candidates = set()
+        self.evaluated_candidates = {}
+
+    def _parallel_search(self, combinations: List) -> List[Candidate]:
+        """Execute searches in parallel with immediate candidate evaluation."""
+        all_candidates = []
+        evaluation_futures = []
+
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=self.parallel_workers
+        ) as executor:
+            # Submit all searches
+            search_futures = {
+                executor.submit(self._execute_combination_search, combo): combo
+                for combo in combinations
+            }
+
+            # Process results as they complete
+            for future in concurrent.futures.as_completed(search_futures):
+                # Check if we should stop early
+                if self.found_answer.is_set():
+                    logger.info(
+                        f"Early stop triggered - found answer: {self.best_candidate}"
+                    )
+                    break
+
+                combo = search_futures[future]
+                try:
+                    candidates = future.result()
+                    all_candidates.extend(candidates)
+
+                    # Start evaluating candidates immediately if concurrent evaluation is enabled
+                    if self.concurrent_evaluation:
+                        for candidate in candidates:
+                            if candidate.name not in self.evaluating_candidates:
+                                self.evaluating_candidates.add(candidate.name)
+                                eval_future = executor.submit(
+                                    self._evaluate_candidate_immediately,
+                                    candidate,
+                                )
+                                evaluation_futures.append(eval_future)
+
+                    if self.progress_callback:
+                        self.progress_callback(
+                            f"Found {len(candidates)} candidates, evaluating...",
+                            None,
+                            {
+                                "phase": "parallel_search_with_eval",
+                                "candidates": len(all_candidates),
+                                "best_score": self.best_score,
+                                "best_candidate": self.best_candidate,
+                            },
+                        )
+
+                except Exception as e:
+                    logger.error(f"Search failed for {combo.query}: {e}")
+
+            # Wait for evaluation futures to complete
+            for future in concurrent.futures.as_completed(evaluation_futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    logger.error(f"Evaluation failed: {e}")
+
+        return all_candidates
+
+    def _evaluate_candidate_immediately(self, candidate: Candidate) -> float:
+        """Evaluate a candidate against all constraints immediately."""
+        try:
+            logger.info(f"Immediately evaluating candidate: {candidate.name}")
+
+            # Calculate overall score across all constraints
+            total_score = 0.0
+            constraint_scores = []
+
+            for constraint in self.constraint_ranking:
+                # Get evidence for this constraint
+                evidence = self._gather_evidence_for_constraint(
+                    candidate, constraint
+                )
+                score = self._evaluate_evidence(evidence, constraint)
+                constraint_scores.append(score)
+
+                # Update progress
+                if self.progress_callback:
+                    symbol = "✓" if score >= 0.8 else "○"
+                    self.progress_callback(
+                        f"{symbol} {candidate.name} | {constraint.type.value}: {int(score * 100)}%",
+                        None,
+                        {
+                            "phase": "immediate_evaluation",
+                            "candidate": candidate.name,
+                            "constraint": constraint.value,
+                            "score": score,
+                        },
+                    )
+
+                # If this candidate fails a critical constraint badly, skip remaining checks
+                if score < 0.3 and constraint.weight > 0.8:
+                    logger.info(
+                        f"Candidate {candidate.name} failed critical constraint early"
+                    )
+                    break
+
+            # Calculate average score
+            if constraint_scores:
+                total_score = sum(constraint_scores) / len(constraint_scores)
+
+            # Thread-safe update of best candidate
+            with self.evaluation_lock:
+                self.evaluated_candidates[candidate.name] = total_score
+
+                if total_score > self.best_score:
+                    self.best_score = total_score
+                    self.best_candidate = candidate.name
+
+                    logger.info(
+                        f"New best candidate: {candidate.name} with score {total_score:.2f}"
+                    )
+
+                    # Check for early stop
+                    if total_score >= self.early_stop_threshold:
+                        logger.info(
+                            f"EARLY STOP: Found {candidate.name} with {total_score:.2f} confidence!"
+                        )
+                        self.found_answer.set()
+
+                        if self.progress_callback:
+                            self.progress_callback(
+                                f"Found answer: {candidate.name} ({int(total_score * 100)}% confidence)",
+                                95,
+                                {
+                                    "phase": "early_stop",
+                                    "final_answer": candidate.name,
+                                    "confidence": total_score,
+                                },
+                            )
+
+            return total_score
+
+        except Exception as e:
+            logger.error(f"Error evaluating candidate {candidate.name}: {e}")
+            return 0.0
+
+    def _progressive_constraint_search(self):
+        """Override to implement early stopping."""
+        current_candidates = []
+        search_iterations = 0
+        max_search_iterations = 3
+
+        # Detect entity type
+        self.entity_type = self._detect_entity_type()
+        logger.info(f"Detected entity type: {self.entity_type}")
+
+        while (
+            search_iterations < max_search_iterations
+            and not self.found_answer.is_set()
+        ):
+            search_iterations += 1
+
+            # Create search combinations based on iteration
+            if search_iterations == 1:
+                combinations = self._create_strict_combinations()
+                strictness = "strict"
+            elif search_iterations == 2:
+                combinations = self._create_relaxed_combinations()
+                strictness = "relaxed"
+            else:
+                combinations = self._create_individual_combinations()
+                strictness = "individual"
+
+            logger.info(
+                f"Iteration {search_iterations}: {strictness} mode with {len(combinations)} combinations"
+            )
+
+            # Run searches in parallel with immediate evaluation
+            new_candidates = self._parallel_search(combinations)
+            current_candidates.extend(new_candidates)
+
+            # Check if we have enough results or found the answer
+            unique_candidates = self._deduplicate_candidates(current_candidates)
+
+            if self.found_answer.is_set():
+                logger.info(f"Early stop - found answer: {self.best_candidate}")
+                break
+
+            if len(unique_candidates) >= self.min_results_threshold:
+                logger.info(
+                    f"Found {len(unique_candidates)} candidates - checking if we need more"
+                )
+                # Continue only if best score is below threshold
+                if self.best_score >= 0.9:
+                    logger.info(
+                        f"Best score {self.best_score:.2f} is high enough - stopping search"
+                    )
+                    break
+
+        # Set final candidates
+        self.candidates = [
+            c for c in unique_candidates if c.name == self.best_candidate
+        ]
+        if not self.candidates and unique_candidates:
+            # If best candidate wasn't in the list somehow, use top scored candidates
+            scored_candidates = sorted(
+                unique_candidates,
+                key=lambda c: self.evaluated_candidates.get(c.name, 0),
+                reverse=True,
+            )
+            self.candidates = scored_candidates[: self.candidate_limit]
+
+        self.final_answer = self.best_candidate
+        self.confidence = self.best_score
+
+    def analyze_topic(self, topic: str) -> Dict:
+        """Analyze topic with early stopping."""
+        # Call parent's analyze_topic to handle constraint extraction
+        result = super().analyze_topic(topic)
+
+        # Add our early stopping information
+        result["early_stopped"] = self.found_answer.is_set()
+        result["evaluated_candidates"] = self.evaluated_candidates
+        result["best_candidate"] = self.best_candidate
+        result["best_score"] = self.best_score
+
+        return result
+
+    def _gather_evidence_for_constraint(
+        self, candidate: Candidate, constraint: Constraint
+    ) -> List:
+        """Gather evidence for a specific candidate-constraint pair."""
+        # Run targeted search for this specific combination
+        query = f'"{candidate.name}" {constraint.value} verification'
+
+        try:
+            results = self._execute_search(query)
+            evidence = self._extract_evidence_from_results(
+                results, candidate, constraint
+            )
+            return evidence
+        except Exception as e:
+            logger.error(f"Error gathering evidence for {candidate.name}: {e}")
+            return []
+
+    def _extract_evidence_from_results(
+        self, results: Dict, candidate: Candidate, constraint: Constraint
+    ) -> List:
+        """Extract relevant evidence from search results."""
+        evidence = []
+        content = results.get("current_knowledge", "")
+
+        if content:
+            # Use LLM to extract evidence
+            prompt = f"""
+            Extract evidence regarding whether "{candidate.name}" satisfies this constraint:
+
+            Constraint: {constraint.value}
+            Constraint Type: {constraint.type.value}
+
+            Search Results:
+            {content[:3000]}
+
+            Extract specific evidence that either supports or refutes the constraint.
+            Return a confidence score from 0 to 1.
+            """
+
+            try:
+                response = self.model.invoke(prompt).content
+                evidence.append(
+                    {
+                        "text": response,
+                        "source": "search_results",
+                        "confidence": self._extract_confidence_from_response(
+                            response
+                        ),
+                    }
+                )
+            except Exception as e:
+                logger.error(f"Error extracting evidence: {e}")
+
+        return evidence
+
+    def _extract_confidence_from_response(self, response: str) -> float:
+        """Extract confidence score from LLM response."""
+        # Simple extraction - look for number between 0 and 1
+        import re
+
+        pattern = r"\b0?\.\d+\b|\b1\.0\b|\b1\b"
+        matches = re.findall(pattern, response)
+
+        if matches:
+            try:
+                return float(matches[-1])  # Use last number found
+            except:
+                pass
+
+        # Default confidence based on keywords
+        if any(
+            word in response.lower()
+            for word in ["definitely", "certainly", "absolutely"]
+        ):
+            return 0.9
+        elif any(
+            word in response.lower()
+            for word in ["likely", "probably", "appears"]
+        ):
+            return 0.7
+        elif any(
+            word in response.lower() for word in ["possibly", "maybe", "might"]
+        ):
+            return 0.5
+        elif any(
+            word in response.lower() for word in ["unlikely", "doubtful", "not"]
+        ):
+            return 0.3
+
+        return 0.5
+
+    def _evaluate_evidence(
+        self, evidence: List, constraint: Constraint
+    ) -> float:
+        """Evaluate evidence to determine constraint satisfaction score."""
+        if not evidence:
+            return 0.0
+
+        # Average confidence across all evidence
+        confidences = [e.get("confidence", 0.5) for e in evidence]
+        return sum(confidences) / len(confidences) if confidences else 0.0