local-deep-research 0.4.4__py3-none-any.whl → 0.5.2__py3-none-any.whl

local_deep_research/advanced_search_system/constraint_checking/dual_confidence_checker.py

@@ -0,0 +1,424 @@
+"""
+Dual confidence constraint checker implementation.
+
+This implementation uses dual confidence scoring (positive/negative/uncertainty)
+to evaluate constraints and make rejection decisions.
+"""
+
+from typing import Dict, List, Tuple
+
+from loguru import logger
+
+from ..candidates.base_candidate import Candidate
+from ..constraints.base_constraint import Constraint
+from .base_constraint_checker import (
+    BaseConstraintChecker,
+    ConstraintCheckResult,
+)
+from .evidence_analyzer import ConstraintEvidence, EvidenceAnalyzer
+
+
+class DualConfidenceChecker(BaseConstraintChecker):
+    """
+    Constraint checker using dual confidence scoring.
+
+    This checker:
+    1. Analyzes evidence using positive/negative/uncertainty scores
+    2. Makes rejection decisions based on confidence thresholds
+    3. Provides detailed scoring breakdown
+    """
+
+    def __init__(
+        self,
+        *args,
+        negative_threshold: float = 0.25,  # Reject if negative evidence > 25%
+        positive_threshold: float = 0.4,  # Reject if positive evidence < 40%
+        uncertainty_penalty: float = 0.2,
+        negative_weight: float = 0.5,
+        uncertainty_threshold: float = 0.6,  # Re-evaluate if uncertainty > 60%
+        max_reevaluations: int = 2,  # Maximum re-evaluation rounds
+        **kwargs,
+    ):
+        """
+        Initialize dual confidence checker.
+
+        Args:
+            negative_threshold: Threshold for negative evidence rejection
+            positive_threshold: Minimum positive evidence required
+            uncertainty_penalty: Penalty for uncertain evidence
+            negative_weight: Weight for negative evidence in scoring
+            uncertainty_threshold: Re-evaluate if uncertainty exceeds this
+            max_reevaluations: Maximum number of re-evaluation rounds
+        """
+        super().__init__(*args, **kwargs)
+
+        self.negative_threshold = negative_threshold
+        self.positive_threshold = positive_threshold
+        self.uncertainty_penalty = uncertainty_penalty
+        self.negative_weight = negative_weight
+        self.uncertainty_threshold = uncertainty_threshold
+        self.max_reevaluations = max_reevaluations
+
+        # Initialize evidence analyzer
+        self.evidence_analyzer = EvidenceAnalyzer(self.model)
+
+    def check_candidate(
+        self,
+        candidate: Candidate,
+        constraints: List[Constraint],
+        original_query: str = None,
+    ) -> ConstraintCheckResult:
+        """Check candidate using dual confidence analysis with LLM pre-screening."""
+        logger.info(f"Checking candidate: {candidate.name} (dual confidence)")
+
+        # LLM PRE-SCREENING: Check all constraints in one call to save SearXNG capacity
+        pre_screen_result = self._llm_prescreen_candidate(
+            candidate, constraints, original_query
+        )
+        if pre_screen_result["should_reject"]:
+            logger.info(
+                f"🚫 LLM pre-screen rejected {candidate.name}: {pre_screen_result['reason']}"
+            )
+            return ConstraintCheckResult(
+                should_reject=True,
+                rejection_reason=pre_screen_result["reason"],
+                total_score=0.0,
+                detailed_results=pre_screen_result["detailed_results"],
+            )
+
+        constraint_scores = {}
+        detailed_results = []
+        rejection_reason = None
+        should_reject = False
+
+        for constraint in constraints:
+            # Perform initial evaluation with re-evaluation for uncertain constraints
+            result = self._evaluate_constraint_with_reevaluation(
+                candidate, constraint
+            )
+
+            avg_positive = result["positive"]
+            avg_negative = result["negative"]
+            avg_uncertainty = result["uncertainty"]
+            score = result["score"]
+            reevaluation_count = result.get("reevaluation_count", 0)
+
+            # Check for rejection based on final results
+            reject, reason = self.should_reject_candidate_from_averages(
+                candidate, constraint, avg_positive, avg_negative
+            )
+
+            if reject and not should_reject:  # Only record first rejection
+                should_reject = True
+                rejection_reason = reason
+
+            # Store results
+            constraint_scores[constraint.value] = {
+                "total": score,
+                "positive": avg_positive,
+                "negative": avg_negative,
+                "uncertainty": avg_uncertainty,
+                "weight": constraint.weight,
+                "reevaluation_count": reevaluation_count,
+            }
+
+            detailed_results.append(
+                {
+                    "constraint": constraint.value,
+                    "score": score,
+                    "positive": avg_positive,
+                    "negative": avg_negative,
+                    "uncertainty": avg_uncertainty,
+                    "weight": constraint.weight,
+                    "type": constraint.type.value,
+                    "reevaluation_count": reevaluation_count,
+                }
+            )
+
+            # Log detailed result with re-evaluation info
+            self._log_constraint_result_detailed(
+                candidate,
+                constraint,
+                score,
+                avg_positive,
+                avg_negative,
+                avg_uncertainty,
+                reevaluation_count,
+            )
+
+        # Calculate total score
+        if should_reject:
+            total_score = 0.0
+        else:
+            if detailed_results:
+                weights = [r["weight"] for r in detailed_results]
+                scores = [r["score"] for r in detailed_results]
+                total_score = self._calculate_weighted_score(scores, weights)
+            else:
+                total_score = 0.0
+
+        logger.info(f"Final score for {candidate.name}: {total_score:.2%}")
+
+        return ConstraintCheckResult(
+            candidate=candidate,
+            total_score=total_score,
+            constraint_scores=constraint_scores,
+            should_reject=should_reject,
+            rejection_reason=rejection_reason,
+            detailed_results=detailed_results,
+        )
+
+    def _evaluate_constraint_with_reevaluation(
+        self, candidate: Candidate, constraint: Constraint
+    ) -> Dict:
+        """Evaluate constraint with potential re-evaluation for uncertain results."""
+        reevaluation_count = 0
+        evidence_list = []
+
+        while reevaluation_count <= self.max_reevaluations:
+            # Gather evidence (fresh each time for re-evaluation)
+            evidence_list = self._gather_evidence_for_constraint(
+                candidate, constraint
+            )
+
+            if not evidence_list:
+                # No evidence found
+                return {
+                    "positive": 0.0,
+                    "negative": 0.0,
+                    "uncertainty": 1.0,
+                    "score": 0.5 - self.uncertainty_penalty,
+                    "evidence_list": [],
+                    "reevaluation_count": reevaluation_count,
+                }
+
+            # Analyze with dual confidence
+            dual_evidence = [
+                self.evidence_analyzer.analyze_evidence_dual_confidence(
+                    e, constraint
+                )
+                for e in evidence_list
+            ]
+
+            # Calculate averages
+            avg_positive = sum(
+                e.positive_confidence for e in dual_evidence
+            ) / len(dual_evidence)
+            avg_negative = sum(
+                e.negative_confidence for e in dual_evidence
+            ) / len(dual_evidence)
+            avg_uncertainty = sum(e.uncertainty for e in dual_evidence) / len(
+                dual_evidence
+            )
+
+            # Calculate score
+            score = self.evidence_analyzer.evaluate_evidence_list(
+                evidence_list,
+                constraint,
+                self.uncertainty_penalty,
+                self.negative_weight,
+            )
+
+            # Check if we need re-evaluation
+            if (
+                reevaluation_count < self.max_reevaluations
+                and avg_uncertainty > self.uncertainty_threshold
+                and not self._should_early_reject(avg_positive, avg_negative)
+            ):
+                reevaluation_count += 1
+                logger.info(
+                    f"🔄 Re-evaluating {candidate.name} | {constraint.value} "
+                    f"(round {reevaluation_count}) - high uncertainty: {avg_uncertainty:.0%}"
+                )
+                continue
+            else:
+                # Final result or early rejection
+                if reevaluation_count > 0:
+                    logger.info(
+                        f"✅ Final evaluation for {candidate.name} | {constraint.value} "
+                        f"after {reevaluation_count} re-evaluation(s)"
+                    )
+
+                return {
+                    "positive": avg_positive,
+                    "negative": avg_negative,
+                    "uncertainty": avg_uncertainty,
+                    "score": score,
+                    "evidence_list": evidence_list,
+                    "reevaluation_count": reevaluation_count,
+                }
+
+        # Should not reach here, but fallback
+        return {
+            "positive": avg_positive,
+            "negative": avg_negative,
+            "uncertainty": avg_uncertainty,
+            "score": score,
+            "evidence_list": evidence_list,
+            "reevaluation_count": reevaluation_count,
+        }
+
+    def _should_early_reject(
+        self, avg_positive: float, avg_negative: float
+    ) -> bool:
+        """Check if candidate should be rejected early (before re-evaluation)."""
+        return (
+            avg_negative > self.negative_threshold
+            or avg_positive < self.positive_threshold
+        )
+
+    def should_reject_candidate_from_averages(
+        self,
+        candidate: Candidate,
+        constraint: Constraint,
+        avg_positive: float,
+        avg_negative: float,
+    ) -> Tuple[bool, str]:
+        """Determine rejection based on average confidence scores."""
+        # PRIMARY REJECTION: High negative evidence
+        if avg_negative > self.negative_threshold:
+            reason = f"High negative evidence ({avg_negative:.0%}) for constraint '{constraint.value}'"
+            logger.info(f"❌ REJECTION: {candidate.name} - {reason}")
+            return True, reason
+
+        # SECONDARY REJECTION: Low positive evidence
+        if avg_positive < self.positive_threshold:
+            reason = f"Insufficient positive evidence ({avg_positive:.0%}) for constraint '{constraint.value}'"
+            logger.info(f"❌ REJECTION: {candidate.name} - {reason}")
+            return True, reason
+
+        return False, ""
+
+    def should_reject_candidate(
+        self,
+        candidate: Candidate,
+        constraint: Constraint,
+        dual_evidence: List[ConstraintEvidence],
+    ) -> Tuple[bool, str]:
+        """Determine rejection based on dual confidence scores."""
+        if not dual_evidence:
+            return False, ""
+
+        # Calculate averages
+        avg_positive = sum(e.positive_confidence for e in dual_evidence) / len(
+            dual_evidence
+        )
+        avg_negative = sum(e.negative_confidence for e in dual_evidence) / len(
+            dual_evidence
+        )
+
+        # PRIMARY REJECTION: High negative evidence
+        if avg_negative > self.negative_threshold:
+            reason = f"High negative evidence ({avg_negative:.0%}) for constraint '{constraint.value}'"
+            logger.info(f"❌ REJECTION: {candidate.name} - {reason}")
+            return True, reason
+
+        # SECONDARY REJECTION: Low positive evidence
+        if avg_positive < self.positive_threshold:
+            reason = f"Insufficient positive evidence ({avg_positive:.0%}) for constraint '{constraint.value}'"
+            logger.info(f"❌ REJECTION: {candidate.name} - {reason}")
+            return True, reason
+
+        return False, ""
+
+    def _log_constraint_result_detailed(
+        self,
+        candidate,
+        constraint,
+        score,
+        positive,
+        negative,
+        uncertainty,
+        reevaluation_count=0,
+    ):
+        """Log detailed constraint result."""
+        symbol = "✓" if score >= 0.8 else "○" if score >= 0.5 else "✗"
+
+        # Add re-evaluation indicator
+        reeval_indicator = (
+            f" [R{reevaluation_count}]" if reevaluation_count > 0 else ""
+        )
+
+        logger.info(
+            f"{symbol} {candidate.name} | {constraint.value}: {int(score * 100)}% "
+            f"(+{int(positive * 100)}% -{int(negative * 100)}% ?{int(uncertainty * 100)}%){reeval_indicator}"
+        )
+
+    def _llm_prescreen_candidate(
+        self, candidate, constraints, original_query=None
+    ):
+        """Simple quality check for answer candidates."""
+
+        if not original_query:
+            return {
+                "should_reject": False,
+                "reason": "No original query provided",
+                "detailed_results": [],
+            }
+
+        prompt = f"""Question: {original_query}
+Answer: {candidate.name}
+
+Is this a good answer to the question? Rate 0-100 where:
+- 90-100: Excellent direct answer
+- 70-89: Good answer
+- 50-69: Partial answer
+- 30-49: Weak answer
+- 0-29: Poor/wrong answer
+
+Just give the number:"""
+
+        try:
+            response = self.model.generate(prompt)
+
+            # Parse confidence score
+            import re
+
+            confidence_match = re.search(r"(\d{1,3})", response.strip())
+
+            if confidence_match:
+                quality_score = int(confidence_match.group(1))
+
+                # Accept good answers (50+ out of 100)
+                if quality_score >= 50:
+                    return {
+                        "should_reject": False,
+                        "reason": f"Good answer quality: {quality_score}%",
+                        "detailed_results": [
+                            {
+                                "constraint": "answer_quality",
+                                "positive_confidence": quality_score / 100.0,
+                                "source": "answer_quality_check",
+                            }
+                        ],
+                    }
+                else:
+                    return {
+                        "should_reject": True,
+                        "reason": f"Poor answer quality: {quality_score}%",
+                        "detailed_results": [
+                            {
+                                "constraint": "answer_quality",
+                                "negative_confidence": (100 - quality_score)
+                                / 100.0,
+                                "source": "answer_quality_check",
+                            }
+                        ],
+                    }
+
+            # Parsing failed - accept by default
+            return {
+                "should_reject": False,
+                "reason": "Could not parse quality score - accepting",
+                "detailed_results": [],
+            }
+
+        except Exception as e:
+            logger.warning(
+                f"Fast LLM pre-screening failed for {candidate.name}: {e}"
+            )
+            return {
+                "should_reject": False,
+                "reason": "",
+                "detailed_results": [],
+            }

local_deep_research/advanced_search_system/constraint_checking/evidence_analyzer.py

@@ -0,0 +1,174 @@
+"""
+Evidence analysis for constraint checking.
+
+This module provides dual confidence evidence analysis that separates
+positive evidence, negative evidence, and uncertainty.
+"""
+
+import re
+from dataclasses import dataclass
+from typing import Dict, List
+
+from langchain_core.language_models import BaseChatModel
+from loguru import logger
+
+from ..constraints.base_constraint import Constraint
+
+
+@dataclass
+class ConstraintEvidence:
+    """Evidence for a constraint with dual confidence scores."""
+
+    positive_confidence: float  # How sure we are the constraint IS satisfied
+    negative_confidence: (
+        float  # How sure we are the constraint is NOT satisfied
+    )
+    uncertainty: float  # How uncertain we are (neither positive nor negative)
+    evidence_text: str
+    source: str
+
+
+class EvidenceAnalyzer:
+    """
+    Analyzes evidence using dual confidence scoring.
+
+    This approach separates:
+    - Positive confidence: Evidence that constraint IS satisfied
+    - Negative confidence: Evidence that constraint is NOT satisfied
+    - Uncertainty: Lack of clear evidence either way
+    """
+
+    def __init__(self, model: BaseChatModel):
+        """Initialize the evidence analyzer."""
+        self.model = model
+
+    def analyze_evidence_dual_confidence(
+        self, evidence: Dict, constraint: Constraint
+    ) -> ConstraintEvidence:
+        """Analyze evidence to extract dual confidence scores."""
+        text = evidence.get("text", "")
+
+        # Use LLM to analyze evidence with dual confidence
+        prompt = f"""
+Analyze this evidence for the constraint "{constraint.value}" (type: {constraint.type.value}).
+
+Evidence:
+{text[:1000]}
+
+Provide three confidence scores (0-1):
+1. POSITIVE_CONFIDENCE: How confident are you that this constraint IS satisfied?
+2. NEGATIVE_CONFIDENCE: How confident are you that this constraint is NOT satisfied?
+3. UNCERTAINTY: How uncertain are you (lack of clear evidence)?
+
+The three scores should approximately sum to 1.0.
+
+Format:
+POSITIVE: [score]
+NEGATIVE: [score]
+UNCERTAINTY: [score]
+"""
+
+        try:
+            response = self.model.invoke(prompt).content
+
+            # Extract scores
+            positive = self._extract_score(response, "POSITIVE")
+            negative = self._extract_score(response, "NEGATIVE")
+            uncertainty = self._extract_score(response, "UNCERTAINTY")
+
+            # Normalize if needed
+            total = positive + negative + uncertainty
+            if total > 0:
+                positive /= total
+                negative /= total
+                uncertainty /= total
+            else:
+                # Default to high uncertainty
+                uncertainty = 0.8
+                positive = 0.1
+                negative = 0.1
+
+            return ConstraintEvidence(
+                positive_confidence=positive,
+                negative_confidence=negative,
+                uncertainty=uncertainty,
+                evidence_text=text[:500],
+                source=evidence.get("source", "search"),
+            )
+
+        except Exception as e:
+            logger.error(f"Error analyzing evidence: {e}")
+            # Default to high uncertainty
+            return ConstraintEvidence(
+                positive_confidence=0.1,
+                negative_confidence=0.1,
+                uncertainty=0.8,
+                evidence_text=text[:500],
+                source=evidence.get("source", "search"),
+            )
+
+    def _extract_score(self, text: str, label: str) -> float:
+        """Extract confidence score from LLM response."""
+        pattern = rf"{label}:\s*\[?(\d*\.?\d+)\]?"
+        match = re.search(pattern, text, re.IGNORECASE)
+        if match:
+            try:
+                return float(match.group(1))
+            except:
+                pass
+        return 0.1  # Default low score
+
+    def evaluate_evidence_list(
+        self,
+        evidence_list: List[Dict],
+        constraint: Constraint,
+        uncertainty_penalty: float = 0.2,
+        negative_weight: float = 0.5,
+    ) -> float:
+        """
+        Evaluate a list of evidence using dual confidence scoring.
+
+        Args:
+            evidence_list: List of evidence dictionaries
+            constraint: The constraint being evaluated
+            uncertainty_penalty: Penalty for uncertainty
+            negative_weight: Weight for negative evidence
+
+        Returns:
+            float: Overall score between 0.0 and 1.0
+        """
+        if not evidence_list:
+            # No evidence means high uncertainty
+            return 0.5 - uncertainty_penalty
+
+        # Convert evidence to dual confidence format
+        constraint_evidence = []
+        for evidence in evidence_list:
+            dual_evidence = self.analyze_evidence_dual_confidence(
+                evidence, constraint
+            )
+            constraint_evidence.append(dual_evidence)
+
+        # Calculate overall score
+        total_positive = sum(e.positive_confidence for e in constraint_evidence)
+        total_negative = sum(e.negative_confidence for e in constraint_evidence)
+        total_uncertainty = sum(e.uncertainty for e in constraint_evidence)
+
+        # Normalize
+        evidence_count = len(constraint_evidence)
+        avg_positive = total_positive / evidence_count
+        avg_negative = total_negative / evidence_count
+        avg_uncertainty = total_uncertainty / evidence_count
+
+        # Calculate final score
+        # High positive + low negative = high score
+        # Low positive + high negative = low score
+        # High uncertainty = penalty
+        score = (
+            avg_positive
+            - (avg_negative * negative_weight)
+            - (avg_uncertainty * uncertainty_penalty)
+        )
+
+        # Clamp to [0, 1]
+        return max(0.0, min(1.0, score))