cite-agent 1.3.9__py3-none-any.whl → 1.4.3__py3-none-any.whl

cite_agent/confidence_calibration.py

@@ -0,0 +1,381 @@
+"""
+Confidence Calibration - Know When Uncertain
+Assesses confidence in responses and adds appropriate caveats
+
+This prevents overconfident wrong answers
+"""
+
+import logging
+import re
+from typing import Dict, Any, Optional
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConfidenceAssessment:
+    """Result of confidence assessment"""
+    confidence_score: float  # 0.0-1.0
+    confidence_level: str  # "high", "medium", "low"
+    should_add_caveat: bool
+    caveat_text: Optional[str]
+    factors: Dict[str, float]  # What contributed to confidence
+    reasoning: str
+
+
+class ConfidenceCalibrator:
+    """
+    Assesses confidence in responses
+
+    Factors that affect confidence:
+    1. Data quality (Do we have good data?)
+    2. Query clarity (Is the question clear?)
+    3. Answer completeness (Did we answer fully?)
+    4. Source reliability (Are sources trustworthy?)
+    5. Response consistency (Does answer make sense?)
+    """
+
+    # Confidence thresholds
+    HIGH_CONFIDENCE = 0.8
+    MEDIUM_CONFIDENCE = 0.6
+    LOW_CONFIDENCE = 0.4
+
+    @classmethod
+    def assess_confidence(
+        cls,
+        response: str,
+        query: str,
+        context: Dict[str, Any]
+    ) -> ConfidenceAssessment:
+        """
+        Assess confidence in a response
+
+        Args:
+            response: The generated response
+            query: Original user query
+            context: Context including tools used, data sources, etc.
+
+        Returns:
+            Confidence assessment with score and caveat if needed
+        """
+        factors = {}
+
+        # Factor 1: Data quality (40% weight)
+        factors['data_quality'] = cls._assess_data_quality(context)
+
+        # Factor 2: Query clarity (20% weight)
+        factors['query_clarity'] = cls._assess_query_clarity(query)
+
+        # Factor 3: Answer completeness (25% weight)
+        factors['answer_completeness'] = cls._assess_completeness(response, query)
+
+        # Factor 4: Source reliability (10% weight)
+        factors['source_reliability'] = cls._assess_source_reliability(context)
+
+        # Factor 5: Response consistency (5% weight)
+        factors['response_consistency'] = cls._assess_consistency(response)
+
+        # Calculate weighted confidence score
+        weights = {
+            'data_quality': 0.40,
+            'query_clarity': 0.20,
+            'answer_completeness': 0.25,
+            'source_reliability': 0.10,
+            'response_consistency': 0.05
+        }
+
+        confidence_score = sum(factors[k] * weights[k] for k in weights)
+
+        # Determine confidence level
+        if confidence_score >= cls.HIGH_CONFIDENCE:
+            confidence_level = "high"
+        elif confidence_score >= cls.MEDIUM_CONFIDENCE:
+            confidence_level = "medium"
+        else:
+            confidence_level = "low"
+
+        # Determine if we should add a caveat
+        should_add_caveat = confidence_score < cls.MEDIUM_CONFIDENCE
+
+        # Generate caveat text if needed
+        caveat_text = None
+        if should_add_caveat:
+            caveat_text = cls._generate_caveat(confidence_score, factors, context)
+
+        # Generate reasoning
+        reasoning = cls._explain_confidence(confidence_score, factors)
+
+        return ConfidenceAssessment(
+            confidence_score=confidence_score,
+            confidence_level=confidence_level,
+            should_add_caveat=should_add_caveat,
+            caveat_text=caveat_text,
+            factors=factors,
+            reasoning=reasoning
+        )
+
+    @classmethod
+    def _assess_data_quality(cls, context: Dict[str, Any]) -> float:
+        """
+        Assess quality of data used
+
+        High quality: Multiple reliable sources, recent data
+        Low quality: No sources, old data, incomplete data
+        """
+        score = 0.5  # Neutral start
+
+        tools_used = context.get('tools_used', [])
+        api_results = context.get('api_results', {})
+
+        # Check if we have data sources
+        has_data = bool(api_results) or bool(tools_used)
+
+        if not has_data:
+            return 0.3  # Low confidence if no data
+
+        # Check quality of data sources
+        reliable_sources = ['archive_api', 'finsight_api', 'shell_execution']
+        unreliable_sources = ['web_search']
+
+        reliable_count = sum(1 for tool in tools_used if tool in reliable_sources)
+        unreliable_count = sum(1 for tool in tools_used if tool in unreliable_sources)
+
+        if reliable_count > 0:
+            score += 0.3
+
+        if unreliable_count > 0 and reliable_count == 0:
+            score -= 0.2
+
+        # Check for empty results
+        if api_results:
+            # Check if results are empty
+            for key, value in api_results.items():
+                if isinstance(value, dict) and value.get('results') == []:
+                    score -= 0.4  # Major penalty for empty results
+                elif isinstance(value, list) and not value:
+                    score -= 0.3
+
+        return min(1.0, max(0.0, score))
+
+    @classmethod
+    def _assess_query_clarity(cls, query: str) -> float:
+        """
+        Assess how clear the query is
+
+        Clear query: Specific, unambiguous
+        Unclear query: Vague, pronouns without context, too short
+        """
+        score = 1.0  # Start optimistic
+
+        # Too short is often ambiguous
+        word_count = len(query.split())
+        if word_count < 3:
+            score -= 0.3
+
+        # Pronouns without context
+        pronouns = ['it', 'that', 'those', 'this', 'them']
+        has_pronoun = any(pronoun in query.lower().split() for pronoun in pronouns)
+
+        if has_pronoun and word_count < 8:
+            score -= 0.2
+
+        # Very vague terms
+        vague_terms = ['something', 'stuff', 'things', 'anything']
+        if any(term in query.lower() for term in vague_terms):
+            score -= 0.25
+
+        # Question marks without clear question
+        if '?' in query and word_count < 4:
+            score -= 0.15
+
+        return min(1.0, max(0.0, score))
+
+    @classmethod
+    def _assess_completeness(cls, response: str, query: str) -> float:
+        """
+        Assess if response fully answers the query
+
+        Complete: Addresses all aspects of query
+        Incomplete: Partial answer, deflects, says "I don't know"
+        """
+        score = 0.7  # Assume mostly complete
+
+        response_lower = response.lower()
+
+        # Check for deflection
+        deflection_phrases = [
+            "i don't know",
+            "i'm not sure",
+            "i don't have",
+            "i can't",
+            "unclear"
+        ]
+
+        if any(phrase in response_lower for phrase in deflection_phrases):
+            score -= 0.4
+
+        # Check response length relative to query complexity
+        query_words = len(query.split())
+        response_words = len(response.split())
+
+        if query_words > 10 and response_words < 30:
+            score -= 0.2  # Too brief for complex query
+
+        # Check if response addresses key terms from query
+        query_terms = set(query.lower().split())
+        response_terms = set(response_lower.split())
+
+        # Remove stop words
+        stop_words = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'what', 'how', 'why'}
+        query_terms -= stop_words
+
+        if query_terms:
+            overlap = len(query_terms & response_terms) / len(query_terms)
+            if overlap < 0.3:
+                score -= 0.25
+
+        return min(1.0, max(0.0, score))
+
+    @classmethod
+    def _assess_source_reliability(cls, context: Dict[str, Any]) -> float:
+        """
+        Assess reliability of sources used
+
+        Reliable: Official APIs, verified databases
+        Unreliable: Web scraping, unverified sources
+        """
+        tools_used = context.get('tools_used', [])
+
+        if not tools_used:
+            return 0.5  # Neutral if no tools used
+
+        # Reliability scores for different tools
+        reliability_map = {
+            'archive_api': 0.95,  # Academic papers - very reliable
+            'finsight_api': 0.90,  # SEC filings - very reliable
+            'shell_execution': 0.85,  # Direct file access - reliable
+            'web_search': 0.60,  # Web - less reliable
+            'fallback': 0.30  # Fallback responses - low reliability
+        }
+
+        # Average reliability of tools used
+        reliabilities = [reliability_map.get(tool, 0.5) for tool in tools_used]
+        avg_reliability = sum(reliabilities) / len(reliabilities) if reliabilities else 0.5
+
+        return avg_reliability
+
+    @classmethod
+    def _assess_consistency(cls, response: str) -> float:
+        """
+        Assess internal consistency of response
+
+        Consistent: No contradictions, logical flow
+        Inconsistent: Contradictions, illogical statements
+        """
+        score = 1.0  # Assume consistent
+
+        # Check for hedge words that indicate uncertainty
+        hedge_words = ['maybe', 'possibly', 'might', 'could', 'perhaps']
+        hedge_count = sum(1 for word in hedge_words if f' {word} ' in response.lower())
+
+        if hedge_count > 3:
+            score -= 0.2  # Too many hedges suggests uncertainty
+
+        # Check for contradictions
+        contradiction_patterns = [
+            ('but ', 'however '),
+            ('although ', 'though '),
+            ('not ', 'no ')
+        ]
+
+        contradiction_count = sum(
+            1 for pattern_words in contradiction_patterns
+            if all(word in response.lower() for word in pattern_words)
+        )
+
+        if contradiction_count > 2:
+            score -= 0.15
+
+        return min(1.0, max(0.0, score))
+
+    @classmethod
+    def _generate_caveat(cls, confidence_score: float, factors: Dict[str, float], context: Dict[str, Any]) -> str:
+        """
+        Generate appropriate caveat text based on confidence factors
+        """
+        # Identify the weakest factor
+        weakest_factor = min(factors.items(), key=lambda x: x[1])
+        factor_name, factor_score = weakest_factor
+
+        caveats = {
+            'data_quality': "Based on limited data available, ",
+            'query_clarity': "I interpreted your question to mean: ",
+            'answer_completeness': "Based on what I could find, ",
+            'source_reliability': "According to available sources, ",
+            'response_consistency': "To the best of my understanding, "
+        }
+
+        caveat_prefix = caveats.get(factor_name, "Based on available information, ")
+
+        # Add severity based on overall confidence
+        if confidence_score < cls.LOW_CONFIDENCE:
+            caveat_prefix = "⚠️ Low confidence: " + caveat_prefix
+
+        return caveat_prefix
+
+    @classmethod
+    def _explain_confidence(cls, confidence_score: float, factors: Dict[str, float]) -> str:
+        """Generate explanation of confidence level"""
+        level = "high" if confidence_score >= cls.HIGH_CONFIDENCE else \
+                "medium" if confidence_score >= cls.MEDIUM_CONFIDENCE else "low"
+
+        # Identify top contributing factors
+        sorted_factors = sorted(factors.items(), key=lambda x: x[1], reverse=True)
+        top_factor = sorted_factors[0][0]
+        bottom_factor = sorted_factors[-1][0]
+
+        reasoning = (
+            f"Confidence level: {level} ({confidence_score:.2f}). "
+            f"Strongest factor: {top_factor} ({factors[top_factor]:.2f}). "
+            f"Weakest factor: {bottom_factor} ({factors[bottom_factor]:.2f})."
+        )
+
+        return reasoning
+
+    @classmethod
+    def add_caveat_to_response(cls, response: str, caveat: str) -> str:
+        """
+        Add caveat to response in a natural way
+
+        Inserts caveat at the beginning of the response
+        """
+        # If response already starts with a caveat-like phrase, don't add another
+        caveat_starters = ['based on', 'according to', 'to the best', 'from what']
+
+        response_lower = response.lower()
+        if any(starter in response_lower[:50] for starter in caveat_starters):
+            return response  # Already has a caveat
+
+        # Add caveat at the start
+        return caveat + response[0].lower() + response[1:]
+
+
+# Convenience function
+def assess_and_apply_caveat(response: str, query: str, context: Dict[str, Any]) -> tuple:
+    """
+    Assess confidence and apply caveat if needed
+
+    Returns:
+        (final_response, confidence_assessment)
+    """
+    assessment = ConfidenceCalibrator.assess_confidence(response, query, context)
+
+    final_response = response
+    if assessment.should_add_caveat and assessment.caveat_text:
+        final_response = ConfidenceCalibrator.add_caveat_to_response(
+            response,
+            assessment.caveat_text
+        )
+
+    return final_response, assessment

cite_agent/deduplication.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""
+Paper Deduplication - Remove Duplicate Papers
+
+When searching multiple sources (Semantic Scholar, OpenAlex, PubMed),
+the same paper often appears multiple times.
+
+This module provides intelligent deduplication:
+- By DOI (most reliable)
+- By title similarity (fuzzy matching)
+- By arXiv ID
+- Merge metadata from multiple sources
+"""
+
+import logging
+from typing import List, Dict, Any, Set, Optional
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PaperIdentifiers:
+    """All possible identifiers for a paper"""
+    doi: Optional[str] = None
+    arxiv_id: Optional[str] = None
+    pmid: Optional[str] = None  # PubMed ID
+    semantic_scholar_id: Optional[str] = None
+    openalex_id: Optional[str] = None
+    title: Optional[str] = None
+
+
+class PaperDeduplicator:
+    """
+    Intelligent paper deduplication
+
+    Strategy:
+    1. Exact DOI match (highest confidence)
+    2. arXiv ID match
+    3. PubMed ID match
+    4. Title fuzzy match (>90% similarity)
+    5. Keep best version (most complete metadata)
+    """
+
+    def __init__(self, title_similarity_threshold: float = 0.9):
+        """
+        Initialize deduplicator
+
+        Args:
+            title_similarity_threshold: Minimum similarity for title matching (0.0-1.0)
+        """
+        self.title_threshold = title_similarity_threshold
+
+    def deduplicate(self, papers: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Deduplicate list of papers
+
+        Args:
+            papers: List of papers from multiple sources
+
+        Returns:
+            Deduplicated list with merged metadata
+        """
+        if not papers:
+            return []
+
+        # Group papers by identifiers
+        groups = self._group_duplicates(papers)
+
+        # Merge each group into best representation
+        deduplicated = []
+        for group in groups:
+            merged = self._merge_papers(group)
+            deduplicated.append(merged)
+
+        original_count = len(papers)
+        final_count = len(deduplicated)
+        removed = original_count - final_count
+
+        if removed > 0:
+            logger.info(f"🔍 Deduplicated: {original_count} → {final_count} papers ({removed} duplicates removed)")
+
+        return deduplicated
+
+    def _group_duplicates(self, papers: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]:
+        """
+        Group duplicate papers together
+
+        Args:
+            papers: List of papers
+
+        Returns:
+            List of groups, where each group contains duplicate papers
+        """
+        # Track which papers are already grouped
+        grouped_indices: Set[int] = set()
+        groups: List[List[Dict[str, Any]]] = []
+
+        for i, paper1 in enumerate(papers):
+            if i in grouped_indices:
+                continue
+
+            # Start new group with this paper
+            group = [paper1]
+            grouped_indices.add(i)
+
+            # Find all duplicates
+            for j, paper2 in enumerate(papers[i+1:], start=i+1):
+                if j in grouped_indices:
+                    continue
+
+                if self._are_duplicates(paper1, paper2):
+                    group.append(paper2)
+                    grouped_indices.add(j)
+
+            groups.append(group)
+
+        return groups
+
+    def _are_duplicates(self, paper1: Dict[str, Any], paper2: Dict[str, Any]) -> bool:
+        """
+        Check if two papers are duplicates
+
+        Args:
+            paper1: First paper
+            paper2: Second paper
+
+        Returns:
+            True if papers are duplicates
+        """
+        # Extract identifiers
+        id1 = self._extract_identifiers(paper1)
+        id2 = self._extract_identifiers(paper2)
+
+        # Check DOI match (most reliable)
+        if id1.doi and id2.doi:
+            # Normalize DOIs (remove https://doi.org/ prefix)
+            doi1 = id1.doi.lower().replace("https://doi.org/", "").strip()
+            doi2 = id2.doi.lower().replace("https://doi.org/", "").strip()
+            if doi1 == doi2:
+                logger.debug(f"DOI match: {doi1}")
+                return True
+
+        # Check arXiv ID match
+        if id1.arxiv_id and id2.arxiv_id:
+            if id1.arxiv_id.lower() == id2.arxiv_id.lower():
+                logger.debug(f"arXiv match: {id1.arxiv_id}")
+                return True
+
+        # Check PubMed ID match
+        if id1.pmid and id2.pmid:
+            if id1.pmid == id2.pmid:
+                logger.debug(f"PMID match: {id1.pmid}")
+                return True
+
+        # Check title similarity (fuzzy matching)
+        if id1.title and id2.title:
+            similarity = self._title_similarity(id1.title, id2.title)
+            if similarity >= self.title_threshold:
+                logger.debug(f"Title match: {similarity:.2%} - '{id1.title[:50]}...'")
+                return True
+
+        return False
+
+    def _extract_identifiers(self, paper: Dict[str, Any]) -> PaperIdentifiers:
+        """
+        Extract all identifiers from paper
+
+        Args:
+            paper: Paper dictionary
+
+        Returns:
+            PaperIdentifiers object
+        """
+        # Handle different API formats
+        return PaperIdentifiers(
+            doi=paper.get("doi") or paper.get("DOI"),
+            arxiv_id=paper.get("arxivId") or paper.get("arxiv_id"),
+            pmid=paper.get("pmid") or paper.get("pubmed_id"),
+            semantic_scholar_id=paper.get("paperId") or paper.get("semantic_scholar_id"),
+            openalex_id=paper.get("id") if isinstance(paper.get("id"), str) and paper.get("id", "").startswith("https://openalex.org/") else None,
+            title=paper.get("title")
+        )
+
+    def _title_similarity(self, title1: str, title2: str) -> float:
+        """
+        Calculate similarity between two titles
+
+        Args:
+            title1: First title
+            title2: Second title
+
+        Returns:
+            Similarity score (0.0-1.0)
+        """
+        # Normalize titles
+        t1 = self._normalize_title(title1)
+        t2 = self._normalize_title(title2)
+
+        # Calculate similarity using SequenceMatcher
+        return SequenceMatcher(None, t1, t2).ratio()
+
+    def _normalize_title(self, title: str) -> str:
+        """
+        Normalize title for comparison
+
+        Args:
+            title: Original title
+
+        Returns:
+            Normalized title
+        """
+        # Convert to lowercase
+        normalized = title.lower()
+
+        # Remove common punctuation
+        for char in [".", ",", ":", ";", "!", "?", "-", "(", ")", "[", "]", "{", "}"]:
+            normalized = normalized.replace(char, " ")
+
+        # Normalize whitespace
+        normalized = " ".join(normalized.split())
+
+        return normalized
+
+    def _merge_papers(self, papers: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """
+        Merge duplicate papers into best representation
+
+        Strategy:
+        - Keep most complete metadata
+        - Prefer DOI > arXiv > Semantic Scholar
+        - Merge citation counts (take max)
+        - Merge sources
+
+        Args:
+            papers: List of duplicate papers
+
+        Returns:
+            Merged paper with best metadata
+        """
+        if len(papers) == 1:
+            return papers[0]
+
+        # Start with paper that has most fields
+        merged = max(papers, key=lambda p: len([v for v in p.values() if v is not None]))
+
+        # Merge citation counts (take maximum)
+        citation_counts = [p.get("citationCount", 0) for p in papers]
+        if citation_counts:
+            merged["citationCount"] = max(citation_counts)
+
+        # Merge sources (track which APIs returned this paper)
+        sources = set()
+        for paper in papers:
+            if paper.get("paperId"):  # Semantic Scholar
+                sources.add("semantic_scholar")
+            if paper.get("id", "").startswith("https://openalex.org/"):
+                sources.add("openalex")
+            if paper.get("pmid"):
+                sources.add("pubmed")
+
+        merged["_sources"] = list(sources)
+        merged["_duplicate_count"] = len(papers)
+
+        # Fill in any missing fields from other papers
+        for paper in papers:
+            for key, value in paper.items():
+                if value is not None and merged.get(key) is None:
+                    merged[key] = value
+
+        return merged
+
+    def get_deduplication_stats(self, original: List[Dict[str, Any]], deduplicated: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """
+        Get statistics about deduplication
+
+        Args:
+            original: Original paper list
+            deduplicated: Deduplicated list
+
+        Returns:
+            Deduplication statistics
+        """
+        removed = len(original) - len(deduplicated)
+        removal_rate = (removed / len(original) * 100) if original else 0
+
+        # Count by source
+        sources = {}
+        for paper in deduplicated:
+            for source in paper.get("_sources", []):
+                sources[source] = sources.get(source, 0) + 1
+
+        return {
+            "original_count": len(original),
+            "deduplicated_count": len(deduplicated),
+            "removed_count": removed,
+            "removal_rate": removal_rate,
+            "sources": sources
+        }
+
+
+# Global deduplicator instance
+_deduplicator = None
+
+
+def get_deduplicator() -> PaperDeduplicator:
+    """Get global deduplicator instance"""
+    global _deduplicator
+    if _deduplicator is None:
+        _deduplicator = PaperDeduplicator()
+    return _deduplicator
+
+
+def deduplicate_papers(papers: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Convenience function to deduplicate papers
+
+    Args:
+        papers: List of papers
+
+    Returns:
+        Deduplicated list
+    """
+    return get_deduplicator().deduplicate(papers)