local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/advanced_search_system/evidence/base_evidence.py

@@ -0,0 +1,57 @@
+"""
+Base evidence classes for the advanced search system.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, Optional
+
+
+class EvidenceType(Enum):
+    """Types of evidence with inherent reliability scores."""
+
+    DIRECT_STATEMENT = "direct_statement"
+    OFFICIAL_RECORD = "official_record"
+    RESEARCH_FINDING = "research_finding"
+    NEWS_REPORT = "news_report"
+    STATISTICAL_DATA = "statistical_data"
+    INFERENCE = "inference"
+    CORRELATION = "correlation"
+    SPECULATION = "speculation"
+
+    @property
+    def base_confidence(self) -> float:
+        """Get base confidence for this evidence type."""
+        confidence_map = {
+            EvidenceType.DIRECT_STATEMENT: 0.95,
+            EvidenceType.OFFICIAL_RECORD: 0.90,
+            EvidenceType.RESEARCH_FINDING: 0.85,
+            EvidenceType.STATISTICAL_DATA: 0.85,
+            EvidenceType.NEWS_REPORT: 0.75,
+            EvidenceType.INFERENCE: 0.50,
+            EvidenceType.CORRELATION: 0.30,
+            EvidenceType.SPECULATION: 0.10,
+        }
+        return confidence_map.get(self, 0.5)
+
+
+@dataclass
+class Evidence:
+    """Evidence supporting or refuting a claim."""
+
+    claim: str
+    type: EvidenceType
+    source: str
+    confidence: float = 0.0
+    reasoning: Optional[str] = None
+    raw_text: Optional[str] = None
+    timestamp: str = field(
+        default_factory=lambda: datetime.utcnow().isoformat()
+    )
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Calculate initial confidence if not provided."""
+        if self.confidence == 0.0:
+            self.confidence = self.type.base_confidence

local_deep_research/advanced_search_system/evidence/evaluator.py

@@ -0,0 +1,159 @@
+"""
+Evidence evaluator for assessing evidence quality and relevance.
+"""
+
+from typing import Dict
+
+from langchain_core.language_models import BaseChatModel
+from loguru import logger
+
+from ...utilities.search_utilities import remove_think_tags
+from ..constraints.base_constraint import Constraint
+from .base_evidence import Evidence, EvidenceType
+
+
+class EvidenceEvaluator:
+    """Evaluates evidence quality and relevance."""
+
+    def __init__(self, model: BaseChatModel):
+        """Initialize the evidence evaluator."""
+        self.model = model
+        self.source_reliability = {
+            "official": 1.0,
+            "research": 0.95,
+            "news": 0.8,
+            "community": 0.6,
+            "inference": 0.5,
+            "speculation": 0.3,
+        }
+
+    def extract_evidence(
+        self, search_result: str, candidate: str, constraint: Constraint
+    ) -> Evidence:
+        """Extract evidence from search results for a specific constraint."""
+        prompt = f"""
+Extract evidence regarding whether "{candidate}" satisfies this constraint:
+
+Constraint: {constraint.description}
+Constraint Type: {constraint.type.value}
+Required Value: {constraint.value}
+
+Search Results:
+{search_result[:3000]}
+
+Provide:
+1. CLAIM: What the evidence claims about the constraint
+2. TYPE: direct_statement, official_record, research_finding, news_report, statistical_data, inference, correlation, or speculation
+3. SOURCE: Where this evidence comes from
+4. CONFIDENCE: How confident you are this evidence is accurate (0.0-1.0)
+5. REASONING: Why this evidence supports or refutes the constraint
+6. QUOTE: Relevant quote from the search results (if any)
+
+Format:
+CLAIM: [specific claim]
+TYPE: [evidence type]
+SOURCE: [source description]
+CONFIDENCE: [0.0-1.0]
+REASONING: [explanation]
+QUOTE: [relevant text]
+"""
+
+        response = self.model.invoke(prompt)
+        content = remove_think_tags(response.content)
+
+        # Parse response
+        parsed = self._parse_evidence_response(content)
+
+        # Create evidence object
+        # Safely parse confidence value, handling potential errors
+        confidence_str = parsed.get("confidence", "0.5")
+        try:
+            confidence = float(confidence_str)
+            # Ensure confidence is between 0 and 1
+            confidence = max(0.0, min(1.0, confidence))
+        except ValueError:
+            logger.warning(
+                f"Failed to parse confidence value: {confidence_str}"
+            )
+            confidence = 0.5
+
+        evidence = Evidence(
+            claim=parsed.get("claim", "No clear claim"),
+            type=self._parse_evidence_type(parsed.get("type", "speculation")),
+            source=parsed.get("source", "Unknown"),
+            confidence=confidence,
+            reasoning=parsed.get("reasoning", ""),
+            raw_text=parsed.get("quote", ""),
+            metadata={
+                "candidate": candidate,
+                "constraint_id": constraint.id,
+                "constraint_type": constraint.type.value,
+            },
+        )
+
+        # Adjust confidence based on how well it matches the constraint
+        evidence.confidence *= self._assess_match_quality(evidence, constraint)
+
+        return evidence
+
+    def _parse_evidence_response(self, content: str) -> Dict[str, str]:
+        """Parse the LLM response into evidence components."""
+        import re
+
+        parsed = {}
+
+        for line in content.strip().split("\n"):
+            if ":" in line:
+                key, value = line.split(":", 1)
+                key = key.strip().lower()
+                value = value.strip()
+
+                if key in [
+                    "claim",
+                    "type",
+                    "source",
+                    "confidence",
+                    "reasoning",
+                    "quote",
+                ]:
+                    # Special handling for confidence to extract just the float value
+                    if key == "confidence":
+                        # Extract the first float from the value string
+                        match = re.search(r"(\d*\.?\d+)", value)
+                        if match:
+                            parsed[key] = match.group(1)
+                        else:
+                            parsed[key] = value
+                    else:
+                        parsed[key] = value
+
+        return parsed
+
+    def _parse_evidence_type(self, type_str: str) -> EvidenceType:
+        """Parse evidence type from string."""
+        type_map = {
+            "direct_statement": EvidenceType.DIRECT_STATEMENT,
+            "official_record": EvidenceType.OFFICIAL_RECORD,
+            "research_finding": EvidenceType.RESEARCH_FINDING,
+            "news_report": EvidenceType.NEWS_REPORT,
+            "statistical_data": EvidenceType.STATISTICAL_DATA,
+            "inference": EvidenceType.INFERENCE,
+            "correlation": EvidenceType.CORRELATION,
+            "speculation": EvidenceType.SPECULATION,
+        }
+        return type_map.get(type_str.lower(), EvidenceType.SPECULATION)
+
+    def _assess_match_quality(
+        self, evidence: Evidence, constraint: Constraint
+    ) -> float:
+        """Assess how well the evidence matches the constraint."""
+        # This is a simplified version - could be made more sophisticated
+        if constraint.value.lower() in evidence.claim.lower():
+            return 1.0
+        elif any(
+            word in evidence.claim.lower()
+            for word in constraint.value.lower().split()
+        ):
+            return 0.8
+        else:
+            return 0.6  # Partial match at best

local_deep_research/advanced_search_system/evidence/requirements.py

@@ -0,0 +1,122 @@
+"""
+Evidence requirements for different constraint types.
+"""
+
+from typing import Dict, List
+
+from ..constraints.base_constraint import ConstraintType
+
+
+class EvidenceRequirements:
+    """Define evidence requirements for different constraint types."""
+
+    @staticmethod
+    def get_requirements(
+        constraint_type: ConstraintType,
+    ) -> Dict[str, List[str]]:
+        """Get evidence requirements for a constraint type.
+
+        Args:
+            constraint_type: The type of constraint
+
+        Returns:
+            Dictionary of evidence types and their sources
+        """
+        requirements = {
+            ConstraintType.PROPERTY: {
+                "preferred": ["direct_statement", "official_record"],
+                "acceptable": ["research_finding", "inference"],
+                "sources": [
+                    "scientific papers",
+                    "official documents",
+                    "encyclopedias",
+                ],
+            },
+            ConstraintType.NAME_PATTERN: {
+                "preferred": ["direct_statement", "linguistic_analysis"],
+                "acceptable": ["correlation", "inference"],
+                "sources": [
+                    "etymology sources",
+                    "naming databases",
+                    "historical records",
+                ],
+            },
+            ConstraintType.EVENT: {
+                "preferred": ["news_report", "official_record"],
+                "acceptable": ["testimonial", "correlation"],
+                "sources": [
+                    "news archives",
+                    "government reports",
+                    "witness accounts",
+                ],
+            },
+            ConstraintType.STATISTIC: {
+                "preferred": ["statistical_data", "official_record"],
+                "acceptable": ["research_finding"],
+                "sources": [
+                    "government databases",
+                    "research papers",
+                    "official reports",
+                ],
+            },
+            ConstraintType.TEMPORAL: {
+                "preferred": ["official_record", "news_report"],
+                "acceptable": ["historical_record", "inference"],
+                "sources": ["archives", "newspapers", "official timelines"],
+            },
+            ConstraintType.LOCATION: {
+                "preferred": ["geographical_data", "official_record"],
+                "acceptable": ["mapping_data", "inference"],
+                "sources": [
+                    "geographical surveys",
+                    "maps",
+                    "location databases",
+                ],
+            },
+            ConstraintType.COMPARISON: {
+                "preferred": ["statistical_comparison", "research_finding"],
+                "acceptable": ["inference", "correlation"],
+                "sources": [
+                    "comparative studies",
+                    "statistical analyses",
+                    "research papers",
+                ],
+            },
+            ConstraintType.EXISTENCE: {
+                "preferred": ["direct_statement", "official_record"],
+                "acceptable": ["news_report", "inference"],
+                "sources": [
+                    "official registries",
+                    "databases",
+                    "authoritative sources",
+                ],
+            },
+        }
+
+        return requirements.get(
+            constraint_type,
+            {
+                "preferred": ["direct_statement"],
+                "acceptable": ["inference"],
+                "sources": ["general sources"],
+            },
+        )
+
+    @staticmethod
+    def get_minimum_confidence(constraint_type: ConstraintType) -> float:
+        """Get minimum confidence required for constraint type.
+
+        Args:
+            constraint_type: The type of constraint
+
+        Returns:
+            Minimum confidence threshold
+        """
+        thresholds = {
+            ConstraintType.STATISTIC: 0.8,  # High accuracy needed
+            ConstraintType.EVENT: 0.7,  # Moderate accuracy
+            ConstraintType.PROPERTY: 0.6,  # Some flexibility
+            ConstraintType.NAME_PATTERN: 0.5,  # More interpretive
+        }
+
+        return thresholds.get(constraint_type, 0.6)

local_deep_research/advanced_search_system/filters/base_filter.py

@@ -22,7 +22,9 @@ class BaseFilter(ABC):
         self.model = model
 
     @abstractmethod
-    def filter_results(self, results: List[Dict], query: str, **kwargs) -> List[Dict]:
+    def filter_results(
+        self, results: List[Dict], query: str, **kwargs
+    ) -> List[Dict]:
         """
         Filter search results by relevance to the query.
 

local_deep_research/advanced_search_system/filters/cross_engine_filter.py

@@ -16,7 +16,11 @@ class CrossEngineFilter(BaseFilter):
     """Filter that ranks and filters results from multiple search engines."""
 
     def __init__(
-        self, model, max_results=None, default_reorder=True, default_reindex=True
+        self,
+        model,
+        max_results=None,
+        default_reorder=True,
+        default_reindex=True,
     ):
         """
         Initialize the cross-engine filter.
@@ -30,7 +34,9 @@ class CrossEngineFilter(BaseFilter):
         super().__init__(model)
         # Get max_results from database settings if not provided
         if max_results is None:
-            max_results = int(get_db_setting("search.cross_engine_max_results", 100))
+            max_results = int(
+                get_db_setting("search.cross_engine_max_results", 100)
+            )
         self.max_results = max_results
         self.default_reorder = default_reorder
         self.default_reindex = default_reindex

local_deep_research/advanced_search_system/filters/journal_reputation_filter.py

@@ -69,13 +69,17 @@ class JournalReputationFilter(BaseFilter):
         self.__exclude_non_published = exclude_non_published
         if self.__exclude_non_published is None:
             self.__exclude_non_published = bool(
-                get_db_setting("search.journal_reputation.exclude_non_published", False)
+                get_db_setting(
+                    "search.journal_reputation.exclude_non_published", False
+                )
             )
         self.__quality_reanalysis_period = quality_reanalysis_period
         if self.__quality_reanalysis_period is None:
             self.__quality_reanalysis_period = timedelta(
                 days=int(
-                    get_db_setting("search.journal_reputation.reanalysis_period", 365)
+                    get_db_setting(
+                        "search.journal_reputation.reanalysis_period", 365
+                    )
                 )
             )
 
@@ -85,8 +89,6 @@ class JournalReputationFilter(BaseFilter):
         if self.__engine is None:
             raise JournalFilterError("SearXNG initialization failed.")
 
-        self.__db_session = get_db_session()
-
     @classmethod
     def create_default(
         cls, model: BaseChatModel | None = None, *, engine_name: str
@@ -159,7 +161,9 @@ class JournalReputationFilter(BaseFilter):
             f"ranking and peer review status. Be sure to specify the journal "
             f"name in any generated questions."
         )
-        journal_info = "\n".join([f["content"] for f in journal_info["findings"]])
+        journal_info = "\n".join(
+            [f["content"] for f in journal_info["findings"]]
+        )
         logger.debug(f"Received raw info about journal: {journal_info}")
 
         # Have the LLM assess the reliability based on this information.
@@ -190,7 +194,9 @@ class JournalReputationFilter(BaseFilter):
             reputation_score = int(response.strip())
         except ValueError:
             logger.error("Failed to parse reputation score from LLM response.")
-            raise ValueError("Failed to parse reputation score from LLM response.")
+            raise ValueError(
+                "Failed to parse reputation score from LLM response."
+            )
 
         return max(min(reputation_score, 10), 1)
 
@@ -203,21 +209,22 @@ class JournalReputationFilter(BaseFilter):
             quality: The quality assessment for the journal.
 
         """
-        journal = self.__db_session.query(Journal).filter_by(name=name).first()
-        if journal is not None:
-            journal.quality = quality
-            journal.quality_model = self.model.name
-            journal.quality_analysis_time = int(time.time())
-        else:
-            journal = Journal(
-                name=name,
-                quality=quality,
-                quality_model=self.model.name,
-                quality_analysis_time=int(time.time()),
-            )
-            self.__db_session.add(journal)
+        with get_db_session() as db_session:
+            journal = db_session.query(Journal).filter_by(name=name).first()
+            if journal is not None:
+                journal.quality = quality
+                journal.quality_model = self.model.name
+                journal.quality_analysis_time = int(time.time())
+            else:
+                journal = Journal(
+                    name=name,
+                    quality=quality,
+                    quality_model=self.model.name,
+                    quality_analysis_time=int(time.time()),
+                )
+                db_session.add(journal)
 
-        self.__db_session.commit()
+            db_session.commit()
 
     def __clean_journal_name(self, journal_name: str) -> str:
         """
@@ -268,14 +275,19 @@ class JournalReputationFilter(BaseFilter):
         journal_name = self.__clean_journal_name(journal_name)
 
         # Check the database first.
-        journal = self.__db_session.query(Journal).filter_by(name=journal_name).first()
-        if (
-            journal is not None
-            and (time.time() - journal.quality_analysis_time)
-            < self.__quality_reanalysis_period.total_seconds()
-        ):
-            logger.debug(f"Found existing reputation for {journal_name} in database.")
-            return journal.quality >= self.__threshold
+        with get_db_session() as session:
+            journal = (
+                session.query(Journal).filter_by(name=journal_name).first()
+            )
+            if (
+                journal is not None
+                and (time.time() - journal.quality_analysis_time)
+                < self.__quality_reanalysis_period.total_seconds()
+            ):
+                logger.debug(
+                    f"Found existing reputation for {journal_name} in database."
+                )
+                return journal.quality >= self.__threshold
 
         # Evaluate reputation.
         try:
@@ -288,7 +300,9 @@ class JournalReputationFilter(BaseFilter):
             # okay.
             return True
 
-    def filter_results(self, results: List[Dict], query: str, **kwargs) -> List[Dict]:
+    def filter_results(
+        self, results: List[Dict], query: str, **kwargs
+    ) -> List[Dict]:
         try:
             return list(filter(self.__check_result, results))
         except Exception as e:

local_deep_research/advanced_search_system/findings/repository.py

@@ -63,7 +63,10 @@ class FindingsRepository(BaseFindingsRepository):
 
             # Store raw synthesized content if it's the final synthesis
             # Only check for phase if it's a dictionary
-            if isinstance(finding, dict) and finding.get("phase") == "Final synthesis":
+            if (
+                isinstance(finding, dict)
+                and finding.get("phase") == "Final synthesis"
+            ):
                 self.findings[query + "_synthesis"] = [
                     {
                         "phase": "Synthesis",
@@ -117,7 +120,9 @@ class FindingsRepository(BaseFindingsRepository):
             questions_by_iteration: Dictionary mapping iteration numbers to lists of questions
         """
         self.questions_by_iteration = questions_by_iteration.copy()
-        logger.info(f"Set questions for {len(questions_by_iteration)} iterations")
+        logger.info(
+            f"Set questions for {len(questions_by_iteration)} iterations"
+        )
 
     def format_findings_to_text(
         self, findings_list: List[Dict], synthesized_content: str
@@ -142,7 +147,9 @@ class FindingsRepository(BaseFindingsRepository):
             f"Questions by iteration keys: {list(self.questions_by_iteration.keys())}"
         )
         if findings_list:
-            logger.debug(f"First finding item keys: {list(findings_list[0].keys())}")
+            logger.debug(
+                f"First finding item keys: {list(findings_list[0].keys())}"
+            )
 
         try:
             # Pass the detailed findings list, the synthesized content (as current_knowledge), and the stored questions
@@ -211,14 +218,18 @@ class FindingsRepository(BaseFindingsRepository):
                     )
             elif isinstance(findings[0], str):
                 logger.info(f"first finding string length: {len(findings[0])}")
-                logger.info(f"first finding string preview: {findings[0][:100]}...")
+                logger.info(
+                    f"first finding string preview: {findings[0][:100]}..."
+                )
 
         if old_formatting:
             # Convert findings list if it contains strings instead of dictionaries
             findings_list = []
             for i, item in enumerate(findings):
                 if isinstance(item, str):
-                    findings_list.append({"phase": f"Finding {i + 1}", "content": item})
+                    findings_list.append(
+                        {"phase": f"Finding {i + 1}", "content": item}
+                    )
                 elif isinstance(item, dict):
                     findings_list.append(item)
 
@@ -237,12 +248,16 @@ class FindingsRepository(BaseFindingsRepository):
                     finding_texts.append(item)
 
             # Use finding_texts for the prompt
-            current_knowledge = "\n\n".join(finding_texts) if finding_texts else ""
+            current_knowledge = (
+                "\n\n".join(finding_texts) if finding_texts else ""
+            )
 
             # Check if knowledge exceeds a reasonable token limit (rough estimate based on characters)
             # 1 token ≈ 4 characters in English
             estimated_tokens = len(current_knowledge) / 4
-            max_safe_tokens = 12000  # Adjust based on your model's context window
+            max_safe_tokens = (
+                12000  # Adjust based on your model's context window
+            )
 
             if estimated_tokens > max_safe_tokens:
                 logger.warning(
@@ -251,10 +266,16 @@ class FindingsRepository(BaseFindingsRepository):
                 # Truncate if needed (keeping the beginning and end which are often most important)
                 # This is a simple approach - a more sophisticated chunking might be better
                 if len(current_knowledge) > 24000:  # ~6000 tokens
-                    first_part = current_knowledge[:12000]  # ~3000 tokens from start
-                    last_part = current_knowledge[-12000:]  # ~3000 tokens from end
+                    first_part = current_knowledge[
+                        :12000
+                    ]  # ~3000 tokens from start
+                    last_part = current_knowledge[
+                        -12000:
+                    ]  # ~3000 tokens from end
                     current_knowledge = f"{first_part}\n\n[...content truncated due to length...]\n\n{last_part}"
-                    logger.info("Knowledge truncated to fit within token limits")
+                    logger.info(
+                        "Knowledge truncated to fit within token limits"
+                    )
 
             prompt = f"""Use IEEE style citations [1], [2], etc. Never make up your own citations. Synthesize the following accumulated knowledge into a comprehensive answer for the original query.
 Format the response with clear sections, citations, and a concise summary.
@@ -280,7 +301,9 @@ Use IEEE style citations [1], [2], etc. Never make up your own citations.
                 f"Synthesizing final answer. Query: '{query}'. Knowledge length: {len(current_knowledge)}. Prompt length: {len(prompt)}"
             )
             # Log first 500 chars of prompt for debugging context length issues
-            logger.debug(f"Synthesis prompt (first 500 chars): {prompt[:500]}...")
+            logger.debug(
+                f"Synthesis prompt (first 500 chars): {prompt[:500]}..."
+            )
 
             try:
                 # Add timeout handling
@@ -300,7 +323,9 @@ Use IEEE style citations [1], [2], etc. Never make up your own citations.
                         timer.daemon = True
                         return timer
 
-                    def invoke_with_timeout(timeout_seconds, func, *args, **kwargs):
+                    def invoke_with_timeout(
+                        timeout_seconds, func, *args, **kwargs
+                    ):
                         """
                         Function for implementing timeouts on Windows
                         """
@@ -338,7 +363,9 @@ Use IEEE style citations [1], [2], etc. Never make up your own citations.
                         logger.info(
                             "Using Windows-compatible timeout for LLM invocation"
                         )
-                        response = invoke_with_timeout(120, self.model.invoke, prompt)
+                        response = invoke_with_timeout(
+                            120, self.model.invoke, prompt
+                        )
 
                         # Handle different response types (string or object with content attribute)
                         if hasattr(response, "content"):
@@ -376,7 +403,9 @@ Use IEEE style citations [1], [2], etc. Never make up your own citations.
 
                     # Try with a timeout (adjust seconds as needed)
                     try:
-                        with timeout(120, "LLM invocation timed out after 120 seconds"):
+                        with timeout(
+                            120, "LLM invocation timed out after 120 seconds"
+                        ):
                             response = self.model.invoke(prompt)
 
                             # Handle different response types (string or object with content attribute)
@@ -417,11 +446,19 @@ Use IEEE style citations [1], [2], etc. Never make up your own citations.
                     or "token limit" in error_message
                 ):
                     error_type = "token_limit"
-                elif "rate limit" in error_message or "rate_limit" in error_message:
+                elif (
+                    "rate limit" in error_message
+                    or "rate_limit" in error_message
+                ):
                     error_type = "rate_limit"
-                elif "connection" in error_message or "network" in error_message:
+                elif (
+                    "connection" in error_message or "network" in error_message
+                ):
                     error_type = "connection"
-                elif "api key" in error_message or "authentication" in error_message:
+                elif (
+                    "api key" in error_message
+                    or "authentication" in error_message
+                ):
                     error_type = "authentication"
 
                 # Return more detailed error message based on type

local_deep_research/advanced_search_system/knowledge/standard_knowledge.py

@@ -132,7 +132,9 @@ Compressed Knowledge:"""
         try:
             response = self.model.invoke(prompt)
             compressed_knowledge = response.content
-            logger.info(f"Compressed knowledge length: {len(compressed_knowledge)}")
+            logger.info(
+                f"Compressed knowledge length: {len(compressed_knowledge)}"
+            )
             return compressed_knowledge
         except Exception as e:
             logger.error(f"Error compressing knowledge: {str(e)}")