hindsight-api 0.0.13__py3-none-any.whl

hindsight_api/engine/search/fusion.py

@@ -0,0 +1,122 @@
+"""
+Helper functions for hybrid search (semantic + BM25 + graph).
+"""
+
+from typing import List, Dict, Any, Tuple
+import asyncio
+from .types import RetrievalResult, MergedCandidate
+
+
+def reciprocal_rank_fusion(
+    result_lists: List[List[RetrievalResult]],
+    k: int = 60
+) -> List[MergedCandidate]:
+    """
+    Merge multiple ranked result lists using Reciprocal Rank Fusion.
+
+    RRF formula: score(d) = sum_over_lists(1 / (k + rank(d)))
+
+    Args:
+        result_lists: List of result lists, each containing RetrievalResult objects
+        k: Constant for RRF formula (default: 60)
+
+    Returns:
+        Merged list of MergedCandidate objects, sorted by RRF score
+
+    Example:
+        semantic_results = [RetrievalResult(...), RetrievalResult(...), ...]
+        bm25_results = [RetrievalResult(...), RetrievalResult(...), ...]
+        graph_results = [RetrievalResult(...), RetrievalResult(...), ...]
+
+        merged = reciprocal_rank_fusion([semantic_results, bm25_results, graph_results])
+        # Returns: [MergedCandidate(...), MergedCandidate(...), ...]
+    """
+    # Track scores from each list
+    rrf_scores = {}
+    source_ranks = {}  # Track rank from each source for each doc_id
+    all_retrievals = {}  # Store the actual RetrievalResult (use first occurrence)
+
+    source_names = ["semantic", "bm25", "graph", "temporal"]
+
+    for source_idx, results in enumerate(result_lists):
+        source_name = source_names[source_idx] if source_idx < len(source_names) else f"source_{source_idx}"
+
+        for rank, retrieval in enumerate(results, start=1):
+            # Type check to catch tuple issues
+            if isinstance(retrieval, tuple):
+                raise TypeError(
+                    f"Expected RetrievalResult but got tuple in {source_name} results at rank {rank}. "
+                    f"Tuple value: {retrieval[:2] if len(retrieval) >= 2 else retrieval}. "
+                    f"This suggests the retrieval function returned tuples instead of RetrievalResult objects."
+                )
+            if not isinstance(retrieval, RetrievalResult):
+                raise TypeError(
+                    f"Expected RetrievalResult but got {type(retrieval).__name__} in {source_name} results at rank {rank}"
+                )
+            doc_id = retrieval.id
+
+            # Store retrieval result (use first occurrence)
+            if doc_id not in all_retrievals:
+                all_retrievals[doc_id] = retrieval
+
+            # Calculate RRF score contribution
+            if doc_id not in rrf_scores:
+                rrf_scores[doc_id] = 0.0
+                source_ranks[doc_id] = {}
+
+            rrf_scores[doc_id] += 1.0 / (k + rank)
+            source_ranks[doc_id][f"{source_name}_rank"] = rank
+
+    # Combine into final results with metadata
+    merged_results = []
+    for rrf_rank, (doc_id, rrf_score) in enumerate(
+        sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True), start=1
+    ):
+        merged_candidate = MergedCandidate(
+            retrieval=all_retrievals[doc_id],
+            rrf_score=rrf_score,
+            rrf_rank=rrf_rank,
+            source_ranks=source_ranks[doc_id]
+        )
+        merged_results.append(merged_candidate)
+
+    return merged_results
+
+
+def normalize_scores_on_deltas(
+    results: List[Dict[str, Any]],
+    score_keys: List[str]
+) -> List[Dict[str, Any]]:
+    """
+    Normalize scores based on deltas (min-max normalization within result set).
+
+    This ensures all scores are in [0, 1] range based on the spread in THIS result set.
+
+    Args:
+        results: List of result dicts
+        score_keys: Keys to normalize (e.g., ["recency", "frequency"])
+
+    Returns:
+        Results with normalized scores added as "{key}_normalized"
+    """
+    for key in score_keys:
+        values = [r.get(key, 0.0) for r in results if key in r]
+
+        if not values:
+            continue
+
+        min_val = min(values)
+        max_val = max(values)
+        delta = max_val - min_val
+
+        if delta > 0:
+            for r in results:
+                if key in r:
+                    r[f"{key}_normalized"] = (r[key] - min_val) / delta
+        else:
+            # All values are the same, set to 0.5
+            for r in results:
+                if key in r:
+                    r[f"{key}_normalized"] = 0.5
+
+    return results

hindsight_api/engine/search/observation_utils.py

@@ -0,0 +1,132 @@
+"""
+Observation utilities for generating entity observations from facts.
+
+Observations are objective facts synthesized from multiple memory facts
+about an entity, without personality influence.
+"""
+
+import logging
+from typing import List, Dict, Any
+from pydantic import BaseModel, Field
+
+from ..response_models import MemoryFact
+
+logger = logging.getLogger(__name__)
+
+
+class Observation(BaseModel):
+    """An observation about an entity."""
+    observation: str = Field(description="The observation text - a factual statement about the entity")
+
+
+class ObservationExtractionResponse(BaseModel):
+    """Response containing extracted observations."""
+    observations: List[Observation] = Field(
+        default_factory=list,
+        description="List of observations about the entity"
+    )
+
+
+def format_facts_for_observation_prompt(facts: List[MemoryFact]) -> str:
+    """Format facts as text for observation extraction prompt."""
+    import json
+
+    if not facts:
+        return "[]"
+    formatted = []
+    for fact in facts:
+        fact_obj = {
+            "text": fact.text
+        }
+
+        # Add context if available
+        if fact.context:
+            fact_obj["context"] = fact.context
+
+        # Add occurred_start if available
+        if fact.occurred_start:
+            fact_obj["occurred_at"] = fact.occurred_start
+
+        formatted.append(fact_obj)
+
+    return json.dumps(formatted, indent=2)
+
+
+def build_observation_prompt(
+    entity_name: str,
+    facts_text: str,
+) -> str:
+    """Build the observation extraction prompt for the LLM."""
+    return f"""Based on the following facts about "{entity_name}", generate a list of key observations.
+
+FACTS ABOUT {entity_name.upper()}:
+{facts_text}
+
+Your task: Synthesize the facts into clear, objective observations about {entity_name}.
+
+GUIDELINES:
+1. Each observation should be a factual statement about {entity_name}
+2. Combine related facts into single observations where appropriate
+3. Be objective - do not add opinions, judgments, or interpretations
+4. Focus on what we KNOW about {entity_name}, not what we assume
+5. Include observations about: identity, characteristics, roles, relationships, activities
+6. Write in third person (e.g., "John is..." not "I think John is...")
+7. If there are conflicting facts, note the most recent or most supported one
+
+EXAMPLES of good observations:
+- "John works at Google as a software engineer"
+- "John is detail-oriented and methodical in his approach"
+- "John collaborates frequently with Sarah on the AI project"
+- "John joined the company in 2023"
+
+EXAMPLES of bad observations (avoid these):
+- "John seems like a good person" (opinion/judgment)
+- "John probably likes his job" (assumption)
+- "I believe John is reliable" (first-person opinion)
+
+Generate 3-7 observations based on the available facts. If there are very few facts, generate fewer observations."""
+
+
+def get_observation_system_message() -> str:
+    """Get the system message for observation extraction."""
+    return "You are an objective observer synthesizing facts about an entity. Generate clear, factual observations without opinions or personality influence. Be concise and accurate."
+
+
+async def extract_observations_from_facts(
+    llm_config,
+    entity_name: str,
+    facts: List[MemoryFact]
+) -> List[str]:
+    """
+    Extract observations from facts about an entity using LLM.
+
+    Args:
+        llm_config: LLM configuration to use
+        entity_name: Name of the entity to generate observations about
+        facts: List of facts mentioning the entity
+
+    Returns:
+        List of observation strings
+    """
+    if not facts:
+        return []
+
+    facts_text = format_facts_for_observation_prompt(facts)
+    prompt = build_observation_prompt(entity_name, facts_text)
+
+    try:
+        result = await llm_config.call(
+            messages=[
+                {"role": "system", "content": get_observation_system_message()},
+                {"role": "user", "content": prompt}
+            ],
+            response_format=ObservationExtractionResponse,
+            scope="memory_extract_observation"
+        )
+
+        observations = [op.observation for op in result.observations]
+        return observations
+
+    except Exception as e:
+        logger.warning(f"Failed to extract observations for {entity_name}: {str(e)}")
+        return []

hindsight_api/engine/search/reranking.py

@@ -0,0 +1,103 @@
+"""
+Cross-encoder neural reranking for search results.
+"""
+
+from typing import List
+from .types import MergedCandidate, ScoredResult
+
+
+class CrossEncoderReranker:
+    """
+    Neural reranking using a cross-encoder model.
+
+    Uses cross-encoder/ms-marco-MiniLM-L-6-v2 by default:
+    - Fast inference (~80ms for 100 pairs on CPU)
+    - Small model (80MB)
+    - Trained for passage re-ranking
+    """
+
+    def __init__(self, cross_encoder=None):
+        """
+        Initialize cross-encoder reranker.
+
+        Args:
+            cross_encoder: CrossEncoderReranker instance. If None, uses default
+                          SentenceTransformersCrossEncoder with ms-marco-MiniLM-L-6-v2
+                          (loaded lazily for faster startup)
+        """
+        if cross_encoder is None:
+            from hindsight_api.engine.cross_encoder import SentenceTransformersCrossEncoder
+            # Model is loaded lazily - call ensure_loaded() during initialize()
+            cross_encoder = SentenceTransformersCrossEncoder()
+        self.cross_encoder = cross_encoder
+
+    def rerank(
+        self,
+        query: str,
+        candidates: List[MergedCandidate]
+    ) -> List[ScoredResult]:
+        """
+        Rerank candidates using cross-encoder scores.
+
+        Args:
+            query: Search query
+            candidates: Merged candidates from RRF
+
+        Returns:
+            List of ScoredResult objects sorted by cross-encoder score
+        """
+        if not candidates:
+            return []
+
+        # Prepare query-document pairs with date information
+        pairs = []
+        for candidate in candidates:
+            retrieval = candidate.retrieval
+
+            # Use text + context for better ranking
+            doc_text = retrieval.text
+            if retrieval.context:
+                doc_text = f"{retrieval.context}: {doc_text}"
+
+            # Add formatted date information for temporal awareness
+            if retrieval.occurred_start:
+                occurred_start = retrieval.occurred_start
+
+                # Format in two styles for better model understanding
+                # 1. ISO format: YYYY-MM-DD
+                date_iso = occurred_start.strftime("%Y-%m-%d")
+
+                # 2. Human-readable: "June 5, 2022"
+                date_readable = occurred_start.strftime("%B %d, %Y")
+
+                # Prepend date to document text
+                doc_text = f"[Date: {date_readable} ({date_iso})] {doc_text}"
+
+            pairs.append([query, doc_text])
+
+        # Get cross-encoder scores
+        scores = self.cross_encoder.predict(pairs)
+
+        # Normalize scores using sigmoid to [0, 1] range
+        # Cross-encoder returns logits which can be negative
+        import numpy as np
+        def sigmoid(x):
+            return 1 / (1 + np.exp(-x))
+
+        normalized_scores = [sigmoid(score) for score in scores]
+
+        # Create ScoredResult objects with cross-encoder scores
+        scored_results = []
+        for candidate, raw_score, norm_score in zip(candidates, scores, normalized_scores):
+            scored_result = ScoredResult(
+                candidate=candidate,
+                cross_encoder_score=float(raw_score),
+                cross_encoder_score_normalized=float(norm_score),
+                weight=float(norm_score)  # Initial weight is just cross-encoder score
+            )
+            scored_results.append(scored_result)
+
+        # Sort by cross-encoder score
+        scored_results.sort(key=lambda x: x.weight, reverse=True)
+
+        return scored_results