mirage-benchmark 1.0.4__py3-none-any.whl

mirage/embeddings/rerankers_text.py

@@ -0,0 +1,149 @@
+
+from typing import List, Dict, Any, Tuple
+import logging
+import re
+from mirage.core.llm import call_vlm_with_multiple_images
+from mirage.core.prompts import PROMPTS
+
+class LLMReranker:
+    """
+    Uses an LLM to evaluate, rank, and merge QA pairs.
+    """
+    def __init__(self, model_name="gpt-oss", expert_persona: str = None, 
+                 domain: str = None):
+        self.model_name = model_name
+        self.expert_persona = expert_persona
+        self.domain = domain
+        self.tuple_delimiter = PROMPTS.get("DEFAULT_TUPLE_DELIMITER", "<|#|>")
+        self.completion_delimiter = PROMPTS.get("DEFAULT_COMPLETION_DELIMITER", "<|#|>END<|#|>")
+
+    def _parse_qa_pairs(self, response_text: str) -> List[Dict[str, str]]:
+        """Parses QA pairs from the |#| delimited format."""
+        qa_pairs = []
+        try:
+            # Remove completion delimiter if present
+            if self.completion_delimiter in response_text:
+                response_text = response_text.split(self.completion_delimiter)[0].strip()
+        
+            # Remove START delimiter if present
+            start_delimiter = self.tuple_delimiter + "START" + self.tuple_delimiter
+            if response_text.startswith(start_delimiter):
+                response_text = response_text[len(start_delimiter):].strip()
+            
+            lines = [line.strip() for line in response_text.split('\n') if line.strip()]
+            
+            for line in lines:
+                # Skip NEXT lines
+                next_delimiter = self.tuple_delimiter + "NEXT" + self.tuple_delimiter
+                if line == next_delimiter:
+                    continue
+                
+                # Check for Question delimiter pattern
+                if line.startswith("Question" + self.tuple_delimiter) or line.startswith("question" + self.tuple_delimiter):
+                    parts = line.split(self.tuple_delimiter)
+                    # Expected: Question<|#|>Q<|#|>Answer<|#|>A...
+                    if len(parts) >= 4 and parts[0].lower() == "question" and parts[2].lower() == "answer":
+                        question = parts[1].strip()
+                        answer = parts[3].strip()
+                        if question and answer:
+                            qa_pairs.append({"question": question, "answer": answer})
+                            
+            return qa_pairs
+        except Exception as e:
+            logging.error(f"Error parsing QA pairs: {e}")
+            return []
+
+    def rank_cluster(self, cluster_candidates: List[Dict]) -> List[Dict]:
+        """
+        Step 3: Order/Rank the QA pairs in the cluster.
+        """
+        if not cluster_candidates or len(cluster_candidates) < 2:
+            return cluster_candidates
+
+        candidates_text = ""
+        for idx, item in enumerate(cluster_candidates, 1):
+            candidates_text += f"\n--- Candidate {idx} ---\n"
+            candidates_text += f"Question: {item.get('question', '')}\n"
+            candidates_text += f"Answer: {item.get('answer', '')}\n"
+
+        prompt_template = PROMPTS.get("deduplication_rank", "")
+        if not prompt_template:
+            logging.warning("deduplication_rank prompt not found.")
+            return cluster_candidates
+
+        prompt = prompt_template.format(
+            candidates_text=candidates_text,
+            expert_persona=self.expert_persona,
+            domain=self.domain
+        )
+        
+        try:
+            response = call_vlm_with_multiple_images(prompt, [])
+            ordered_pairs = self._parse_qa_pairs(response)
+            
+            if not ordered_pairs:
+                logging.warning("LLM returned empty ranking, using original order.")
+                return cluster_candidates
+                
+            # Attach metadata from original if possible (heuristic matching) or just return new order
+            # Since we are just reordering, we can try to match back to original items to keep metadata,
+            # but for deduplication, text is primary.
+            # For simplicity, we return the parsed pairs as the ordered list.
+            return ordered_pairs
+            
+        except Exception as e:
+            logging.error(f"Error in LLM ranking: {e}")
+            return cluster_candidates
+
+    def deduplicate_and_merge(self, cluster_candidates: List[Dict]) -> List[Dict]:
+        """
+        Step 5: Deduplicate and merge based on the ordered cluster.
+        """
+        if not cluster_candidates:
+            return []
+        
+        # First, Rank/Order them
+        ordered_candidates = self.rank_cluster(cluster_candidates)
+        
+        # Prepare text for merge prompt
+        candidates_text = ""
+        for idx, item in enumerate(ordered_candidates, 1):
+            candidates_text += f"\n--- Candidate {idx} ---\n"
+            candidates_text += f"Question: {item.get('question', '')}\n"
+            candidates_text += f"Answer: {item.get('answer', '')}\n"
+
+        prompt_template = PROMPTS.get("deduplication_merge", "")
+        if not prompt_template:
+             logging.warning("deduplication_merge prompt not found.")
+             return cluster_candidates[:1] # Fallback
+
+        prompt = prompt_template.format(
+            candidates_text=candidates_text,
+            expert_persona=self.expert_persona,
+            domain=self.domain
+        )
+
+        try:
+            response = call_vlm_with_multiple_images(prompt, [])
+            merged_pairs = self._parse_qa_pairs(response)
+            
+            if not merged_pairs:
+                logging.warning("LLM returned empty merge, returning first candidate.")
+                return cluster_candidates[:1]
+            
+            # Propagate metadata from the first original candidate to all new pairs
+            # (approximate, since we might have merged multiple)
+            base_metadata = cluster_candidates[0].copy()
+            final_results = []
+            for pair in merged_pairs:
+                new_item = base_metadata.copy()
+                new_item["question"] = pair["question"]
+                new_item["answer"] = pair["answer"]
+                new_item["merged_from_count"] = len(cluster_candidates)
+                final_results.append(new_item)
+                
+            return final_results
+
+        except Exception as e:
+            logging.error(f"Error in LLM deduplication: {e}")
+            return cluster_candidates[:1]

mirage/evaluation/__init__.py

@@ -0,0 +1,26 @@
+"""
+Evaluation module for MiRAGE - Metrics and dataset evaluation.
+
+Imports are lazy to avoid loading optional dependencies at import time.
+"""
+
+_LAZY_IMPORTS = {
+    # Basic metrics
+    "evaluate_qa_dataset": ("metrics", "evaluate_qa_dataset"),
+    "compute_faithfulness": ("metrics", "compute_faithfulness"),
+    "compute_relevancy": ("metrics", "compute_relevancy"),
+    # Optimized evaluation
+    "OptimizedEvaluator": ("metrics_optimized", "OptimizedEvaluator"),
+    "evaluate_subset": ("metrics_optimized", "evaluate_subset"),
+    "generate_evaluation_report": ("metrics_optimized", "generate_evaluation_report"),
+}
+
+
+def __getattr__(name):
+    """Lazy import to avoid loading optional dependencies at import time."""
+    if name in _LAZY_IMPORTS:
+        module_name, attr_name = _LAZY_IMPORTS[name]
+        import importlib
+        module = importlib.import_module(f"mirage.evaluation.{module_name}")
+        return getattr(module, attr_name)
+    raise AttributeError(f"module 'mirage.evaluation' has no attribute '{name}'")