wisent 0.1.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

wisent/core/data_loaders/loaders/custom.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+from typing import Any, Iterable
+import logging
+
+from wisent_guard.core.data_loaders.core.atoms import BaseDataLoader, DataLoaderError, LoadDataResult
+from wisent_guard.core.contrastive_pairs.core.set import ContrastivePairSet         
+from wisent_guard.core.contrastive_pairs.core.serialization import load_contrastive_pair_set
+
+__all__ = [
+    "CustomUserDataLoader",
+]
+log = logging.getLogger(__name__)
+
+class CustomUserDataLoader(BaseDataLoader):
+    """
+    Load a ContrastivePairSet from a JSONL file, split into train/test,
+    and optionally cap each split.
+
+    attributes:
+        name: "custom"
+            The unique name of this data loader.
+        description: "Load contrastive pairs from custom JSONL and split."
+            A brief description of this data loader.
+    """
+    name = "custom"
+    description = "Load contrastive pairs from custom JSONL and split."
+
+    @staticmethod
+    def _shuffle_indices(n: int, seed: int | None) -> list[int]:
+        """
+        Generate a shuffled list of indices from 0 to n-1.
+        
+        arguments:
+            n: The number of indices to generate.
+            seed: Optional random seed for reproducibility.
+        
+        returns:
+            A list of shuffled indices.
+        """
+        idx = list(range(n))
+        if seed is None:
+            return idx
+        try:
+            from numpy.random import default_rng  
+        except Exception:
+            import random
+            rnd = random.Random(seed)
+            rnd.shuffle(idx)
+            return idx
+        else:
+            return default_rng(seed).permutation(n).tolist()
+
+    def load(
+        self,
+        path: str,
+        split_ratio: float | None = None,
+        seed: int | None = None,
+        training_limit: int | None = None,
+        testing_limit: int | None = None,
+        **_: Any,
+    ) -> LoadDataResult:
+        """
+        Load contrastive pairs from a JSONL file, split into train/test sets,
+        and optionally limit the number of pairs in each set.
+
+        arguments:
+            path: 
+                Path to the JSONL file containing contrastive pairs.
+            split_ratio:
+                Float in [0.0, 1.0] representing the proportion of data to use for training.
+                Defaults to 0.8 if None.
+            seed:
+                Optional random seed for shuffling the data before splitting.
+            training_limit:
+                Optional maximum number of training pairs to return.
+            testing_limit:
+                Optional maximum number of testing pairs to return.
+            **_:
+                Additional keyword arguments (ignored).
+        returns:
+            LoadDataResult with train/test ContrastivePairSets and metadata.
+
+        raises:
+            DataLoaderError if loading or processing fails.
+        """
+
+        if not path:
+            raise DataLoaderError("'path' is required for custom loader.")
+
+        split = self._effective_split(split_ratio)
+        data: ContrastivePairSet = load_contrastive_pair_set(path)
+        log.info("Loaded custom data: %r", data)
+
+        if not data.pairs:
+            raise DataLoaderError("No contrastive pairs found in the input file.")
+
+        n = len(data.pairs)
+        idx = self._shuffle_indices(n, seed)
+        split_at = int(n * split)
+
+        train_pairs = [data.pairs[i] for i in idx[:split_at]]
+        test_pairs  = [data.pairs[i] for i in idx[split_at:]]
+
+        if training_limit is not None:
+            train_pairs = train_pairs[: max(0, int(training_limit))]
+        if testing_limit is not None:
+            test_pairs = test_pairs[: max(0, int(testing_limit))]
+
+        train_set = ContrastivePairSet(name=f"{data.name}_train", pairs=train_pairs, task_type=data.task_type)
+        test_set  = ContrastivePairSet(name=f"{data.name}_test",  pairs=test_pairs,  task_type=data.task_type)
+
+        train_set.validate()
+        test_set.validate()
+
+        return LoadDataResult( 
+            train_qa_pairs=train_set,
+            test_qa_pairs=test_set,
+            task_type=data.task_type or "custom",
+            lm_task_data=None,
+        )

wisent/core/data_loaders/loaders/lm_loader.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+from typing import Any, TYPE_CHECKING
+import logging
+
+from wisent_guard.core.data_loaders.core.atoms import BaseDataLoader, DataLoaderError, LoadDataResult
+from wisent_guard.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent_guard.core.contrastive_pairs.core.set import ContrastivePairSet
+from lm_eval.tasks import get_task_dict
+from lm_eval.tasks import TaskManager as LMTaskManager
+from wisent_guard.core.contrastive_pairs.lm_eval_pairs.lm_task_pairs_generation import (
+    lm_build_contrastive_pairs,
+)
+
+if TYPE_CHECKING:
+    from lm_eval.api.task import ConfigurableTask
+
+__all__ = [
+    "LMEvalDataLoader",
+]
+
+log = logging.getLogger(__name__)
+
+class LMEvalDataLoader(BaseDataLoader):
+    """
+    Load contrastive pairs from a single lm-evaluation-harness task via `load_lm_eval_task`,
+    split into train/test, and return a canonical LoadDataResult.
+    """
+    name = "lm_eval"
+    description = "Load from a single lm-eval task."
+
+    def _load_one_task(
+        self,
+        task_name: str,
+        split_ratio: float,
+        seed: int,
+        limit: int | None,
+        training_limit: int | None,
+        testing_limit: int | None,
+    ) -> LoadDataResult:
+        """
+        Load a single lm-eval task by name, convert to contrastive pairs,
+        split into train/test, and return a LoadDataResult.
+        
+        arguments:
+            task_name: The name of the lm-eval task to load.
+            split_ratio: The fraction of data to use for training (between 0 and 1).
+            seed: Random seed for shuffling/splitting.
+            limit: Optional limit on total number of pairs to load.
+            training_limit: Optional limit on number of training pairs.
+            testing_limit: Optional limit on number of testing pairs.
+            
+        returns:
+            A LoadDataResult containing train/test pairs and task info.
+            
+        raises:
+            DataLoaderError if the task cannot be found or if splits are empty.
+            ValueError if split_ratio is not in [0.0, 1.0].
+            NotImplementedError if load_lm_eval_task is not implemented.
+        
+        note:
+            This loader only supports single tasks, not mixtures. To load mixtures,
+            use a custom data loader or extend this one."""
+        loaded = self.load_lm_eval_task(task_name)
+
+        if isinstance(loaded, dict):
+            if len(loaded) != 1:
+                keys = ", ".join(sorted(loaded.keys()))
+                raise DataLoaderError(
+                    f"Task '{task_name}' returned {len(loaded)} subtasks ({keys}). "
+                    "Specify an explicit subtask, e.g. 'benchmark/subtask'."
+                )
+            (subname, task_obj), = loaded.items()
+            pairs_task_name = subname
+        else:
+            task_obj = loaded
+            pairs_task_name = task_name
+
+        pairs = lm_build_contrastive_pairs(
+            task_name=pairs_task_name,
+            lm_eval_task=task_obj,
+            limit=limit,
+        )
+
+        train_pairs, test_pairs = self._split_pairs(
+            pairs, split_ratio, seed, training_limit, testing_limit
+        )
+
+        if not train_pairs or not test_pairs:
+            raise DataLoaderError("One of the splits is empty after splitting.")
+
+        train_set = ContrastivePairSet("lm_eval_train", train_pairs, task_type=task_name)
+        test_set = ContrastivePairSet("lm_eval_test", test_pairs, task_type=task_name)
+
+        train_set.validate()
+        test_set.validate()
+
+        return LoadDataResult(
+            train_qa_pairs=train_set,
+            test_qa_pairs=test_set,
+            task_type=task_name,
+            lm_task_data=task_obj,
+        )
+
+    def load(
+        self,
+        task: str,  
+        split_ratio: float | None = None,
+        seed: int = 42,
+        limit: int | None = None,
+        training_limit: int | None = None,
+        testing_limit: int | None = None,
+        **_: Any,
+    ) -> LoadDataResult:
+        """
+        Load contrastive pairs from a single lm-eval-harness task, split into train/test sets.
+        arguments:
+            task:
+                The name of the lm-eval task to load (e.g., "winogrande", "hellaswag").
+                Must be a single task, not a mixture.
+            split_ratio:
+                Float in [0.0, 1.0] representing the proportion of data to use for training.
+                Defaults to 0.8 if None.
+            seed:
+                Random seed for shuffling the data before splitting.
+            limit:
+                Optional maximum number of total pairs to load from the task.
+            training_limit:
+                Optional maximum number of training pairs to return.
+            testing_limit:
+                Optional maximum number of testing pairs to return.
+            **_:
+                Additional keyword arguments (ignored).
+        
+        returns:
+            LoadDataResult with train/test ContrastivePairSets and metadata.
+        
+        raises:
+            DataLoaderError if loading or processing fails.
+            ValueError if split_ratio is not in [0.0, 1.0].
+            NotImplementedError if load_lm_eval_task is not implemented.
+        """
+        split = self._effective_split(split_ratio)
+
+        # Single-task path only
+        return self._load_one_task(
+            task_name=str(task),
+            split_ratio=split,
+            seed=seed,
+            limit=limit,
+            training_limit=training_limit,
+            testing_limit=testing_limit,
+        )
+
+    @staticmethod
+    def load_lm_eval_task(task_name: str) -> ConfigurableTask | dict[str, ConfigurableTask]:
+        """
+        Load a single lm-eval-harness task by name.
+
+        arguments:
+            task_name: The name of the lm-eval task to load.
+        
+        returns:
+            A ConfigurableTask instance or a dict of subtask name to ConfigurableTask.
+        
+        raises:
+            DataLoaderError if the task cannot be found.
+        """
+        task_manager = LMTaskManager()
+        task_manager.initialize_tasks()
+
+        task_dict = get_task_dict([task_name], task_manager=task_manager)
+        if task_name in task_dict:
+            return task_dict[task_name]
+        raise DataLoaderError(f"lm-eval task '{task_name}' not found.")
+    
+    def _split_pairs(
+        self,
+        pairs: list[ContrastivePair],
+        split_ratio: float,
+        seed: int,
+        training_limit: int | None,
+        testing_limit: int | None,
+    ) -> tuple[list[ContrastivePair], list[ContrastivePair]]:
+        """
+        Split a list of ContrastivePairs into train/test sets.
+
+        arguments:
+            pairs: List of ContrastivePair to split.
+            split_ratio: Float in [0.0, 1.0] for the training set proportion.
+            seed: Random seed for shuffling.
+            training_limit: Optional max number of training pairs.
+            testing_limit: Optional max number of testing pairs.
+        
+        returns:
+            A tuple of (train_pairs, test_pairs).
+        raises:
+            ValueError if split_ratio is not in [0.0, 1.0].
+        """
+        if not pairs:
+            return [], []
+        from numpy.random import default_rng
+
+        idx = list(range(len(pairs)))
+        default_rng(seed).shuffle(idx)
+        cut = int(len(pairs) * split_ratio)
+        train_idx = set(idx[:cut])
+
+        train_pairs: list[ContrastivePair] = []
+        test_pairs: list[ContrastivePair] = []
+        for i in idx:
+            (train_pairs if i in train_idx else test_pairs).append(pairs[i])
+
+        if training_limit and training_limit > 0:
+            train_pairs = train_pairs[:training_limit]
+        if testing_limit and testing_limit > 0:
+            test_pairs = test_pairs[:testing_limit]
+
+        return train_pairs, test_pairs

wisent/core/detection_handling.py

@@ -0,0 +1,257 @@
+"""
+Detection handling module for wisent-guard.
+
+This module provides different strategies for handling responses that have been
+detected as problematic (hallucinations, harmful content, bias, etc.).
+"""
+
+from enum import Enum
+from typing import Optional, Callable, Dict, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DetectionAction(Enum):
+    """Actions to take when problematic content is detected."""
+    PASS_THROUGH = "pass_through"  # Output the response as-is
+    REPLACE_WITH_PLACEHOLDER = "replace_with_placeholder"  # Replace with safe message
+    REGENERATE_UNTIL_SAFE = "regenerate_until_safe"  # Keep regenerating until safe
+
+
+class DetectionHandler:
+    """
+    Handles responses when problematic content is detected.
+    
+    Provides configurable strategies for dealing with detected issues like
+    hallucinations, harmful content, bias, etc.
+    """
+    
+    def __init__(
+        self,
+        action: DetectionAction = DetectionAction.REPLACE_WITH_PLACEHOLDER,
+        placeholder_message: Optional[str] = None,
+        max_regeneration_attempts: int = 3,
+        custom_placeholder_generator: Optional[Callable[[str, str], str]] = None,
+        log_detections: bool = True
+    ):
+        """
+        Initialize the detection handler.
+        
+        Args:
+            action: What action to take when detection occurs
+            placeholder_message: Custom placeholder message (if None, uses default)
+            max_regeneration_attempts: Maximum times to regenerate before giving up
+            custom_placeholder_generator: Function to generate custom placeholders
+            log_detections: Whether to log detection events
+        """
+        self.action = action
+        self.placeholder_message = placeholder_message
+        self.max_regeneration_attempts = max_regeneration_attempts
+        self.custom_placeholder_generator = custom_placeholder_generator
+        self.log_detections = log_detections
+        
+        # Default placeholder messages for different detection types
+        self.default_placeholders = {
+            "hallucination": "I apologize, but I may not have accurate information about this topic. Please verify any factual claims from reliable sources.",
+            "harmful_content": "I cannot provide information that could be harmful or dangerous. Please ask about something else I can help with safely.",
+            "bias": "I want to avoid potentially biased responses. Let me try to provide a more balanced perspective on this topic.",
+            "personal_info": "I cannot generate or discuss personal information. Please ask about general topics instead.",
+            "scheming": "I cannot provide advice on deceptive or manipulative behavior. Let me help you with ethical approaches instead.",
+            "bad_code": "I cannot provide code examples that may contain security vulnerabilities. Let me suggest secure coding practices instead.",
+            "default": "I apologize, but I cannot provide an appropriate response to this request. Please try rephrasing your question."
+        }
+    
+    def handle_detection(
+        self,
+        original_response: str,
+        detection_type: str,
+        confidence_score: float,
+        original_prompt: str,
+        regenerate_function: Optional[Callable[[], str]] = None
+    ) -> str:
+        """
+        Handle a detected problematic response based on the configured action.
+        
+        Args:
+            original_response: The response that was flagged
+            detection_type: Type of issue detected (e.g., "hallucination", "bias")
+            confidence_score: Confidence score of the detection (0.0 to 1.0)
+            original_prompt: The original prompt that generated the response
+            regenerate_function: Function to call for regeneration (if needed)
+            
+        Returns:
+            The final response to return to the user
+        """
+        if self.log_detections:
+            logger.warning(
+                f"Detected {detection_type} with confidence {confidence_score:.3f} "
+                f"in response: {original_response[:100]}..."
+            )
+        
+        if self.action == DetectionAction.PASS_THROUGH:
+            return self._handle_pass_through(original_response, detection_type, confidence_score)
+            
+        elif self.action == DetectionAction.REPLACE_WITH_PLACEHOLDER:
+            return self._handle_replacement(original_response, detection_type, original_prompt)
+            
+        elif self.action == DetectionAction.REGENERATE_UNTIL_SAFE:
+            return self._handle_regeneration(
+                original_response, detection_type, original_prompt, regenerate_function
+            )
+        
+        else:
+            raise ValueError(f"Unknown detection action: {self.action}")
+    
+    def _handle_pass_through(
+        self, 
+        original_response: str, 
+        detection_type: str, 
+        confidence_score: float
+    ) -> str:
+        """Handle pass-through action - return response as-is with optional warning."""
+        if self.log_detections:
+            logger.info(f"Passing through response despite {detection_type} detection")
+        
+        # Optionally add a warning prefix (can be configured)
+        return original_response
+    
+    def _handle_replacement(
+        self, 
+        original_response: str, 
+        detection_type: str, 
+        original_prompt: str
+    ) -> str:
+        """Handle replacement action - return placeholder message."""
+        if self.custom_placeholder_generator:
+            return self.custom_placeholder_generator(detection_type, original_prompt)
+        
+        if self.placeholder_message:
+            return self.placeholder_message
+        
+        # Use default placeholder for the detection type
+        return self.default_placeholders.get(detection_type, self.default_placeholders["default"])
+    
+    def _handle_regeneration(
+        self,
+        original_response: str,
+        detection_type: str,
+        original_prompt: str,
+        regenerate_function: Optional[Callable[[], str]]
+    ) -> str:
+        """Handle regeneration action - keep generating until safe response."""
+        if not regenerate_function:
+            logger.warning("No regeneration function provided, falling back to placeholder")
+            return self._handle_replacement(original_response, detection_type, original_prompt)
+        
+        attempts = 0
+        current_response = original_response
+        
+        while attempts < self.max_regeneration_attempts:
+            attempts += 1
+            
+            if self.log_detections:
+                logger.info(f"Regeneration attempt {attempts}/{self.max_regeneration_attempts}")
+            
+            try:
+                # Generate a new response
+                new_response = regenerate_function()
+                
+                # Note: In a real implementation, you would re-run the detection here
+                # For now, we'll assume the regeneration function handles this
+                return new_response
+                
+            except Exception as e:
+                logger.error(f"Error during regeneration attempt {attempts}: {e}")
+                continue
+        
+        # If we've exhausted attempts, fall back to placeholder
+        if self.log_detections:
+            logger.warning(
+                f"Failed to generate safe response after {self.max_regeneration_attempts} attempts, "
+                f"using placeholder"
+            )
+        
+        return self._handle_replacement(original_response, detection_type, original_prompt)
+    
+    def set_custom_placeholder(self, detection_type: str, message: str):
+        """Set a custom placeholder message for a specific detection type."""
+        self.default_placeholders[detection_type] = message
+    
+    def get_detection_stats(self) -> Dict[str, Any]:
+        """Get statistics about detection handling (placeholder for future implementation)."""
+        return {
+            "action": self.action.value,
+            "max_regeneration_attempts": self.max_regeneration_attempts,
+            "available_placeholders": list(self.default_placeholders.keys())
+        }
+
+
+# Convenience functions for common use cases
+
+def create_pass_through_handler() -> DetectionHandler:
+    """Create a handler that passes through all responses unchanged."""
+    return DetectionHandler(action=DetectionAction.PASS_THROUGH)
+
+
+def create_placeholder_handler(custom_message: Optional[str] = None) -> DetectionHandler:
+    """Create a handler that replaces detected responses with placeholders."""
+    return DetectionHandler(
+        action=DetectionAction.REPLACE_WITH_PLACEHOLDER,
+        placeholder_message=custom_message
+    )
+
+
+def create_regeneration_handler(max_attempts: int = 3) -> DetectionHandler:
+    """Create a handler that regenerates responses until they're safe."""
+    return DetectionHandler(
+        action=DetectionAction.REGENERATE_UNTIL_SAFE,
+        max_regeneration_attempts=max_attempts
+    )
+
+
+def create_custom_handler(
+    placeholder_generator: Callable[[str, str], str],
+    action: DetectionAction = DetectionAction.REPLACE_WITH_PLACEHOLDER
+) -> DetectionHandler:
+    """Create a handler with a custom placeholder generator function."""
+    return DetectionHandler(
+        action=action,
+        custom_placeholder_generator=placeholder_generator
+    )
+
+
+# Example custom placeholder generators
+
+def educational_placeholder_generator(detection_type: str, original_prompt: str) -> str:
+    """Generate educational placeholders that explain why content was flagged."""
+    explanations = {
+        "hallucination": f"The response to '{original_prompt}' may contain inaccurate information. "
+                        "Please verify facts from reliable sources before relying on this information.",
+        "harmful_content": f"I cannot provide a response to '{original_prompt}' as it may involve "
+                          "harmful or dangerous content. Please ask about safer topics.",
+        "bias": f"The response to '{original_prompt}' might contain biased perspectives. "
+               "Consider seeking multiple viewpoints on this topic.",
+        "personal_info": f"I cannot respond to '{original_prompt}' as it involves personal information. "
+                        "Please ask about general topics instead."
+    }
+    
+    return explanations.get(
+        detection_type,
+        f"I cannot provide an appropriate response to '{original_prompt}'. "
+        "Please try rephrasing your question."
+    )
+
+
+def brief_placeholder_generator(detection_type: str, original_prompt: str) -> str:
+    """Generate brief, minimal placeholder messages."""
+    brief_messages = {
+        "hallucination": "Information may be inaccurate.",
+        "harmful_content": "Cannot provide harmful content.",
+        "bias": "Response may be biased.",
+        "personal_info": "Cannot share personal information.",
+        "scheming": "Cannot provide deceptive advice.",
+        "bad_code": "Cannot provide insecure code."
+    }
+    
+    return brief_messages.get(detection_type, "Cannot provide response.")