wisent 0.7.379__py3-none-any.whl

wisent/core/synthetic/generators/pairs_generator.py

@@ -0,0 +1,313 @@
+from __future__ import annotations
+
+import logging
+
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair
+from wisent.core.contrastive_pairs.core.response import PositiveResponse, NegativeResponse
+from wisent.core.contrastive_pairs.core.set import ContrastivePairSet
+
+from wisent.core.models.wisent_model import WisentModel
+from wisent.core.synthetic.db_instructions.core.atoms import DB_Instructions
+
+from wisent.core.synthetic.generators.core.atoms import GenerationReport
+
+from wisent.core.synthetic.generators.diversities.core.core import Diversity
+
+from wisent.core.synthetic.cleaners.pairs_cleaner import PairsCleaner
+
+__all__ = [
+    "SyntheticContrastivePairsGenerator",
+]
+
+logger = logging.getLogger(__name__)
+
+class SyntheticContrastivePairsGenerator:
+    """Small, fast contrastive-pairs generator with an extensible cleaning pipeline."""
+
+    def __init__(
+        self,
+        model: WisentModel,
+        generation_config: dict[str, int | float | str],
+        contrastive_set_name: str,
+        trait_description: str,
+        trait_label: str,
+        db_instructions: DB_Instructions,
+        cleaner: PairsCleaner,
+        diversity: Diversity,
+        nonsense_mode: str | None = None,
+    ) -> None:
+        self.model = model
+        self.db_instructions = db_instructions
+        self.generation_config = generation_config
+        self.cleaner = cleaner
+        self.diversity = diversity
+
+        self.contrastive_set_name = contrastive_set_name
+        self.trait_description = trait_description
+        self.trait_label = trait_label
+        self.nonsense_mode = nonsense_mode
+
+
+    def generate(
+        self,
+        num_pairs: int = 10,
+    ) -> tuple[ContrastivePairSet, GenerationReport]:
+        """
+        Generate synthetic contrastive pairs for the given topic and trait.
+
+        Generates pairs one at a time:
+        1. Generate a prompt/scenario
+        2. Generate a positive response (exhibits the trait)
+        3. Generate a negative response (does NOT exhibit the trait)
+
+        arguments:
+            num_pairs:
+                Number of contrastive pairs to generate (default: 10).
+
+        returns:
+            Tuple of ContrastivePairSet with the generated pairs and GenerationReport with statistics about the generation
+        """
+        parsed = ContrastivePairSet(
+            name=self.contrastive_set_name,
+            task_type=self.trait_label,
+        )
+
+        # Generate opposite trait description once
+        opposite_instruction = (
+            f"What is the OPPOSITE personality trait of: {self.trait_description}?\n\n"
+            f"Describe the opposite in one sentence, be specific about what words/style/tone to use."
+        )
+        opposite_raw = self.model.generate(
+            inputs=[[{"role": "user", "content": opposite_instruction}]],
+            **self.generation_config,
+        )
+        opposite_trait = opposite_raw[0].strip() if opposite_raw else "neutral and plain"
+        logger.info(f"[GENERATE] Opposite trait: {opposite_trait}")
+
+        # Generate pairs one at a time, retry until we have num_pairs
+        max_attempts = num_pairs * 3  # Prevent infinite loops
+        attempts = 0
+
+        while len(parsed) < num_pairs and attempts < max_attempts:
+            attempts += 1
+            logger.info(f"[GENERATE] Generating pair {len(parsed)+1}/{num_pairs} (attempt {attempts})")
+
+            # 1) Generate a prompt/scenario - simple question format
+            prompt_instruction = (
+                f"Write one short question a user might ask. Example: 'What is your favorite hobby?' "
+                f"Just the question, nothing else."
+            )
+            prompt_raw = self.model.generate(
+                inputs=[[{"role": "user", "content": prompt_instruction}]],
+                **self.generation_config,
+            )
+            prompt = prompt_raw[0].strip() if prompt_raw else ""
+
+            if not prompt:
+                logger.warning(f"[GENERATE] Failed to generate prompt, retrying...")
+                continue
+
+            logger.info(f"[GENERATE] Prompt: {prompt[:100]}")
+
+            # 2) Generate positive response (exhibits the trait)
+            positive_instruction = (
+                f"Question: {prompt}\n\n"
+                f"Answer the question AS IF you have this personality: {self.trait_description}\n\n"
+                f"Write 1-2 sentences showing this personality clearly. Just the answer."
+            )
+            positive_raw = self.model.generate(
+                inputs=[[{"role": "user", "content": positive_instruction}]],
+                **self.generation_config,
+            )
+            positive = positive_raw[0].strip() if positive_raw else ""
+
+            if not positive:
+                logger.warning(f"[GENERATE] Failed to generate positive, retrying...")
+                continue
+
+            logger.info(f"[GENERATE] Positive: {positive[:100]}")
+
+            # 3) Generate negative response - using the opposite trait
+            negative_instruction = (
+                f"Question: {prompt}\n\n"
+                f"Answer the question AS IF you have this personality: {opposite_trait}\n\n"
+                f"Write 1-2 sentences showing this personality clearly. Just the answer."
+            )
+            negative_raw = self.model.generate(
+                inputs=[[{"role": "user", "content": negative_instruction}]],
+                **self.generation_config,
+            )
+            negative = negative_raw[0].strip() if negative_raw else ""
+
+            if not negative:
+                logger.warning(f"[GENERATE] Failed to generate negative, retrying...")
+                continue
+
+            logger.info(f"[GENERATE] Negative: {negative[:100]}")
+
+            # Create the pair
+            cp = ContrastivePair(
+                prompt=prompt,
+                positive_response=PositiveResponse(model_response=positive),
+                negative_response=NegativeResponse(model_response=negative),
+                label=self.trait_label,
+                trait_description=self.trait_description,
+            )
+            parsed.add(cp)
+            logger.info(f"[GENERATE] Successfully added pair {len(parsed)}/{num_pairs}")
+
+        logger.info(f"[GENERATE] Generated {len(parsed)} pairs after {attempts} attempts")
+
+        # Clean (dedupe, refusal check, etc.)
+        cleaned, stats = self.cleaner.clean(parsed)
+
+        refusaler_stats = stats.step_stats.get("refusaler_cleaner")
+        retries = refusaler_stats.modified_items if refusaler_stats else 0
+
+        # Build final domain objects
+        cps = ContrastivePairSet(name=self.contrastive_set_name, task_type=self.trait_label)
+        for item in cleaned.pairs:
+            cps.add(
+                ContrastivePair(
+                    prompt=item.prompt,
+                    positive_response=PositiveResponse(model_response=item.positive_response.model_response),
+                    negative_response=NegativeResponse(model_response=item.negative_response.model_response),
+                    label=item.label or self.trait_label,
+                    trait_description=item.trait_description or self.trait_description,
+                )
+            )
+
+        # Diversity summary (prompts only)
+        prompts = [it.prompt for it in cleaned.pairs]
+        div = self.diversity.compute(prompts)
+
+        report = GenerationReport(
+            requested=num_pairs,
+            kept_after_dedupe=len(cleaned),
+            retries_for_refusals=retries,
+            diversity=div,
+        )
+        return cps, report
+
+    def parse_pairs(self, raw: list[str]) -> ContrastivePairSet:
+        """
+        Parse raw model outputs into ContrastivePairSet objects.
+        Expects simple text format with ---PAIR--- markers, not JSON.
+
+        arguments:
+            raw:
+                Raw model output string to parse.
+        returns:
+            ContrastivePairSet object parsed from the raw string.
+        """
+        import re
+
+        out: ContrastivePairSet = ContrastivePairSet(
+            name=self.contrastive_set_name,
+            task_type=self.trait_label,
+        )
+
+        logger.info(f"[PARSE DEBUG] Received {len(raw)} raw outputs to parse")
+
+        for idx, r in enumerate(raw):
+            logger.info(f"[PARSE DEBUG] Raw output {idx}:\n{r[:500]}")
+
+            # Split by ---PAIR--- markers (flexible with extra dashes)
+            pair_blocks = re.split(r'-+PAIR-+', r)
+
+            for block_idx, block in enumerate(pair_blocks):
+                if not block.strip():
+                    continue
+
+                # Remove ---END--- marker if present (flexible with extra dashes/spaces)
+                block = re.sub(r'-+END-+', '', block).strip()
+
+                if not block:
+                    continue
+
+                logger.info(f"[PARSE DEBUG] Processing block {block_idx}:\n{block[:200]}")
+
+                # Extract ALL occurrences - model generates ANY labels, not just PROMPT/POSITIVE/NEGATIVE
+                # Look for pattern: LABEL1: text1  LABEL2: text2  LABEL3: text3
+                # We assume first is prompt, second is positive, third is negative
+                lines = block.strip().split('\n')
+
+                prompts = []
+                positives = []
+                negatives = []
+
+                current_group = []
+                for line in lines:
+                    line = line.strip()
+                    if not line or line.startswith('---'):
+                        continue
+
+                    # Check if line contains colon (flexible parsing for any LABEL: or LABEL . or LABEL : format)
+                    # Model outputs variations like "POSITIVE:", "POSITIVE :", "POSITIVE.", "_PROMPT:", " PROMPT:", etc.
+                    if ':' in line or '.' in line:
+                        # Find the first colon or period as separator
+                        sep_idx = -1
+                        for sep in [':', '.']:
+                            idx = line.find(sep)
+                            if idx > 0:  # Must have at least one char before separator
+                                if sep_idx == -1 or idx < sep_idx:
+                                    sep_idx = idx
+
+                        if sep_idx > 0:
+                            # Extract text after separator
+                            text = line[sep_idx + 1:].strip()
+                            if text:
+                                current_group.append(text)
+
+                                # When we have 3 items, that's a complete pair
+                                if len(current_group) == 3:
+                                    prompts.append(current_group[0])
+                                    positives.append(current_group[1])
+                                    negatives.append(current_group[2])
+                                    current_group = []
+
+                # All three lists should have same length
+                if not (prompts and positives and negatives and len(prompts) == len(positives) == len(negatives)):
+                    logger.warning(f"[PARSE DEBUG] Mismatched or missing fields in block {block_idx}")
+                    continue
+
+                # Create a pair for each triple
+                for prompt, positive, negative in zip(prompts, positives, negatives):
+                    prompt = prompt.strip()
+                    positive = positive.strip()
+                    negative = negative.strip()
+
+                    if not (prompt and positive and negative):
+                        logger.warning(f"[PARSE DEBUG] Empty field(s) in triple")
+                        continue
+
+                    logger.info(f"[PARSE DEBUG] Extracted - Prompt: {prompt[:50]}, Positive: {positive[:50]}, Negative: {negative[:50]}")
+
+                    cp = ContrastivePair(
+                        prompt=prompt,
+                        positive_response=PositiveResponse(model_response=positive),
+                        negative_response=NegativeResponse(model_response=negative),
+                        label=self.trait_label,
+                        trait_description=self.trait_description,
+                    )
+                    out.add(cp)
+                    logger.info(f"[PARSE DEBUG] Successfully added pair")
+
+        logger.info(f"[PARSE DEBUG] Finished parsing. Total pairs collected: {len(out)}")
+        return out
+
+    @staticmethod
+    def _build_user_prompt(label: str, desc: str, k: int) -> str:
+        return (
+            f"Create {k} contrastive pairs.\n"
+            f"- Trait label: {label}\n"
+            f"- Trait description: {desc}\n"
+            f"\n"
+            f"Tips:\n"
+            f"- Make prompts specific to the topic but varied in wording and intent.\n"
+            f"- Keep negative examples safe (fictional, non-actionable).\n"
+            f"- Avoid meta-text like 'I cannot' or 'As an AI model'.\n"
+            f"\n"
+            f"Generate {k} pairs now."
+        )

wisent/core/task_interface.py

@@ -0,0 +1,143 @@
+"""
+Task-agnostic interface for benchmark integration.
+
+This module provides a unified interface for integrating different benchmarks
+without depending on lm-evaluation-harness.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type
+
+from .benchmark_extractors import BenchmarkExtractor
+from wisent.core.errors import TaskNotFoundError
+
+
+class TaskInterface(ABC):
+    """Abstract interface for benchmark tasks."""
+
+    @abstractmethod
+    def load_data(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load task data."""
+
+    @abstractmethod
+    def get_extractor(self) -> BenchmarkExtractor:
+        """Get the benchmark extractor for this task."""
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Get the task name."""
+
+    @abstractmethod
+    def get_description(self) -> str:
+        """Get the task description."""
+
+    @abstractmethod
+    def get_categories(self) -> List[str]:
+        """Get the task categories (e.g., ['coding', 'reasoning'])."""
+
+
+class TaskRegistry:
+    """Registry for managing available tasks."""
+
+    def __init__(self):
+        self._tasks: Dict[str, Type[TaskInterface]] = {}
+
+    def register_task(self, name: str, task_class: Type[TaskInterface]):
+        """Register a new task."""
+        self._tasks[name] = task_class
+
+    def get_task(self, name: str, limit: Optional[int] = None) -> TaskInterface:
+        """Get a task instance by name."""
+        if name not in self._tasks:
+            raise TaskNotFoundError(task_name=name, available_tasks=list(self._tasks.keys()))
+
+        task_factory = self._tasks[name]
+
+        # Handle different task factory types
+        if callable(task_factory):
+            # Try calling with limit parameter
+            try:
+                return task_factory(limit=limit)
+            except TypeError:
+                # Fallback for factories that don't accept limit
+                return task_factory()
+        else:
+            # Direct class instantiation
+            return task_factory()
+
+    def list_tasks(self) -> List[str]:
+        """List all available task names."""
+        return list(self._tasks.keys())
+
+    def get_task_info(self, name: str) -> Dict[str, Any]:
+        """Get information about a specific task."""
+        task = self.get_task(name)
+        return {"name": task.get_name(), "description": task.get_description(), "categories": task.get_categories()}
+
+    def list_task_info(self) -> List[Dict[str, Any]]:
+        """List information about all available tasks."""
+        return [self.get_task_info(name) for name in self.list_tasks()]
+
+
+# Global task registry instance
+_task_registry = TaskRegistry()
+
+
+def register_task(name: str, task_class: Type[TaskInterface]):
+    """Register a new task globally."""
+    _task_registry.register_task(name, task_class)
+
+
+def get_task(name: str, limit: Optional[int] = None) -> TaskInterface:
+    """Get a task instance by name."""
+    # Ensure tasks are registered before attempting to get a task
+    _ensure_tasks_registered()
+
+    # Check if this is a file path (contains / or \\ or ends with .json)
+    if "/" in name or "\\" in name or name.endswith(".json"):
+        # Treat as file path and load directly
+        from .tasks.file_task import FileTask
+
+        return FileTask(name, limit=limit)
+
+    # Otherwise, try to get from registry
+    try:
+        return _task_registry.get_task(name, limit=limit)
+    except (ValueError, TaskNotFoundError):
+        # Fallback: try to create a dynamic LMEvalTask if an extractor exists
+        try:
+            from .tasks.lm_eval_task import LMEvalTask, get_extractor
+            # Check if extractor exists for this task
+            get_extractor(name)
+            # Create dynamic task
+            return LMEvalTask(
+                task_name=name,
+                description=f"LM-eval task: {name}",
+                categories=["lm-eval"]
+            )
+        except Exception:
+            raise TaskNotFoundError(task_name=name, available_tasks=list(_task_registry._tasks.keys()))
+
+
+def list_tasks() -> List[str]:
+    """List all available task names."""
+    _ensure_tasks_registered()
+    return _task_registry.list_tasks()
+
+
+def get_task_info(name: str) -> Dict[str, Any]:
+    """Get information about a specific task."""
+    return _task_registry.get_task_info(name)
+
+
+def list_task_info() -> List[Dict[str, Any]]:
+    """List information about all available tasks."""
+    return _task_registry.list_task_info()
+
+
+def _ensure_tasks_registered():
+    """Ensure all tasks are registered in the global registry."""
+    if len(_task_registry._tasks) == 0:  # Only register if not already done
+        # Import tasks module to trigger registration
+        # This is crucial for CLI usage where tasks module isn't imported elsewhere
+        from . import tasks  # noqa: F401 # This imports __init__.py which calls register_all_tasks()

wisent/core/task_selector.py

@@ -0,0 +1,232 @@
+"""
+Task selector for choosing tasks based on skills and risks tags.
+"""
+
+import json
+import os
+import random
+import logging
+from typing import List, Dict, Any, Optional, Set
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class TaskSelector:
+    """Select tasks based on skills and risks criteria."""
+    
+    def __init__(self):
+        """Initialize the task selector by loading metadata."""
+        self.base_path = Path(__file__).parent.parent / "parameters" / "tasks"
+        self.skills = self._load_json("skills.json")
+        self.risks = self._load_json("risks.json")
+        self.tasks_data = self._load_json("tasks.json")
+        self.tasks = self.tasks_data.get("tasks", {})
+        
+    def _load_json(self, filename: str) -> Any:
+        """Load JSON file from parameters/tasks directory."""
+        filepath = self.base_path / filename
+        try:
+            with open(filepath, 'r') as f:
+                return json.load(f)
+        except Exception as e:
+            logger.error(f"Failed to load {filename}: {e}")
+            return {} if filename == "tasks.json" else []
+    
+    def get_available_skills(self) -> List[str]:
+        """Get list of available skills."""
+        return self.skills
+    
+    def get_available_risks(self) -> List[str]:
+        """Get list of available risks."""
+        return self.risks
+    
+    def find_tasks_by_tags(
+        self, 
+        skills: Optional[List[str]] = None, 
+        risks: Optional[List[str]] = None,
+        min_quality_score: int = 2
+    ) -> List[str]:
+        """
+        Find tasks that match the given skills and/or risks.
+        
+        Args:
+            skills: List of skill tags to match
+            risks: List of risk tags to match
+            min_quality_score: Minimum quality score for tasks (default: 2)
+            
+        Returns:
+            List of task names that match the criteria
+        """
+        if not skills and not risks:
+            # Return all tasks if no criteria specified
+            return [
+                task_name for task_name, task_data in self.tasks.items()
+                if task_data.get("quality_score", 0) >= min_quality_score
+            ]
+        
+        # Convert to sets for efficient lookup
+        required_tags = set()
+        if skills:
+            required_tags.update(skills)
+        if risks:
+            required_tags.update(risks)
+        
+        matched_tasks = []
+        for task_name, task_data in self.tasks.items():
+            # Check quality score
+            if task_data.get("quality_score", 0) < min_quality_score:
+                continue
+                
+            # Check if task has any of the required tags
+            task_tags = set(task_data.get("tags", []))
+            if task_tags.intersection(required_tags):
+                matched_tasks.append(task_name)
+        
+        return matched_tasks
+    
+    def select_random_tasks(
+        self,
+        skills: Optional[List[str]] = None,
+        risks: Optional[List[str]] = None,
+        num_tasks: Optional[int] = None,
+        min_quality_score: int = 2,
+        seed: Optional[int] = None
+    ) -> List[str]:
+        """
+        Select random tasks based on skills/risks criteria.
+        
+        Args:
+            skills: List of skill tags to match
+            risks: List of risk tags to match
+            num_tasks: Number of tasks to select (None = all matching tasks)
+            min_quality_score: Minimum quality score for tasks
+            seed: Random seed for reproducibility
+            
+        Returns:
+            List of randomly selected task names
+        """
+        # Find matching tasks
+        matched_tasks = self.find_tasks_by_tags(skills, risks, min_quality_score)
+        
+        if not matched_tasks:
+            logger.warning(f"No tasks found matching skills={skills}, risks={risks}")
+            return []
+        
+        # Set random seed if provided
+        if seed is not None:
+            random.seed(seed)
+        
+        # Select tasks
+        if num_tasks is None or num_tasks >= len(matched_tasks):
+            selected = matched_tasks
+        else:
+            selected = random.sample(matched_tasks, num_tasks)
+        
+        logger.info(f"Selected {len(selected)} tasks from {len(matched_tasks)} matching tasks")
+        return selected
+    
+    def validate_skills_and_risks(
+        self, 
+        skills: Optional[List[str]] = None, 
+        risks: Optional[List[str]] = None
+    ) -> Dict[str, List[str]]:
+        """
+        Validate provided skills and risks against available options.
+        
+        Returns:
+            Dictionary with 'invalid_skills' and 'invalid_risks' lists
+        """
+        invalid = {"invalid_skills": [], "invalid_risks": []}
+        
+        if skills:
+            valid_skills = set(self.skills)
+            invalid["invalid_skills"] = [s for s in skills if s not in valid_skills]
+        
+        if risks:
+            valid_risks = set(self.risks)
+            invalid["invalid_risks"] = [r for r in risks if r not in valid_risks]
+        
+        return invalid
+
+
+def get_tasks_for_skills_and_risks(
+    skills: Optional[List[str]] = None,
+    risks: Optional[List[str]] = None,
+    num_tasks: Optional[int] = None,
+    min_quality_score: int = 2,
+    seed: Optional[int] = None
+) -> List[str]:
+    """
+    Convenience function to get tasks matching skills/risks criteria.
+    
+    Args:
+        skills: List of skill tags to match
+        risks: List of risk tags to match
+        num_tasks: Number of tasks to select (None = all)
+        min_quality_score: Minimum quality score
+        seed: Random seed
+        
+    Returns:
+        List of task names
+    """
+    selector = TaskSelector()
+    
+    # Validate inputs
+    invalid = selector.validate_skills_and_risks(skills, risks)
+    if invalid["invalid_skills"]:
+        logger.warning(f"Invalid skills: {invalid['invalid_skills']}")
+    if invalid["invalid_risks"]:
+        logger.warning(f"Invalid risks: {invalid['invalid_risks']}")
+    
+    # Select tasks
+    return selector.select_random_tasks(
+        skills=skills,
+        risks=risks,
+        num_tasks=num_tasks,
+        min_quality_score=min_quality_score,
+        seed=seed
+    )
+
+
+def expand_task_if_skill_or_risk(task: str) -> str:
+    """
+    Expand a task name to comma-separated benchmarks if it's a skill or risk.
+    
+    If the task is a skill name (coding, mathematics, etc.) or a risk name
+    (hallucination, toxicity, etc.), expands it to all matching benchmarks.
+    Otherwise returns the original task unchanged.
+    
+    Args:
+        task: Task name, skill name, risk name, or comma-separated benchmarks
+        
+    Returns:
+        Original task if it's a benchmark name, or comma-separated benchmarks
+        if it was a skill/risk name
+    """
+    if not task:
+        return task
+    
+    task_lower = task.lower().strip()
+    
+    # Already comma-separated, return as-is
+    if "," in task:
+        return task
+    
+    selector = TaskSelector()
+    available_skills = [s.lower() for s in selector.get_available_skills()]
+    available_risks = [r.lower() for r in selector.get_available_risks()]
+    
+    if task_lower in available_skills:
+        matching_tasks = selector.find_tasks_by_tags(skills=[task_lower])
+        if matching_tasks:
+            logger.info(f"Expanded skill '{task}' to {len(matching_tasks)} benchmarks")
+            return ",".join(matching_tasks)
+    elif task_lower in available_risks:
+        matching_tasks = selector.find_tasks_by_tags(risks=[task_lower])
+        if matching_tasks:
+            logger.info(f"Expanded risk '{task}' to {len(matching_tasks)} benchmarks")
+            return ",".join(matching_tasks)
+    
+    # Not a skill or risk, return original
+    return task