wisent 0.1.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

wisent/synthetic/generators/core/atoms.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import logging
+from dataclasses import dataclass
+
+__all__ = [
+    "Diversity",
+    "DiversityScores",
+    "GenerationReport",
+]
+
+logger = logging.getLogger(__name__)
+
+@dataclass(frozen=True)
+class DiversityScores:
+    """
+    Diversity metrics for a list of texts.
+    
+    attributes:
+        unique_unigrams:
+            Unique unigrams (ratio of unique unigrams to total unigrams).
+        unique_bigrams:
+            Unique bigrams (ratio of unique bigrams to total bigrams).
+        avg_jaccard_prompt:
+            Average Jaccard similarity between all pairs of texts.
+        mean_simhash_hamming_prompt:
+            Mean Hamming distance between SimHash fingerprints of all pairs of texts.
+        min_simhash_hamming_prompt:
+            Minimum Hamming distance between SimHash fingerprints of all pairs of texts.
+    """
+    unique_unigrams: float
+    unique_bigrams: float
+    avg_jaccard_prompt: float
+    mean_simhash_hamming_prompt: float
+    min_simhash_hamming_prompt: int
+
+@dataclass(frozen=True)
+class GenerationReport:
+    """
+    Report of a generation+cleaning run.
+
+    attributes:
+        requested:
+            Number of contrastive pairs requested from the model.
+        kept_after_dedupe:
+            Number of contrastive pairs kept after deduplication.
+        retries_for_refusals:
+            Number of retries made to fix refusals in negative examples.
+        diversity:
+            DiversityScores computed on the final prompts.
+
+    notes:
+        LLMs may refuse to generate negative examples that exhibit undesired traits. Increasing
+        'max_refusal_retries' in the generator can help mitigate this, but increases cost and latency.
+    """
+    requested: int
+    kept_after_dedupe: int
+    retries_for_refusals: int
+    diversity: DiversityScores
+
+
+class Diversity(ABC):
+    """
+    Diversity metrics computation.
+    
+    methods:
+        compute(texts: list[str]) -> DiversityScores:
+            Compute diversity metrics for a list of texts. We can compute Distinct-N, Jaccard similarity,
+            and SimHash Hamming distance to get a sense of lexical and structural diversity.
+    """
+    @abstractmethod
+    def compute(self, texts: list[str]) -> DiversityScores: ...

wisent/synthetic/generators/diversities/core/core.py

@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+__all__ = [
+    "Diversity",
+    "DiversityScores",
+    "GenerationReport",
+]
+
+@dataclass(frozen=True)
+class DiversityScores:
+    """
+    Diversity metrics for a list of texts.
+    
+    attributes:
+        unique_unigrams:
+            Unique unigrams (ratio of unique unigrams to total unigrams).
+        unique_bigrams:
+            Unique bigrams (ratio of unique bigrams to total bigrams).
+        avg_jaccard_prompt:
+            Average Jaccard similarity between all pairs of texts.
+        mean_simhash_hamming_prompt:
+            Mean Hamming distance between SimHash fingerprints of all pairs of texts.
+        min_simhash_hamming_prompt:
+            Minimum Hamming distance between SimHash fingerprints of all pairs of texts.
+    """
+    unique_unigrams: float
+    unique_bigrams: float
+    avg_jaccard_prompt: float
+    mean_simhash_hamming_prompt: float
+    min_simhash_hamming_prompt: int
+
+@dataclass(frozen=True)
+class GenerationReport:
+    """
+    Report of a generation+cleaning run.
+
+    attributes:
+        requested:
+            Number of contrastive pairs requested from the model.
+        kept_after_dedupe:
+            Number of contrastive pairs kept after deduplication.
+        retries_for_refusals:
+            Number of retries made to fix refusals in negative examples.
+        diversity:
+            DiversityScores computed on the final prompts.
+
+    notes:
+        LLMs may refuse to generate negative examples that exhibit undesired traits. Increasing
+        'max_refusal_retries' in the generator can help mitigate this, but increases cost and latency.
+    """
+    requested: int
+    kept_after_dedupe: int
+    retries_for_refusals: int
+    diversity: DiversityScores
+
+
+class Diversity(ABC):
+    """
+    Diversity metrics computation.
+    
+    methods:
+        compute(texts: list[str]) -> DiversityScores:
+            Compute diversity metrics for a list of texts. We can compute Distinct-N, Jaccard similarity,
+            and SimHash Hamming distance to get a sense of lexical and structural diversity.
+    """
+    @abstractmethod
+    def compute(self, texts: list[str]) -> DiversityScores: ...

wisent/synthetic/generators/diversities/methods/fast_diversity.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+from typing import Iterable
+import re
+import numpy as np
+from wisent.synthetic.generators.diversities.core.core import Diversity, DiversityScores
+
+__all__ = [
+    "FastDiversity",
+]
+
+class FastDiversity(Diversity):
+    """
+    Fast diversity metrics computation.
+
+    attributes:
+        _rng:
+            Random number generator for sampling.
+    """
+    _TOKEN_RE = re.compile(r"[A-Za-z0-9']+|[^\w\s]")
+
+    def __init__(self, seed: int | None = 13) -> None:
+        self._rng = np.random.default_rng(seed)
+
+    def compute(self, texts: list[str]) -> DiversityScores:
+        """
+        Compute diversity metrics for a list of texts.
+
+        arguments:
+            texts:
+                List of input strings.
+
+        returns:
+            DiversityScores dataclass with various diversity metrics.
+
+        example:
+            >>> fd = FastDiversity(seed=42)
+            >>> texts = ["hello world", "hello there", "different text"]
+            >>> fd.compute(texts)
+            ... DiversityScores(unique_unigrams=0.6666666666666666, unique_bigrams=1.0, avg_jaccard_prompt=0.3333333333333333, mean_simhash_hamming_prompt=21.333333333333332, min_simhash_hamming_prompt=20)
+        
+        intuition:
+            Higher unique scores and lower average Jaccard indicate more lexical diversity.
+            Higher mean and minimum SimHash Hamming distances indicate more structural diversity.
+        """
+        d1 = self._distinct_n(texts, 1)
+        d2 = self._distinct_n(texts, 2)
+        sample = texts if len(texts) <= 20 else self._rng.sample(texts, 20)
+        if len(sample) >= 2:
+            jaccs: list[float] = []
+            fps: list[int] = []
+            for i, s in enumerate(sample):
+                fps.append(self._simhash64(s))
+                for j in range(i + 1, len(sample)):
+                    jaccs.append(self._jaccard(s, sample[j]))
+            avg_j = sum(jaccs) / len(jaccs) if jaccs else 0.0
+            dists: list[int] = []
+            for i in range(len(sample)):
+                for j in range(i + 1, len(sample)):
+                    dists.append(self._hamming(fps[i], fps[j]))
+            mean_h = (sum(dists) / len(dists)) if dists else 0.0
+            min_h = min(dists) if dists else 64
+        else:
+            avg_j, mean_h, min_h = 0.0, 0.0, 64
+        return DiversityScores(d1, d2, avg_j, float(mean_h), int(min_h))
+
+    def _distinct_n(self, texts: Iterable[str], n: int) -> float:
+        """
+        Compute the Distinct-N score for a list of texts.
+
+        arguments:
+            texts:
+                Iterable of input strings.
+            n:
+                N-gram size (e.g., 1 for unigrams, 2 for bigrams).
+        
+        returns:
+            Distinct-N score: ratio of unique n-grams to total n-grams.
+
+        example:
+            >>> fd = FastDiversity()
+            >>> texts = ["hello world", "hello there"]
+            >>> fd._distinct_n(texts, 1)
+            0.75
+            >>> fd._distinct_n(texts, 2)
+            1.0
+        intuition:
+            Distinct-N = (number of unique n-grams) / (total number of n-grams)
+            Higher values indicate more lexical diversity. For example, in ["hello world", "hello there"]:
+                Unigrams: 
+                    ['hello', 'world', 'hello', 'there'] → unique = ['hello', 'world', 'there'] → Distinct-1 = 3/4 = 0.75
+                Bigrams: 
+                    ['hello world', 'hello there'] → unique = ['hello world', 'hello there'] → Distinct-2 = 2/2 = 1.0
+        """
+        ngrams: list[tuple[str, ...]] = []
+        for t in texts:
+            toks = self._tok(t)
+            ngrams.extend(tuple(toks[i : i + n]) for i in range(0, max(0, len(toks) - n + 1)))
+        return (len(set(ngrams)) / float(len(ngrams))) if ngrams else 0.0
+    
+    def _jaccard(self, a: str, b: str) -> float:
+        """
+        Compute Jaccard similarity between two strings.
+
+        arguments:
+            a:
+                First input string.
+            b:
+                Second input string.
+
+        returns:
+            Jaccard similarity score between 0.0 and 1.0.
+
+        example:
+            >>> fd = FastDiversity()
+            >>> fd._jaccard("hello world", "hello there")
+            0.3333333333333333
+            >>> fd._jaccard("abc", "xyz")
+            0.0
+            >>> fd._jaccard("", "")
+            1.0
+            >>> fd._jaccard("abc", "")
+            0.0
+        
+        intuition:
+            Jaccard similarity = |A ∩ B| / |A ∪ B|
+            where A and B are the sets of tokens in strings a and b. Higher values indicate more overlap.
+            For example "hello world" and "hello there" share the token "hello", yielding a similarity of 1/3.
+        """
+        A, B = set(self._tok(a)), set(self._tok(b))
+        if not A and not B:
+            return 1.0
+        if not A or not B:
+            return 0.0
+        inter = len(A & B)
+        union = len(A | B)
+        return inter / union if union else 0.0
+    
+    def _tok(self, s: str) -> list[str]:
+        """
+        Simple whitespace/punctuation tokenizer.
+        
+        arguments:
+            s:
+                Input string to tokenize.
+
+        returns:
+            List of tokens (words and punctuation).
+        """
+        return self._TOKEN_RE.findall(s.lower())
+    
+    def _hamming(self, a: int, b: int) -> int:
+        """
+        Compute the Hamming distance between two 64-bit integers.
+        arguments:
+            a:
+                First 64-bit integer.
+            b:
+                Second 64-bit integer.
+        
+        returns:
+            Hamming distance (number of differing bits).
+
+        example:
+            >>> fd = FastDiversity()
+            >>> fd._hamming(0b1010, 0b1001)
+            2
+            >>> fd._hamming(0b1111, 0b1111)
+            0
+            >>> fd._hamming(0b0, 0b1111111111111111)
+            64
+
+        intuition:
+            Hamming distance counts the number of bit positions where two integers differ.
+            For example, 0b1010 (10) and 0b1001 (9) differ in two bit positions, yielding a distance of 2.
+            The maximum distance for 64-bit integers is 64 (completely different), and identical integers have a distance of 0.
+        """
+        x = a ^ b
+        c = 0
+        while x:
+            x &= x - 1
+            c += 1
+        return c
+
+    def _hash64(self, x: str) -> int:
+        """
+        64-bit FNV-1a hash of a string.
+        
+        arguments:
+            x:
+                Input string to hash.
+        
+        returns:
+            64-bit integer hash value.
+            
+        example:
+            >>> fd = FastDiversity()
+            >>> fd._hash64("hello")
+            11831194018420276491
+            >>> fd._hash64("world")
+            15195822415430384601
+            >>> fd._hash64("")
+            14695981039346656037
+        """
+        h = 0xCBF29CE484222325
+        for c in x.encode("utf-8"):
+            h ^= c
+            h = (h * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
+        return h
+
+    def _simhash64(self, text: str) -> int:
+        """
+        Compute the SimHash of a text.
+
+        arguments:
+            text: Input text to hash.
+
+        returns:
+            64-bit SimHash fingerprint as an integer.
+
+        example:
+            >>> fd = FastDiversity()
+            >>> fd._simhash64("hello world")
+            16204198794447330368
+            >>> fd._simhash64("hello there")
+            16204198794447330368
+            >>> fd._simhash64("different text")
+            1080863910568919040
+            >>> fd._simhash64("")
+            0
+        intuition:
+            SimHash is a locality-sensitive hash that maps similar texts to similar fingerprints.
+            It works by hashing features (tokens) and combining their bits based on frequency.
+            For example, "hello world" and "hello there" share the token "hello", resulting in identical SimHash values.
+            In contrast, "different text" yields a very different fingerprint. An empty string hashes to 0.
+            This makes SimHash useful for deduplication and near-duplicate detection.
+        """
+        feats = self._tok(text)
+        if not feats:
+            return 0
+        vec = [0] * 64
+        for f in feats:
+            hv = self._hash64(f)
+            for i in range(64):
+                vec[i] += 1 if (hv >> i) & 1 else -1
+        out = 0
+        for i, v in enumerate(vec):
+            if v >= 0:
+                out |= (1 << i)
+        return out

wisent/synthetic/generators/pairs_generator.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import logging
+
+
+from wisent.core.contrastive_pairs.core.pair import ContrastivePair  
+from wisent.core.contrastive_pairs.core.response import PositiveResponse, NegativeResponse  
+from wisent.core.contrastive_pairs.core.set import ContrastivePairSet 
+
+from wisent.core.models.wisent_model import WisentModel
+from wisent.synthetic.db_instructions.core.atoms import DB_Instructions
+
+from wisent.synthetic.generators.core.atoms import GenerationReport
+
+from wisent.synthetic.generators.diversities.core.core import Diversity
+
+from wisent.synthetic.cleaners.pairs_cleaner import PairsCleaner
+
+__all__ = [
+    "SyntheticContrastivePairsGenerator",
+]
+
+logger = logging.getLogger(__name__)
+
+class SyntheticContrastivePairsGenerator:
+    """Small, fast contrastive-pairs generator with an extensible cleaning pipeline."""
+
+    def __init__(
+        self,
+        model: WisentModel,
+        generation_config: dict[str, int | float | str],
+        contrastive_set_name: str,
+        trait_description: str,
+        trait_label: str,
+        db_instructions: DB_Instructions,
+        cleaner: PairsCleaner,
+        diversity: Diversity,
+    ) -> None:
+        self.model = model
+        self.db_instructions = db_instructions
+        self.generation_config = generation_config
+        self.cleaner = cleaner
+        self.diversity = diversity
+
+        self.contrastive_set_name = contrastive_set_name
+        self.trait_description = trait_description
+        self.trait_label = trait_label
+
+
+    def generate(
+        self,
+        num_pairs: int = 10,
+    ) -> tuple[ContrastivePairSet, GenerationReport]:
+        """
+        Generate synthetic contrastive pairs for the given topic and trait.
+
+        arguments:
+            num_pairs:
+                Number of contrastive pairs to generate (default: 10).
+        
+        returns:
+            Tuple of ContrastivePairSet with the generated pairs and GenerationReport with statistics about the generation
+        """
+        # 1) generate
+        sys = self.db_instructions.get("generic_pairs")
+        usr = self._build_user_prompt(
+            self.trait_label, self.trait_description, num_pairs
+        )
+        raw = self.model.generate(
+            inputs=[[
+                {"role": "system", "content": sys},
+                {"role": "user", "content": usr}
+            ]],
+                 **self.generation_config,
+        )
+
+        # 2) parse
+        parsed = self.parse_pairs(raw)
+
+        # 3) clean
+        cleaned, stats = self.cleaner.clean(parsed)
+
+        retries = stats.step_stats.get("refusaler_cleaner").modified_items 
+
+        # 4) build domain objects
+        cps = ContrastivePairSet(name=self.contrastive_set_name, task_type=self.trait_label)
+        for item in cleaned.pairs:
+            cps.add(
+                ContrastivePair(
+                    prompt=item.prompt,
+                    positive_response=PositiveResponse(model_response=item.positive_response.model_response),
+                    negative_response=NegativeResponse(model_response=item.negative_response.model_response),
+                    label=item.label or self.trait_label,
+                    trait_description=item.trait_description or self.trait_description,
+                )
+            )
+        # 5) diversity summary (prompts only)
+        prompts = [it.prompt for it in cleaned.pairs]
+        div = self.diversity.compute(prompts)
+
+        report = GenerationReport(
+            requested=num_pairs,
+            kept_after_dedupe=len(cleaned),
+            retries_for_refusals=retries,
+            diversity=div,
+        )
+        return cps, report
+
+    def parse_pairs(self, raw: list[str]) -> ContrastivePairSet:
+        """
+        Parse raw model outputs into ContrastivePairSet objects.
+
+        arguments:
+            raw:
+                Raw model output string to parse.
+        returns:
+            ContrastivePairSet object parsed from the raw string.
+        """
+
+        import json
+
+        out: ContrastivePairSet = ContrastivePairSet(
+            name=self.contrastive_set_name,
+            task_type=self.trait_label,
+        )
+        for r in raw:
+            #TODO: this is very ugly, need to improve robustness
+            # r can have instruction, and i want extacrt everything between ```json and ``` (after - You must return answer in valid JSON format only. Don't include any explanations or additional text.assistant)
+            # also try to recover like Expecting ',' delimiter
+            if "```json" in r:
+                r = r.split("```json")[-1]
+            if "```" in r:
+                r = r.split("```")[0]
+            r = r.strip()
+            try:
+                data = json.loads(r)
+            except json.JSONDecodeError:
+                # try to recover from common errors
+                r = r.replace("'", '"').replace("```", '')
+                try:
+                    data = json.loads(r)
+                except json.JSONDecodeError:
+                    continue
+            for item in data.get("pairs", []):
+                cp = ContrastivePair(
+                    prompt=item["prompt"],
+                    positive_response=PositiveResponse(model_response=item["positive"]),
+                    negative_response=NegativeResponse(model_response=item["negative"]),
+                    label=item.get("label", self.trait_label),
+                    trait_description=item.get("trait_description", self.trait_description),
+                )
+                out.add(cp)
+        return out
+
+    @staticmethod
+    def _build_user_prompt(label: str, desc: str, k: int) -> str:
+        bullets = (
+            f"- Trait label: {label}\n"
+            f"- Trait description: {desc}\n"
+            f"- Num pairs: {k}\n"
+        )
+        schema = (
+            "Return JSON like:\n"
+            "```json\n"
+            "{\n"
+            '  "pairs": [\n'
+            '    {"prompt": "...", "positive": "...", "negative": "...", '
+            f'"label": "{label}", "trait_description": "{desc}"\n'
+            "  ]\n"
+            "}\n"
+        )
+
+        tips = (
+            "- Make prompts specific to the topic but varied in wording and intent.\n"
+            "- Keep negative examples safe (fictional, non-actionable).\n"
+            "- Avoid meta-text like “I cannot” or “As an AI model…”.\n"
+            "- You must return answer in valid JSON format only. Don't include any explanations or additional text.\n"
+        )
+        return f"Create {k} contrastive pairs.\n{bullets}\n{schema}\n{tips}"

wisent-0.5.2.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: wisent
+Version: 0.5.2
+Summary: Monitor and guard against harmful content in language models
+Home-page: https://github.com/yourusername/wisent-activation-guardrails
+Author: Wisent Team
+Author-email: your.email@example.com
+Keywords: nlp,machine learning,language models,safety,guardrails,lm-evaluation-harness
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=1.9.0
+Requires-Dist: transformers>=4.20.0
+Requires-Dist: tqdm>=4.50.0
+Requires-Dist: scikit-learn>=0.24.0
+Requires-Dist: pandas>=1.2.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: datasets>=2.0.0
+Requires-Dist: sentence-transformers>=2.0.0
+Requires-Dist: faiss-cpu>=1.7.0
+Provides-Extra: harness
+Requires-Dist: lm-eval==0.4.8; extra == "harness"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Wisent-Guard
+
+<p align="center">
+  <a href="https://github.com/wisent-ai/wisent-guard/stargazers">
+    <img src="https://img.shields.io/github/stars/wisent-ai/wisent-guard" alt="stars" />
+  </a>
+  <a href="https://pypi.org/project/wisent-guard">
+    <img src="https://static.pepy.tech/badge/wisent-guard" alt="PyPI - Downloads" />
+  </a>
+  <br />
+</p>
+
+<p align="center">
+  <img src="wisent-guard-logo.png" alt="Wisent Guard" width="200">
+</p>
+
+A Python package for latent space monitoring and guardrails. Delivered to you by the [Wisent](https://wisent.ai) team led by [Lukasz Bartoszcze](https://lukaszbartoszcze.com).
+
+## Overview
+
+Wisent-Guard allows you to control your AI by identifying brain patterns corresponding to responses you don't like, like hallucinations or harmful outputs. We use contrastive pairs of representations to detect when a model might be generating harmful content or hallucinating. Learn more at https://www.wisent.ai/wisent-guard.  
+
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.