wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/core/contrastive_pairs/diagnostics/linearity.py

@@ -8,8 +8,7 @@ from enum import Enum
 
 import torch
 
-from wisent.core.activations.core.atoms import ActivationAggregationStrategy
-from wisent.core.activations.prompt_construction_strategy import PromptConstructionStrategy
+from wisent.core.activations.extraction_strategy import ExtractionStrategy
 
 
 class LinearityVerdict(Enum):
@@ -35,11 +34,8 @@ class LinearityConfig:
     layers_to_test: Optional[List[int]] = None
     """Specific layers to test. If None, tests sample across depth."""
     
-    aggregation_strategies: Optional[List[ActivationAggregationStrategy]] = None
-    """Aggregation strategies to test. If None, tests all."""
-    
-    prompt_strategies: Optional[List[PromptConstructionStrategy]] = None
-    """Prompt strategies to test. If None, tests all."""
+    extraction_strategies: Optional[List[ExtractionStrategy]] = None
+    """Extraction strategies to test. If None, tests default set."""
     
     normalize_options: List[bool] = field(default_factory=lambda: [False, True])
     """Normalization options to test."""
@@ -128,24 +124,15 @@ def check_linearity(
     else:
         layers_to_test = cfg.layers_to_test
     
-    # Determine aggregation strategies
-    if cfg.aggregation_strategies is None:
-        aggregation_strategies = [
-            ActivationAggregationStrategy.LAST_TOKEN,
-            ActivationAggregationStrategy.MEAN_POOLING,
-            ActivationAggregationStrategy.MAX_POOLING,
-        ]
-    else:
-        aggregation_strategies = cfg.aggregation_strategies
-    
-    # Determine prompt strategies
-    if cfg.prompt_strategies is None:
-        prompt_strategies = [
-            PromptConstructionStrategy.CHAT_TEMPLATE,
-            PromptConstructionStrategy.DIRECT_COMPLETION,
+    # Determine extraction strategies
+    if cfg.extraction_strategies is None:
+        extraction_strategies = [
+            ExtractionStrategy.CHAT_LAST,
+            ExtractionStrategy.CHAT_MEAN,
+            ExtractionStrategy.CHAT_MAX_NORM,
         ]
     else:
-        prompt_strategies = cfg.prompt_strategies
+        extraction_strategies = cfg.extraction_strategies
     
     # Limit pairs
     test_pairs = pairs[:cfg.max_pairs]
@@ -157,62 +144,66 @@ def check_linearity(
     
     all_results = []
     
-    for prompt_strategy in prompt_strategies:
-        for agg_strategy in aggregation_strategies:
-            for normalize in cfg.normalize_options:
-                # Collect activations
-                pos_activations = {l: [] for l in layers_to_test}
-                neg_activations = {l: [] for l in layers_to_test}
-                
-                for pair in test_pairs:
-                    try:
-                        pair_with_acts = collector.collect_for_pair(
-                            pair,
-                            layers=[str(l) for l in layers_to_test],
-                            aggregation=agg_strategy,
-                            normalize_layers=normalize,
-                            prompt_strategy=prompt_strategy,
-                        )
-                        
-                        pos_la = pair_with_acts.positive_response.layers_activations
-                        neg_la = pair_with_acts.negative_response.layers_activations
-                        
-                        if pos_la and neg_la:
-                            for layer in layers_to_test:
-                                pos_t = pos_la.get(str(layer))
-                                neg_t = neg_la.get(str(layer))
-                                if pos_t is not None and neg_t is not None:
-                                    pos_activations[layer].append(pos_t.flatten().cpu())
-                                    neg_activations[layer].append(neg_t.flatten().cpu())
-                    except Exception:
-                        continue
-                
-                # Analyze each layer
-                for layer in layers_to_test:
-                    pos_list = pos_activations[layer]
-                    neg_list = neg_activations[layer]
-                    
-                    if len(pos_list) < 10 or len(neg_list) < 10:
-                        continue
-                    
-                    pos_tensor = torch.stack(pos_list)
-                    neg_tensor = torch.stack(neg_list)
+    for strategy in extraction_strategies:
+        for normalize in cfg.normalize_options:
+            # Collect activations
+            pos_activations = {l: [] for l in layers_to_test}
+            neg_activations = {l: [] for l in layers_to_test}
+            
+            for pair in test_pairs:
+                try:
+                    pair_with_acts = collector.collect(
+                        pair,
+                        strategy=strategy,
+                        layers=[str(l) for l in layers_to_test],
+                        normalize=normalize,
+                    )
                     
-                    result = detect_geometry_structure(pos_tensor, neg_tensor, geo_config)
+                    pos_la = pair_with_acts.positive_response.layers_activations
+                    neg_la = pair_with_acts.negative_response.layers_activations
                     
-                    linear_score = result.all_scores["linear"].score
-                    linear_details = result.all_scores["linear"].details
-                    
-                    all_results.append({
-                        "prompt_strategy": prompt_strategy.name,
-                        "aggregation": agg_strategy.name,
-                        "normalize": normalize,
-                        "layer": layer,
-                        "linear_score": linear_score,
-                        "cohens_d": linear_details.get("cohens_d", 0),
-                        "variance_explained": linear_details.get("variance_explained", 0),
-                        "best_structure": result.best_structure.value,
-                    })
+                    if pos_la and neg_la:
+                        for layer in layers_to_test:
+                            pos_t = pos_la.get(str(layer))
+                            neg_t = neg_la.get(str(layer))
+                            if pos_t is not None and neg_t is not None:
+                                pos_activations[layer].append(pos_t.flatten().cpu())
+                                neg_activations[layer].append(neg_t.flatten().cpu())
+                except Exception:
+                    continue
+            
+            # Analyze each layer
+            for layer in layers_to_test:
+                pos_list = pos_activations[layer]
+                neg_list = neg_activations[layer]
+                
+                if len(pos_list) < 10 or len(neg_list) < 10:
+                    continue
+                
+                pos_tensor = torch.stack(pos_list)
+                neg_tensor = torch.stack(neg_list)
+                
+                result = detect_geometry_structure(pos_tensor, neg_tensor, geo_config)
+                
+                linear_score = result.all_scores["linear"].score
+                linear_details = result.all_scores["linear"].details
+                
+                # Include all structure scores
+                structure_scores = {
+                    name: {"score": score.score, "confidence": score.confidence}
+                    for name, score in result.all_scores.items()
+                }
+                
+                all_results.append({
+                    "extraction_strategy": strategy.value,
+                    "normalize": normalize,
+                    "layer": layer,
+                    "linear_score": linear_score,
+                    "cohens_d": linear_details.get("cohens_d", 0),
+                    "variance_explained": linear_details.get("variance_explained", 0),
+                    "best_structure": result.best_structure.value,
+                    "all_structure_scores": structure_scores,
+                })
     
     if not all_results:
         return LinearityResult(
@@ -234,7 +225,7 @@ def check_linearity(
         verdict = LinearityVerdict.LINEAR
         recommendation = (
             f"Use CAA (single-direction steering) on layer {best['layer']} "
-            f"with {best['prompt_strategy']} prompt and {best['aggregation']} aggregation."
+            f"with {best['extraction_strategy']} strategy."
         )
     elif best["linear_score"] >= cfg.weak_threshold and best["cohens_d"] >= cfg.min_cohens_d:
         verdict = LinearityVerdict.WEAKLY_LINEAR
@@ -254,8 +245,7 @@ def check_linearity(
         verdict=verdict,
         best_linear_score=best["linear_score"],
         best_config={
-            "prompt_strategy": best["prompt_strategy"],
-            "aggregation": best["aggregation"],
+            "extraction_strategy": best["extraction_strategy"],
             "normalize": best["normalize"],
         },
         best_layer=best["layer"],

wisent/core/contrastive_pairs/diagnostics/vector_quality.py

@@ -281,7 +281,8 @@ def _compute_pca(
         
         n_components = min(5, n - 1)
         pca = PCA(n_components=n_components)
-        pca.fit(difference_vectors.numpy())
+        # Convert to float32 for sklearn compatibility (BFloat16 not supported)
+        pca.fit(difference_vectors.float().numpy())
         
         pc1_var = pca.explained_variance_ratio_[0]
         pc2_var = pca.explained_variance_ratio_[1] if n_components > 1 else 0.0
@@ -372,7 +373,7 @@ def _compute_clustering(
     try:
         from sklearn.metrics import silhouette_score
         
-        all_activations = torch.cat([positive_activations, negative_activations], dim=0).numpy()
+        all_activations = torch.cat([positive_activations, negative_activations], dim=0).float().numpy()
         labels = [0] * n_pos + [1] * n_neg
         
         silhouette = silhouette_score(all_activations, labels)
@@ -436,7 +437,7 @@ def _compute_cv_classification(
         from sklearn.linear_model import LogisticRegression
         from sklearn.model_selection import cross_val_score
         
-        X = torch.cat([positive_activations, negative_activations], dim=0).numpy()
+        X = torch.cat([positive_activations, negative_activations], dim=0).float().numpy()
         y = np.array([1] * n_pos + [0] * n_neg)
         
         n_folds = min(config.cv_folds, min(n_pos, n_neg))
@@ -473,8 +474,8 @@ def _compute_cohens_d(
     direction = direction / direction_norm
     
     # Project all activations onto this direction
-    pos_proj = (positive_activations @ direction).numpy()
-    neg_proj = (negative_activations @ direction).numpy()
+    pos_proj = (positive_activations @ direction).float().numpy()
+    neg_proj = (negative_activations @ direction).float().numpy()
     
     # Cohen's d = (mean1 - mean2) / pooled_std
     mean_diff = pos_proj.mean() - neg_proj.mean()

wisent/core/contrastive_pairs/huggingface_pairs/hf_extractor_manifest.py

@@ -133,14 +133,11 @@ EXTRACTORS: dict[str, str] = {
 
     # Coding benchmarks
     "humaneval": f"{base_import}humaneval:HumanEvalExtractor",
-    "humaneval_plus": f"{base_import}humaneval:HumanEvalExtractor",
-    "humaneval_64_instruct": f"{base_import}instructhumaneval:InstructHumanEvalExtractor",
-    "humaneval_instruct": f"{base_import}instructhumaneval:InstructHumanEvalExtractor",
-    "humanevalpack": f"{base_import}humaneval:HumanEvalExtractor",
-    "instructhumaneval": f"{base_import}instructhumaneval:InstructHumanEvalExtractor",
-    "mbpp": f"{base_import}mbpp:MBPPExtractor",
-    "mbpp_plus": f"{base_import}mbpp:MBPPExtractor",
-    "instruct_humaneval": f"{base_import}instructhumaneval:InstructHumanEvalExtractor",
+    "humaneval_64": f"{base_import}humaneval:HumanEval64Extractor",
+    "humaneval_plus": f"{base_import}humaneval:HumanEvalPlusExtractor",
+    "humaneval_instruct": f"{base_import}humaneval:HumanEvalInstructExtractor",
+    "humaneval_64_instruct": f"{base_import}humaneval:HumanEval64InstructExtractor",
+    "humanevalpack": f"{base_import}humanevalpack:HumanevalpackExtractor",
     "apps": f"{base_import}apps:AppsExtractor",
     "conala": f"{base_import}conala:ConalaExtractor",
     "concode": f"{base_import}concode:ConcodeExtractor",
@@ -156,13 +153,6 @@ EXTRACTORS: dict[str, str] = {
     "multiple_rs": f"{base_import}multipl_e:MultiplEExtractor",
     "multiple_go": f"{base_import}multipl_e:MultiplEExtractor",
     "codexglue": f"{base_import}codexglue:CodexglueExtractor",
-    "code_x_glue": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_python": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_go": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_ruby": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_java": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_javascript": f"{base_import}codexglue:CodexglueExtractor",
-    "codexglue_code_to_text_php": f"{base_import}codexglue:CodexglueExtractor",
     "livecodebench": f"{base_import}livecodebench:LivecodebenchExtractor",
 
     # Reasoning benchmarks
@@ -203,7 +193,6 @@ EXTRACTORS: dict[str, str] = {
     "ds1000": f"{base_import}ds1000:Ds1000Extractor",
     "evalita_mp": f"{base_import}evalita_mp:EvalitaMpExtractor",
     "flores": f"{base_import}flores:FloresExtractor",
-    "freebase": f"{base_import}freebase:FreebaseExtractor",
     "humanevalpack": f"{base_import}humanevalpack:HumanevalpackExtractor",
     "iwslt2017_ar_en": f"{base_import}iwslt2017_ar_en:Iwslt2017ArEnExtractor",
     "iwslt2017_en_ar": f"{base_import}iwslt2017_en_ar:Iwslt2017EnArExtractor",
@@ -229,11 +218,8 @@ EXTRACTORS: dict[str, str] = {
     "flan_held_in": f"{base_import}flan_held_in:FlanHeldInExtractor",
     "gpt3_translation_benchmarks": f"{base_import}gpt3_translation_benchmarks:Gpt3TranslationBenchmarksExtractor",
     "multiple_choice": f"{base_import}multiple_choice:MultipleChoiceExtractor",
-    "non_greedy_robustness_agieval_aqua_rat": f"{base_import}non_greedy_robustness_agieval_aqua_rat:NonGreedyRobustnessAgievalAquaRatExtractor",
-    "option_order_robustness_agieval_aqua_rat": f"{base_import}option_order_robustness_agieval_aqua_rat:OptionOrderRobustnessAgievalAquaRatExtractor",
     "penn_treebank": f"{base_import}penn_treebank:PennTreebankExtractor",
     "ptb": f"{base_import}penn_treebank:PennTreebankExtractor",
-    "prompt_robustness_agieval_aqua_rat": f"{base_import}prompt_robustness_agieval_aqua_rat:PromptRobustnessAgievalAquaRatExtractor",
     "self_consistency": f"{base_import}self_consistency:SelfConsistencyExtractor",
     "t0_eval": f"{base_import}t0_eval:T0EvalExtractor",
     "vaxx_stance": f"{base_import}vaxx_stance:VaxxStanceExtractor",

wisent/core/contrastive_pairs/huggingface_pairs/hf_task_extractors/__init__.py

@@ -8,12 +8,16 @@ from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.concode
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.ds_1000 import Ds1000Extractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.hle import HleExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.hmmt import HMMTExtractor
-from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.humaneval import HumanEvalExtractor
-from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.instructhumaneval import InstructHumanEvalExtractor
+from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.humaneval import (
+    HumanEvalExtractor,
+    HumanEval64Extractor,
+    HumanEvalPlusExtractor,
+    HumanEvalInstructExtractor,
+    HumanEval64InstructExtractor,
+)
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.livecodebench import LivecodebenchExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.livemathbench import LiveMathBenchExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.math500 import MATH500Extractor
-from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.mbpp import MBPPExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.mercury import MercuryExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.multipl_e import MultiplEExtractor
 from wisent.core.contrastive_pairs.huggingface_pairs.hf_task_extractors.polymath import PolyMathExtractor
@@ -30,11 +34,13 @@ __all__ = [
     "HleExtractor",
     "HMMTExtractor",
     "HumanEvalExtractor",
-    "InstructHumanEvalExtractor",
+    "HumanEval64Extractor",
+    "HumanEvalPlusExtractor",
+    "HumanEvalInstructExtractor",
+    "HumanEval64InstructExtractor",
     "LivecodebenchExtractor",
     "LiveMathBenchExtractor",
     "MATH500Extractor",
-    "MBPPExtractor",
     "MercuryExtractor",
     "MultiplEExtractor",
     "PolyMathExtractor",