wisent 0.7.379__py3-none-any.whl → 0.7.901__py3-none-any.whl

wisent/core/cli/generate_pairs_from_task.py

@@ -4,8 +4,6 @@ import sys
 import json
 import os
 
-from wisent.core.errors import InvalidDataFormatError
-
 
 def execute_generate_pairs_from_task(args):
     """Execute the generate-pairs-from-task command - load and save contrastive pairs from a task."""
@@ -14,9 +12,8 @@ def execute_generate_pairs_from_task(args):
     if hasattr(args, 'task_name') and args.task_name:
         args.task_name = expand_task_if_skill_or_risk(args.task_name)
     
-    from wisent.core.contrastive_pairs.huggingface_pairs.hf_extractor_manifest import HF_EXTRACTORS
     from wisent.core.contrastive_pairs.lm_eval_pairs.lm_task_pairs_generation import (
-        lm_build_contrastive_pairs,
+        build_contrastive_pairs,
     )
 
     print(f"\n📊 Generating contrastive pairs from task: {args.task_name}")
@@ -26,58 +23,14 @@ def execute_generate_pairs_from_task(args):
 
     try:
         print(f"\n🔄 Loading task '{args.task_name}'...")
-
-        # Check if task is in HuggingFace manifest (doesn't need lm-eval loading)
-        task_name_lower = args.task_name.lower()
-        is_hf_task = task_name_lower in {k.lower() for k in HF_EXTRACTORS.keys()}
-
-        if is_hf_task:
-            # HuggingFace task - skip lm-eval loading, go directly to extractor
-            print(f"   Found in HuggingFace manifest, using HF extractor...")
-            print(f"   🔨 Building contrastive pairs...")
-            pairs = lm_build_contrastive_pairs(
-                task_name=args.task_name,
-                lm_eval_task=None,  # HF extractors don't need lm_eval_task
-                limit=args.limit,
-            )
-            pairs_task_name = args.task_name
-        else:
-            # lm-eval task - load via LMEvalDataLoader
-            from wisent.core.data_loaders.loaders.lm_loader import LMEvalDataLoader
-            loader = LMEvalDataLoader()
-            task_obj = loader.load_lm_eval_task(args.task_name)
-
-            # Handle both lm-eval tasks (dict or ConfigurableTask)
-            if isinstance(task_obj, dict):
-                # lm-eval task group with subtasks
-                if len(task_obj) != 1:
-                    keys = ", ".join(sorted(task_obj.keys()))
-                    raise InvalidDataFormatError(
-                        reason=f"Task '{args.task_name}' returned {len(task_obj)} subtasks ({keys}). "
-                               "Specify an explicit subtask, e.g. 'benchmark/subtask'."
-                    )
-                (subname, task), = task_obj.items()
-                pairs_task_name = subname
-
-                # Generate contrastive pairs using lm-eval interface
-                print(f"   🔨 Building contrastive pairs...")
-                pairs = lm_build_contrastive_pairs(
-                    task_name=pairs_task_name,
-                    lm_eval_task=task,
-                    limit=args.limit,
-                )
-            else:
-                # Single lm-eval task (ConfigurableTask), not wrapped in dict
-                task = task_obj
-                pairs_task_name = args.task_name
-
-                # Generate contrastive pairs using lm-eval interface
-                print(f"   🔨 Building contrastive pairs...")
-                pairs = lm_build_contrastive_pairs(
-                    task_name=pairs_task_name,
-                    lm_eval_task=task,
-                    limit=args.limit,
-                )
+        print(f"   🔨 Building contrastive pairs...")
+        
+        # Use unified loader - handles HF, lm-eval, and group tasks automatically
+        pairs = build_contrastive_pairs(
+            task_name=args.task_name,
+            limit=args.limit,
+        )
+        pairs_task_name = args.task_name
 
         print(f"   ✓ Generated {len(pairs)} contrastive pairs")
 

wisent/core/cli/generate_vector_from_task.py

@@ -198,6 +198,10 @@ def execute_generate_vector_from_task(args):
             verbose=args.verbose,
             timing=args.timing,
             accept_low_quality_vector=getattr(args, 'accept_low_quality_vector', False),
+            # Universal Subspace options for PRISM/TITAN
+            auto_num_directions=getattr(args, 'auto_num_directions', False),
+            use_universal_basis_init=getattr(args, 'use_universal_basis_init', False),
+            num_directions=getattr(args, 'num_directions', 3),
         )
         
         execute_create_steering_vector(vector_args)

wisent/core/cli/geometry_search.py

@@ -0,0 +1,137 @@
+"""Run geometry search across benchmarks to find unified goodness direction."""
+
+import json
+import sys
+import os
+from pathlib import Path
+
+
+def execute_geometry_search(args):
+    """Execute the geometry-search command."""
+    print(f"\n{'='*60}")
+    print("GEOMETRY SEARCH")
+    print(f"{'='*60}")
+    print(f"Model: {args.model}")
+    print(f"Output: {args.output}")
+    print(f"Pairs per benchmark: {args.pairs_per_benchmark}")
+    print(f"Max layer combo size: {args.max_layer_combo_size}")
+    
+    # Import dependencies
+    from wisent.core.models.wisent_model import WisentModel
+    from wisent.core.geometry_search_space import GeometrySearchSpace, GeometrySearchConfig
+    from wisent.core.geometry_runner import GeometryRunner
+    from wisent.core.activations.extraction_strategy import ExtractionStrategy
+    
+    # Parse strategies
+    if args.strategies:
+        strategy_names = [s.strip() for s in args.strategies.split(',')]
+        strategies = [ExtractionStrategy(s) for s in strategy_names]
+        print(f"Strategies: {strategy_names}")
+    else:
+        strategies = None  # Use default (all 7)
+        print("Strategies: all 7 default strategies")
+    
+    # Parse benchmarks
+    if args.benchmarks:
+        if args.benchmarks.endswith('.txt'):
+            with open(args.benchmarks) as f:
+                benchmarks = [line.strip() for line in f if line.strip()]
+        else:
+            benchmarks = [b.strip() for b in args.benchmarks.split(',')]
+        print(f"Benchmarks: {len(benchmarks)} specified")
+    else:
+        benchmarks = None  # Use default (all)
+        print("Benchmarks: all available")
+    
+    # Create config
+    config = GeometrySearchConfig(
+        pairs_per_benchmark=args.pairs_per_benchmark,
+        max_layer_combo_size=args.max_layer_combo_size,
+        random_seed=args.seed,
+        cache_activations=True,
+        cache_dir=args.cache_dir,
+    )
+    
+    # Create search space
+    search_space = GeometrySearchSpace(
+        models=[args.model],
+        strategies=strategies,
+        benchmarks=benchmarks,
+        config=config,
+    )
+    
+    print(f"\n{search_space.summary()}")
+    
+    # Load model
+    print(f"\nLoading model {args.model}...")
+    model = WisentModel(args.model, device=args.device)
+    print(f"Model loaded: {model.num_layers} layers, hidden_size={model.hidden_size}")
+    
+    # Create runner
+    cache_dir = args.cache_dir or f"/tmp/wisent_geometry_cache_{args.model.replace('/', '_')}"
+    runner = GeometryRunner(search_space, model, cache_dir=cache_dir)
+    
+    # Run search
+    print(f"\nStarting geometry search...")
+    results = runner.run(show_progress=True)
+    
+    # Save results
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    results.save(str(output_path))
+    print(f"\nResults saved to: {output_path}")
+    
+    # Print summary
+    print(f"\n{'='*60}")
+    print("SUMMARY")
+    print(f"{'='*60}")
+    print(f"Total time: {results.total_time_seconds / 3600:.2f} hours")
+    print(f"  Extraction: {results.extraction_time_seconds / 3600:.2f} hours")
+    print(f"  Testing: {results.test_time_seconds / 60:.1f} minutes")
+    print(f"Benchmarks tested: {results.benchmarks_tested}")
+    print(f"Strategies tested: {results.strategies_tested}")
+    print(f"Layer combos tested: {results.layer_combos_tested}")
+    
+    print(f"\nStructure distribution:")
+    for struct, count in sorted(results.get_structure_distribution().items(), key=lambda x: -x[1]):
+        pct = 100 * count / results.layer_combos_tested
+        print(f"  {struct}: {count} ({pct:.1f}%)")
+    
+    print(f"\nTop 10 by linear score:")
+    for r in results.get_best_by_linear_score(10):
+        print(f"  {r.benchmark}/{r.strategy} layers={r.layers}: linear={r.linear_score:.3f} best={r.best_structure}")
+    
+    print(f"\nTop 10 by cone score:")
+    for r in results.get_best_by_structure('cone', 10):
+        print(f"  {r.benchmark}/{r.strategy} layers={r.layers}: cone={r.cone_score:.3f} best={r.best_structure}")
+    
+    # Summary by benchmark
+    print(f"\nSummary by benchmark (avg linear score):")
+    by_bench = results.get_summary_by_benchmark()
+    sorted_benches = sorted(by_bench.items(), key=lambda x: -x[1]['mean'])[:20]
+    for bench, stats in sorted_benches:
+        print(f"  {bench}: mean={stats['mean']:.3f} max={stats['max']:.3f}")
+    
+    print(f"\n{'='*60}")
+    print("CONCLUSION")
+    print(f"{'='*60}")
+    
+    # Determine if unified direction exists
+    dist = results.get_structure_distribution()
+    total = sum(dist.values())
+    linear_pct = 100 * dist.get('linear', 0) / total if total > 0 else 0
+    cone_pct = 100 * dist.get('cone', 0) / total if total > 0 else 0
+    orthogonal_pct = 100 * dist.get('orthogonal', 0) / total if total > 0 else 0
+    
+    if linear_pct > 50:
+        print(f"UNIFIED LINEAR DIRECTION EXISTS ({linear_pct:.1f}% linear)")
+        print("Recommendation: Use CAA with the best layer/strategy combination")
+    elif cone_pct > 30:
+        print(f"CONE STRUCTURE DETECTED ({cone_pct:.1f}% cone)")
+        print("Recommendation: Use PRISM with multi-directional steering")
+    elif orthogonal_pct > 50:
+        print(f"ORTHOGONAL STRUCTURE ({orthogonal_pct:.1f}% orthogonal)")
+        print("Recommendation: No unified direction - use per-benchmark directions or TITAN")
+    else:
+        print("MIXED STRUCTURE - no clear unified direction")
+        print("Recommendation: Use TITAN for adaptive multi-component steering")

wisent/core/cli/get_activations.py

@@ -10,8 +10,8 @@ def execute_get_activations(args):
     """Execute the get-activations command - load pairs and collect activations."""
     from wisent.core.models.wisent_model import WisentModel
     from wisent.core.activations.activations_collector import ActivationCollector
-    from wisent.core.activations.core.atoms import ActivationAggregationStrategy
-    from wisent.core.activations.prompt_construction_strategy import PromptConstructionStrategy
+    from wisent.core.activations.extraction_strategy import ExtractionStrategy
+    
     from wisent.core.contrastive_pairs.core.pair import ContrastivePair
     from wisent.core.contrastive_pairs.core.response import PositiveResponse, NegativeResponse
     from wisent.core.contrastive_pairs.core.set import ContrastivePairSet
@@ -52,44 +52,24 @@ def execute_get_activations(args):
         model = WisentModel(args.model, device=args.device)
         print(f"   ✓ Model loaded with {model.num_layers} layers")
 
-        # 3. Determine layers to collect
+        # 3. Determine layers to collect (1-indexed for API)
         if args.layers is None:
-            # Default: use middle layer (1-indexed for API)
-            layers = [model.num_layers // 2 + 1]
+            # Default: use ALL layers (1-indexed: 1..num_layers)
+            layers = list(range(1, model.num_layers + 1))
         elif args.layers.lower() == 'all':
-            # Use 1-indexed layers for API (1 to num_layers)
+            # Use all layers (1-indexed: 1..num_layers)
             layers = list(range(1, model.num_layers + 1))
         else:
             layers = [int(l.strip()) for l in args.layers.split(',')]
 
-        # Convert to strings for API
+        # Convert to strings for API (1-indexed)
         layer_strs = [str(l) for l in layers]
 
         print(f"\n🎯 Collecting activations from {len(layers)} layer(s): {layers}")
 
-        # 4. Set up aggregation strategy
-        aggregation_map = {
-            'average': 'MEAN_POOLING',
-            'final': 'LAST_TOKEN',
-            'first': 'FIRST_TOKEN',
-            'max': 'MAX_POOLING',
-            'min': 'MAX_POOLING',
-        }
-        aggregation_key = aggregation_map.get(args.token_aggregation.lower(), 'MEAN_POOLING')
-        aggregation_strategy = ActivationAggregationStrategy[aggregation_key]
-
-        # 5. Map prompt strategy string to enum
-        prompt_strategy_map = {
-            'chat_template': PromptConstructionStrategy.CHAT_TEMPLATE,
-            'direct_completion': PromptConstructionStrategy.DIRECT_COMPLETION,
-            'instruction_following': PromptConstructionStrategy.INSTRUCTION_FOLLOWING,
-            'multiple_choice': PromptConstructionStrategy.MULTIPLE_CHOICE,
-            'role_playing': PromptConstructionStrategy.ROLE_PLAYING,
-        }
-        prompt_strategy = prompt_strategy_map.get(args.prompt_strategy.lower(), PromptConstructionStrategy.CHAT_TEMPLATE)
-
-        print(f"   Token aggregation: {args.token_aggregation} ({aggregation_key})")
-        print(f"   Prompt strategy: {args.prompt_strategy}")
+        # 4. Get extraction strategy from args
+        extraction_strategy = ExtractionStrategy(getattr(args, 'extraction_strategy', 'chat_last'))
+        print(f"   Extraction strategy: {extraction_strategy.value}")
 
         # 5. Create pair set and reconstruct pairs
         pair_set = ContrastivePairSet(name=task_name, task_type=trait_label)
@@ -110,7 +90,7 @@ def execute_get_activations(args):
 
         # 6. Collect activations
         print(f"\n⚡ Collecting activations...")
-        collector = ActivationCollector(model=model, store_device="cpu")
+        collector = ActivationCollector(model=model)
 
         enriched_pairs = []
         for i, pair in enumerate(pair_set.pairs):
@@ -118,13 +98,9 @@ def execute_get_activations(args):
                 print(f"   Processing pair {i+1}/{len(pair_set.pairs)}...")
 
             # Collect activations for all requested layers at once
-            updated_pair = collector.collect_for_pair(
-                pair,
+            updated_pair = collector.collect(
+                pair, strategy=extraction_strategy,
                 layers=layer_strs,
-                aggregation=aggregation_strategy,
-                return_full_sequence=False,
-                normalize_layers=False,
-                prompt_strategy=prompt_strategy
             )
 
             enriched_pairs.append(updated_pair)