ai-metacognition-toolkit 0.3.0__py3-none-any.whl

ai_metacognition/__init__.py

@@ -0,0 +1,123 @@
+"""AI Metacognition Toolkit.
+
+A comprehensive toolkit for detecting and analyzing metacognitive capabilities
+in AI systems, particularly around situational awareness, evaluation contexts,
+and strategic underperformance (sandbagging).
+"""
+
+__version__ = "0.2.0"
+
+from .detectors import (
+    SituationalAwarenessDetector,
+    ObserverEffectMonitor,
+    SandbaggingDetector,
+    Alert,
+    AlertSeverity,
+    ContextType,
+    Interaction,
+    AlertHandler,
+    ConsoleAlertHandler,
+    PerformanceSample,
+    SandbaggingResult,
+)
+
+from .analyzers import (
+    CounterfactualCoTAnalyzer,
+    ModelAPI,
+    InterventionType,
+    ReasoningType,
+    ReasoningNode,
+)
+
+from .utils import (
+    extract_behavioral_features,
+    count_hedging_phrases,
+    detect_meta_commentary,
+    extract_reasoning_depth,
+    compute_kl_divergence,
+    compute_js_divergence,
+    cosine_similarity,
+    normalize_distribution,
+    bayesian_update,
+    compute_confidence_interval,
+    z_score,
+    assess_divergence_significance,
+    SignificanceLevel,
+    compute_beta_mean,
+    compute_beta_variance,
+    beta_mode,
+)
+
+from .visualizations import (
+    plot_awareness_over_time,
+    plot_causal_attribution,
+    plot_divergence_heatmap,
+    plot_feature_distributions,
+)
+
+# Probing module (requires torch, transformers, sklearn)
+# Import separately to avoid hard dependency
+try:
+    from .probing import (
+        ActivationHook,
+        SteeringVector,
+        LinearProbe,
+        extract_caa_vector,
+        extract_activations,
+    )
+    _PROBING_AVAILABLE = True
+except ImportError:
+    _PROBING_AVAILABLE = False
+
+__all__ = [
+    # Detectors
+    "SituationalAwarenessDetector",
+    "ObserverEffectMonitor",
+    "SandbaggingDetector",
+    "Alert",
+    "AlertSeverity",
+    "ContextType",
+    "Interaction",
+    "AlertHandler",
+    "ConsoleAlertHandler",
+    "PerformanceSample",
+    "SandbaggingResult",
+    # Analyzers
+    "CounterfactualCoTAnalyzer",
+    "ModelAPI",
+    "InterventionType",
+    "ReasoningType",
+    "ReasoningNode",
+    # Utils
+    "extract_behavioral_features",
+    "count_hedging_phrases",
+    "detect_meta_commentary",
+    "extract_reasoning_depth",
+    "compute_kl_divergence",
+    "compute_js_divergence",
+    "cosine_similarity",
+    "normalize_distribution",
+    "bayesian_update",
+    "compute_confidence_interval",
+    "z_score",
+    "assess_divergence_significance",
+    "SignificanceLevel",
+    "compute_beta_mean",
+    "compute_beta_variance",
+    "beta_mode",
+    # Visualizations
+    "plot_awareness_over_time",
+    "plot_causal_attribution",
+    "plot_divergence_heatmap",
+    "plot_feature_distributions",
+]
+
+# Add probing exports if available
+if _PROBING_AVAILABLE:
+    __all__.extend([
+        "ActivationHook",
+        "SteeringVector",
+        "LinearProbe",
+        "extract_caa_vector",
+        "extract_activations",
+    ])

ai_metacognition/analyzers/__init__.py

@@ -0,0 +1,24 @@
+"""Analyzers module for quantifying metacognitive capabilities.
+
+This module provides tools for analyzing and measuring various aspects of
+metacognition in AI systems, including confidence calibration, uncertainty
+quantification, and self-awareness metrics.
+"""
+
+from ai_metacognition.analyzers.base import BaseAnalyzer
+from ai_metacognition.analyzers.counterfactual_cot import (
+    CounterfactualCoTAnalyzer,
+    InterventionType,
+    ReasoningNode,
+    ReasoningType,
+)
+from ai_metacognition.analyzers.model_api import ModelAPI
+
+__all__ = [
+    "BaseAnalyzer",
+    "CounterfactualCoTAnalyzer",
+    "InterventionType",
+    "ReasoningNode",
+    "ReasoningType",
+    "ModelAPI",
+]

ai_metacognition/analyzers/base.py

@@ -0,0 +1,39 @@
+"""Base analyzer class for metacognition analysis."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+
+class BaseAnalyzer(ABC):
+    """Abstract base class for all analyzers.
+
+    All analyzer implementations should inherit from this class and implement
+    the analyze method.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the analyzer."""
+        self.name: str = self.__class__.__name__
+
+    @abstractmethod
+    def analyze(self, data: List[str]) -> Dict[str, Any]:
+        """Analyze the given data for metacognitive patterns.
+
+        Args:
+            data: List of text samples to analyze
+
+        Returns:
+            A dictionary containing analysis results with metrics and statistics
+
+        Raises:
+            NotImplementedError: If the method is not implemented
+        """
+        raise NotImplementedError("Subclasses must implement the analyze method")
+
+    def __repr__(self) -> str:
+        """Return string representation of the analyzer.
+
+        Returns:
+            String representation
+        """
+        return f"{self.__class__.__name__}()"