xplia 1.0.1__py3-none-any.whl

xplia/explainers/unified_explainer_utils.py

@@ -0,0 +1,309 @@
+"""
+Fonctions utilitaires avancées pour l'UnifiedExplainer de XPLIA
+===============================================================
+
+Ce module fournit les méthodes auxiliaires nécessaires au bon fonctionnement
+de l'UnifiedExplainer, notamment la détection de framework, la sélection optimale
+des méthodes d'explicabilité, la gestion de conformité, et les utilitaires d'évaluation.
+"""
+import inspect
+import logging
+import numpy as np
+from typing import Dict, List, Any, Union, Optional, Tuple, Set
+
+from ..core.base import ExplainabilityMethod, AudienceLevel
+
+logger = logging.getLogger(__name__)
+
+def detect_model_framework(model: Any) -> str:
+    """
+    Détecte automatiquement le framework du modèle (scikit-learn, TensorFlow, PyTorch, etc.)
+    
+    Args:
+        model: Le modèle à analyser
+        
+    Returns:
+        str: Identifiant du framework ('sklearn', 'tensorflow', 'pytorch', 'xgboost', etc.)
+    """
+    module_name = model.__class__.__module__.split('.')[0].lower()
+    
+    framework_mapping = {
+        'sklearn': 'sklearn',
+        'torch': 'pytorch',
+        'tensorflow': 'tensorflow',
+        'tf': 'tensorflow',
+        'keras': 'tensorflow',
+        'xgboost': 'xgboost',
+        'lightgbm': 'lightgbm',
+        'catboost': 'catboost'
+    }
+    
+    for key, value in framework_mapping.items():
+        if key in module_name:
+            return value
+    
+    # Détection avancée par l'analyse des méthodes/attributs
+    if hasattr(model, 'predict_proba') and hasattr(model, 'feature_importances_'):
+        return 'sklearn-like'
+    elif hasattr(model, 'forward') and hasattr(model, 'parameters'):
+        return 'pytorch-like'
+    elif hasattr(model, 'predict') and hasattr(model, 'layers'):
+        return 'tensorflow-like'
+    
+    return 'unknown'
+
+def detect_is_classifier(model: Any) -> bool:
+    """
+    Détecte si un modèle est un classificateur ou un régresseur
+    
+    Args:
+        model: Le modèle à analyser
+        
+    Returns:
+        bool: True si classificateur, False si régresseur
+    """
+    # Analyse du nom de la classe
+    class_name = model.__class__.__name__.lower()
+    if 'classifier' in class_name:
+        return True
+    elif 'regressor' in class_name:
+        return False
+    
+    # Analyse des méthodes disponibles
+    if hasattr(model, 'predict_proba'):
+        return True
+    
+    # Pour les modèles Tensorflow/Keras
+    if hasattr(model, 'output_shape'):
+        # Classificateur si sortie multi-classes ou activation softmax/sigmoid
+        try:
+            last_layer = model.layers[-1]
+            if hasattr(last_layer, 'activation'):
+                if last_layer.activation.__name__ in ['softmax', 'sigmoid']:
+                    return True
+        except (AttributeError, IndexError):
+            pass
+    
+    # Par défaut, on suppose régresseur (supposition conservatrice)
+    return False
+
+def get_supported_model_types() -> List[str]:
+    """
+    Renvoie la liste exhaustive des types de modèles supportés par l'UnifiedExplainer
+    
+    Returns:
+        List[str]: Liste des types de modèles supportés
+    """
+    return [
+        # scikit-learn
+        'RandomForestClassifier', 'RandomForestRegressor',
+        'GradientBoostingClassifier', 'GradientBoostingRegressor',
+        'LogisticRegression', 'LinearRegression',
+        'SVC', 'SVR',
+        'DecisionTreeClassifier', 'DecisionTreeRegressor',
+        'KNeighborsClassifier', 'KNeighborsRegressor',
+        
+        # XGBoost
+        'XGBClassifier', 'XGBRegressor', 'Booster',
+        
+        # LightGBM
+        'LGBMClassifier', 'LGBMRegressor',
+        
+        # CatBoost
+        'CatBoostClassifier', 'CatBoostRegressor',
+        
+        # TensorFlow/Keras
+        'Sequential', 'Model', 'Functional',
+        
+        # PyTorch
+        'Module', 'Sequential',
+        
+        # Ensemble et pipeline
+        'VotingClassifier', 'VotingRegressor',
+        'StackingClassifier', 'StackingRegressor',
+        'BaggingClassifier', 'BaggingRegressor',
+        'Pipeline',
+        
+        # Autres
+        'TabularPredictor',  # AutoGluon
+        'AutoMLPredictor',   # Auto-Sklearn
+        'H2OEstimator'       # H2O
+    ]
+
+def setup_compliance(compliance_requirements: List[str]) -> Dict[str, Dict[str, Any]]:
+    """
+    Configure et initialise les exigences de conformité réglementaire
+    
+    Args:
+        compliance_requirements: Liste des réglementations à suivre ('rgpd', 'ai_act', 'hipaa', etc.)
+        
+    Returns:
+        Dict[str, Dict[str, Any]]: Configuration des exigences de conformité
+    """
+    compliance_config = {}
+    
+    for req in compliance_requirements:
+        if req.lower() == 'rgpd' or req.lower() == 'gdpr':
+            compliance_config['rgpd'] = {
+                'enabled': True,
+                'explanation_right': True,
+                'data_minimization': True,
+                'audit_trail': True,
+                'user_access': True
+            }
+        elif req.lower() == 'ai_act':
+            compliance_config['ai_act'] = {
+                'enabled': True,
+                'risk_category': 'high',
+                'documentation': True,
+                'human_oversight': True,
+                'technical_robustness': True,
+                'record_keeping': True,
+                'transparency': True
+            }
+        elif req.lower() == 'hipaa':
+            compliance_config['hipaa'] = {
+                'enabled': True,
+                'phi_protection': True,
+                'access_logs': True,
+                'authorizations': True
+            }
+    
+    return compliance_config
+
+def initialize_weights(weights: Dict[ExplainabilityMethod, float], 
+                     methods: List[ExplainabilityMethod],
+                     model_type: str) -> Dict[ExplainabilityMethod, float]:
+    """
+    Initialise ou ajuste les poids des différentes méthodes d'explicabilité
+    
+    Args:
+        weights: Dictionnaire de poids fourni par l'utilisateur
+        methods: Liste des méthodes d'explicabilité utilisées
+        model_type: Type de modèle ('sklearn', 'tensorflow', etc.)
+        
+    Returns:
+        Dict[ExplainabilityMethod, float]: Poids optimisés pour les méthodes
+    """
+    if weights and all(method in weights for method in methods):
+        # Normalisation des poids fournis
+        total = sum(weights.values())
+        return {method: weight / total for method, weight in weights.items()}
+    
+    # Poids par défaut selon le type de modèle
+    default_weights = {
+        # Modèles basés sur les arbres
+        'tree_based': {
+            ExplainabilityMethod.SHAP: 0.35,
+            ExplainabilityMethod.FEATURE_IMPORTANCE: 0.25,
+            ExplainabilityMethod.LIME: 0.15,
+            ExplainabilityMethod.COUNTERFACTUAL: 0.15,
+            ExplainabilityMethod.PARTIAL_DEPENDENCE: 0.1
+        },
+        # Modèles linéaires
+        'linear': {
+            ExplainabilityMethod.FEATURE_IMPORTANCE: 0.4,
+            ExplainabilityMethod.SHAP: 0.25,
+            ExplainabilityMethod.LIME: 0.2,
+            ExplainabilityMethod.COUNTERFACTUAL: 0.15
+        },
+        # Réseaux de neurones
+        'neural_network': {
+            ExplainabilityMethod.SHAP: 0.3,
+            ExplainabilityMethod.LIME: 0.25,
+            ExplainabilityMethod.ATTENTION: 0.2,
+            ExplainabilityMethod.COUNTERFACTUAL: 0.15,
+            ExplainabilityMethod.GRADIENT: 0.1
+        }
+    }
+    
+    # Sélection du profil de poids approprié
+    if 'tree' in model_type or model_type in ['xgboost', 'lightgbm', 'catboost']:
+        weights_profile = default_weights['tree_based']
+    elif 'linear' in model_type:
+        weights_profile = default_weights['linear']
+    elif 'tensorflow' in model_type or 'pytorch' in model_type:
+        weights_profile = default_weights['neural_network']
+    else:
+        # Profil générique
+        weights_profile = default_weights['tree_based']
+    
+    # Filtrer pour ne garder que les méthodes disponibles
+    result_weights = {method: weights_profile.get(method, 0.1) for method in methods}
+    
+    # Normalisation
+    total = sum(result_weights.values())
+    return {method: weight / total for method, weight in result_weights.items()}
+
+def select_optimal_methods(methods: Optional[List[ExplainabilityMethod]], 
+                         model_framework: str,
+                         is_classifier: bool) -> List[ExplainabilityMethod]:
+    """
+    Sélectionne les méthodes d'explicabilité optimales selon le modèle
+    
+    Args:
+        methods: Liste des méthodes spécifiées par l'utilisateur (ou None)
+        model_framework: Type de framework du modèle
+        is_classifier: Si le modèle est un classificateur
+        
+    Returns:
+        List[ExplainabilityMethod]: Méthodes d'explicabilité optimales
+    """
+    if methods:
+        return methods
+    
+    # Méthodes recommandées selon le type de modèle
+    if 'sklearn' in model_framework or model_framework in ['xgboost', 'lightgbm', 'catboost']:
+        methods = [
+            ExplainabilityMethod.SHAP,
+            ExplainabilityMethod.FEATURE_IMPORTANCE,
+            ExplainabilityMethod.LIME,
+            ExplainabilityMethod.PARTIAL_DEPENDENCE
+        ]
+        # Ajouter les counterfactuals pour les classificateurs
+        if is_classifier:
+            methods.append(ExplainabilityMethod.COUNTERFACTUAL)
+            
+    elif 'tensorflow' in model_framework or 'keras' in model_framework:
+        methods = [
+            ExplainabilityMethod.SHAP,
+            ExplainabilityMethod.LIME,
+            ExplainabilityMethod.GRADIENT
+        ]
+        # Ajouter l'attention pour les modèles qui la supportent
+        methods.append(ExplainabilityMethod.ATTENTION)
+        
+    elif 'pytorch' in model_framework:
+        methods = [
+            ExplainabilityMethod.SHAP,
+            ExplainabilityMethod.LIME,
+            ExplainabilityMethod.GRADIENT
+        ]
+        # Ajouter l'attention pour les modèles qui la supportent
+        methods.append(ExplainabilityMethod.ATTENTION)
+        
+    else:
+        # Méthodes génériques qui fonctionnent avec la plupart des modèles
+        methods = [
+            ExplainabilityMethod.SHAP,
+            ExplainabilityMethod.LIME,
+            ExplainabilityMethod.FEATURE_IMPORTANCE
+        ]
+    
+    return methods
+
+def load_plugins():
+    """
+    Charge les plugins d'explicabilité personnalisés
+    
+    Returns:
+        Dict[str, Any]: Dictionnaire des plugins chargés
+    """
+    try:
+        from ..plugins import PluginRegistry
+        plugins = PluginRegistry.auto_discover()
+        logger.info(f"Plugins d'explicabilité chargés: {len(plugins)}")
+        return plugins
+    except (ImportError, AttributeError) as e:
+        logger.warning(f"Erreur lors du chargement des plugins: {str(e)}")
+        return {}

xplia/integrations/__init__.py

@@ -0,0 +1,3 @@
+"""ML Ops integrations for XPLIA."""
+
+__all__ = ['mlflow_integration', 'wandb_integration']

xplia/integrations/mlflow_integration.py

@@ -0,0 +1,331 @@
+"""
+MLflow integration for XPLIA.
+
+Log explanations, compliance reports, and trust metrics to MLflow.
+
+Author: XPLIA Team
+License: MIT
+"""
+
+from typing import Any, Dict, Optional
+import tempfile
+from pathlib import Path
+
+try:
+    import mlflow
+    from mlflow.tracking import MlflowClient
+except ImportError:
+    raise ImportError(
+        "MLflow not installed. Install with: pip install xplia[mlops]"
+    )
+
+from xplia.core.base import ExplanationResult
+
+
+class XPLIAMLflowLogger:
+    """
+    Logger for XPLIA explanations and metrics to MLflow.
+
+    Examples
+    --------
+    >>> logger = XPLIAMLflowLogger()
+    >>> logger.log_explanation(explanation, run_id='...')
+    """
+
+    def __init__(self, tracking_uri: Optional[str] = None, experiment_name: Optional[str] = None):
+        """
+        Initialize MLflow logger.
+
+        Parameters
+        ----------
+        tracking_uri : str, optional
+            MLflow tracking server URI.
+
+        experiment_name : str, optional
+            MLflow experiment name.
+        """
+        if tracking_uri:
+            mlflow.set_tracking_uri(tracking_uri)
+
+        if experiment_name:
+            mlflow.set_experiment(experiment_name)
+
+        self.client = MlflowClient()
+
+    def log_explanation(
+        self,
+        explanation: ExplanationResult,
+        run_id: Optional[str] = None,
+        artifact_path: str = "explanations"
+    ):
+        """
+        Log explanation to MLflow.
+
+        Parameters
+        ----------
+        explanation : ExplanationResult
+            Explanation to log.
+
+        run_id : str, optional
+            MLflow run ID. If None, uses active run.
+
+        artifact_path : str
+            Path within run's artifact directory.
+        """
+        if run_id is None:
+            run = mlflow.active_run()
+            if run is None:
+                raise ValueError("No active MLflow run. Call mlflow.start_run() first or provide run_id.")
+            run_id = run.info.run_id
+
+        # Log parameters
+        mlflow.log_param("explanation_method", explanation.method, run_id=run_id)
+
+        # Log metrics
+        if hasattr(explanation, 'quality_metrics') and explanation.quality_metrics:
+            for metric_name, metric_value in explanation.quality_metrics.items():
+                if isinstance(metric_value, (int, float)):
+                    mlflow.log_metric(f"explanation_{metric_name}", metric_value, run_id=run_id)
+
+        # Log feature importance as artifact
+        if 'feature_importance' in explanation.explanation_data:
+            import json
+            import numpy as np
+
+            importance = explanation.explanation_data['feature_importance']
+            feature_names = explanation.explanation_data.get('feature_names', [])
+
+            # Convert to serializable format
+            if isinstance(importance, np.ndarray):
+                importance = importance.tolist()
+
+            importance_dict = {
+                'method': explanation.method,
+                'feature_importance': dict(zip(feature_names, importance)) if feature_names else importance
+            }
+
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
+                json.dump(importance_dict, f, indent=2)
+                temp_path = f.name
+
+            mlflow.log_artifact(temp_path, artifact_path=artifact_path, run_id=run_id)
+            Path(temp_path).unlink()
+
+    def log_compliance_report(
+        self,
+        compliance_result: Any,
+        regulation: str,
+        run_id: Optional[str] = None,
+        artifact_path: str = "compliance"
+    ):
+        """
+        Log compliance report to MLflow.
+
+        Parameters
+        ----------
+        compliance_result : ComplianceResult
+            Compliance check result.
+
+        regulation : str
+            Regulation name ('gdpr', 'ai_act', 'hipaa').
+
+        run_id : str, optional
+            MLflow run ID.
+
+        artifact_path : str
+            Path within run's artifact directory.
+        """
+        if run_id is None:
+            run = mlflow.active_run()
+            if run is None:
+                raise ValueError("No active MLflow run.")
+            run_id = run.info.run_id
+
+        # Log compliance score
+        mlflow.log_metric(f"{regulation}_compliance_score", compliance_result.score, run_id=run_id)
+        mlflow.log_param(f"{regulation}_compliant", compliance_result.compliant, run_id=run_id)
+
+        # Export and log full report
+        with tempfile.TemporaryDirectory() as tmpdir:
+            report_path = Path(tmpdir) / f"{regulation}_compliance_report.pdf"
+            compliance_result.export(str(report_path), format='pdf')
+            mlflow.log_artifact(str(report_path), artifact_path=artifact_path, run_id=run_id)
+
+    def log_trust_metrics(
+        self,
+        trust_metrics: Dict[str, float],
+        run_id: Optional[str] = None
+    ):
+        """
+        Log trust evaluation metrics to MLflow.
+
+        Parameters
+        ----------
+        trust_metrics : dict
+            Trust metrics (uncertainty, confidence, etc.).
+
+        run_id : str, optional
+            MLflow run ID.
+        """
+        if run_id is None:
+            run = mlflow.active_run()
+            if run is None:
+                raise ValueError("No active MLflow run.")
+            run_id = run.info.run_id
+
+        for metric_name, metric_value in trust_metrics.items():
+            if isinstance(metric_value, (int, float)):
+                mlflow.log_metric(f"trust_{metric_name}", metric_value, run_id=run_id)
+
+    def log_fairwashing_detection(
+        self,
+        fairwashing_result: Any,
+        run_id: Optional[str] = None
+    ):
+        """
+        Log fairwashing detection results.
+
+        Parameters
+        ----------
+        fairwashing_result : FairwashingResult
+            Detection result.
+
+        run_id : str, optional
+            MLflow run ID.
+        """
+        if run_id is None:
+            run = mlflow.active_run()
+            if run is None:
+                raise ValueError("No active MLflow run.")
+            run_id = run.info.run_id
+
+        mlflow.log_param("fairwashing_detected", fairwashing_result.detected, run_id=run_id)
+
+        if fairwashing_result.detected:
+            mlflow.log_param("fairwashing_types", ",".join(fairwashing_result.fairwashing_types), run_id=run_id)
+            mlflow.log_metric("fairwashing_severity", fairwashing_result.severity, run_id=run_id)
+
+    def start_run_with_xplia(
+        self,
+        run_name: Optional[str] = None,
+        tags: Optional[Dict[str, str]] = None
+    ):
+        """
+        Start MLflow run with XPLIA tags.
+
+        Parameters
+        ----------
+        run_name : str, optional
+            Run name.
+
+        tags : dict, optional
+            Additional tags.
+
+        Returns
+        -------
+        ActiveRun
+            MLflow active run context.
+        """
+        default_tags = {
+            "xplia.enabled": "true",
+            "xplia.version": "1.0.0"
+        }
+
+        if tags:
+            default_tags.update(tags)
+
+        return mlflow.start_run(run_name=run_name, tags=default_tags)
+
+
+def log_model_with_explainability(
+    model: Any,
+    explainer: Any,
+    artifact_path: str = "model",
+    registered_model_name: Optional[str] = None,
+    sample_input: Optional[Any] = None
+):
+    """
+    Log model with explainability artifacts to MLflow.
+
+    Parameters
+    ----------
+    model : object
+        The ML model to log.
+
+    explainer : Explainer
+        XPLIA explainer for the model.
+
+    artifact_path : str
+        Artifact path in MLflow.
+
+    registered_model_name : str, optional
+        Name for model registry.
+
+    sample_input : array-like, optional
+        Sample input for signature inference.
+    """
+    # Log the model
+    mlflow.sklearn.log_model(
+        model,
+        artifact_path=artifact_path,
+        registered_model_name=registered_model_name
+    )
+
+    # Log explainer configuration
+    mlflow.log_param("explainer_method", explainer.method if hasattr(explainer, 'method') else 'unknown')
+
+    # Generate and log sample explanation
+    if sample_input is not None:
+        explanation = explainer.explain(sample_input)
+
+        logger = XPLIAMLflowLogger()
+        logger.log_explanation(explanation)
+
+    print(f"Model and explainability artifacts logged to MLflow")
+
+
+# Context manager for XPLIA + MLflow
+class XPLIAMLflowContext:
+    """
+    Context manager for XPLIA + MLflow integration.
+
+    Examples
+    --------
+    >>> with XPLIAMLflowContext(run_name="my_experiment") as logger:
+    ...     # Train model
+    ...     model.fit(X, y)
+    ...
+    ...     # Create explainer
+    ...     explainer = create_explainer(model, method='shap')
+    ...     explanation = explainer.explain(X_test)
+    ...
+    ...     # Log to MLflow
+    ...     logger.log_explanation(explanation)
+    """
+
+    def __init__(
+        self,
+        run_name: Optional[str] = None,
+        experiment_name: Optional[str] = None,
+        tracking_uri: Optional[str] = None
+    ):
+        self.run_name = run_name
+        self.experiment_name = experiment_name
+        self.tracking_uri = tracking_uri
+        self.logger = None
+        self.run = None
+
+    def __enter__(self):
+        self.logger = XPLIAMLflowLogger(
+            tracking_uri=self.tracking_uri,
+            experiment_name=self.experiment_name
+        )
+
+        self.run = self.logger.start_run_with_xplia(run_name=self.run_name)
+        self.run.__enter__()
+
+        return self.logger
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.run:
+            self.run.__exit__(exc_type, exc_val, exc_tb)