ai-critic 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

ai_critic/evaluators/explainability.py

@@ -0,0 +1,64 @@
+# explainability.py
+import numpy as np
+from sklearn.model_selection import cross_val_score
+from sklearn.base import clone
+
+from .validation import make_cv
+
+
+def evaluate(model, X, y, max_features=10):
+    """
+    Model-agnostic feature sensitivity analysis.
+    Measures how much performance drops when each feature is permuted.
+    """
+
+    cv = make_cv(y)
+
+    base_model = clone(model)
+    base_score = cross_val_score(base_model, X, y, cv=cv).mean()
+
+    sensitivities = []
+
+    for i in range(X.shape[1]):
+        X_permuted = X.copy()
+        np.random.shuffle(X_permuted[:, i])
+
+        permuted_model = clone(model)
+        score = cross_val_score(permuted_model, X_permuted, y, cv=cv).mean()
+
+        drop = base_score - score
+
+        sensitivities.append({
+            "feature_index": int(i),
+            "performance_drop": float(drop)
+        })
+
+    sensitivities.sort(
+        key=lambda x: x["performance_drop"],
+        reverse=True
+    )
+
+    top = sensitivities[:max_features]
+
+    verdict = "stable"
+    message = "No single feature dominates model behavior."
+
+    if top and top[0]["performance_drop"] > 0.30:
+        verdict = "feature_leakage_risk"
+        message = (
+            "Model is highly sensitive to a single feature, "
+            "which may indicate leakage or shortcut learning."
+        )
+    elif top and top[0]["performance_drop"] > 0.15:
+        verdict = "feature_dependency"
+        message = (
+            "Model depends strongly on a small subset of features."
+        )
+
+    return {
+        "baseline_score": float(base_score),
+        "top_sensitive_features": top,
+        "max_performance_drop": float(top[0]["performance_drop"]) if top else 0.0,
+        "verdict": verdict,
+        "message": message
+    }

ai_critic/evaluators/scoring.py

@@ -11,6 +11,10 @@ def compute_scores(report: dict) -> dict:
     robustness = report["details"]["robustness"]["verdict"]
     structural = report["details"]["config"]["structural_warnings"]
 
+    explainability = report["details"].get("explainability", {})
+    explain_verdict = explainability.get("verdict")
+    max_feature_drop = explainability.get("max_performance_drop", 0)
+
     if data_leakage:
         score -= 30
 
@@ -25,6 +29,11 @@ def compute_scores(report: dict) -> dict:
     if structural:
         score -= 10
 
+    if explain_verdict == "feature_leakage_risk":
+        score -= 20
+    elif explain_verdict == "feature_dependency":
+        score -= 10
+
     return {
         "global": max(0, min(100, score)),
         "components": {
@@ -35,5 +44,10 @@ def compute_scores(report: dict) -> dict:
                 "fragile": 65,
                 "misleading": 40
             }.get(robustness, 100),
+            "explainability": (
+                40 if explain_verdict == "feature_leakage_risk"
+                else 70 if explain_verdict == "feature_dependency"
+                else 100
+            )
         }
     }

ai_critic/evaluators/summary.py

@@ -10,6 +10,10 @@ class HumanSummary:
         robustness_verdict = report["robustness"].get("verdict")
         structural_warnings = report["config"]["structural_warnings"]
 
+        explainability = report.get("explainability", {})
+        explain_verdict = explainability.get("verdict")
+        max_feature_drop = explainability.get("max_performance_drop", 0)
+
         # =========================
         # Executive summary
         # =========================
@@ -18,11 +22,19 @@ class HumanSummary:
             risk_level = "high"
             deploy = False
             main_reason = "Strong evidence of data leakage inflating model performance."
+        elif explain_verdict == "feature_leakage_risk":
+            verdict = "❌ Unreliable"
+            risk_level = "high"
+            deploy = False
+            main_reason = (
+                "Model behavior is dominated by a single feature, "
+                "suggesting shortcut learning or leakage."
+            )
         elif robustness_verdict in ("fragile", "misleading") or structural_warnings:
             verdict = "⚠️ Risky"
             risk_level = "medium"
             deploy = False
-            main_reason = "Structural or robustness-related risks detected."
+            main_reason = "Structural, robustness, or dependency-related risks detected."
         else:
             verdict = "✅ Acceptable"
             risk_level = "low"
@@ -71,6 +83,21 @@ class HumanSummary:
                 "Reduce model complexity or adjust hyperparameters."
             )
 
+        if explain_verdict == "feature_leakage_risk":
+            key_risks.append(
+                f"Single feature causes a {max_feature_drop:.2f} performance drop when permuted."
+            )
+            recommendations.append(
+                "Remove or heavily regularize the dominant feature and retrain."
+            )
+        elif explain_verdict == "feature_dependency":
+            key_risks.append(
+                "Model relies disproportionately on a small subset of features."
+            )
+            recommendations.append(
+                "Increase regularization or collect more diverse data."
+            )
+
         if robustness_verdict == "misleading":
             key_risks.append(
                 "Robustness metrics are misleading due to inflated baseline performance."
@@ -92,7 +119,8 @@ class HumanSummary:
                 "data_leakage": leakage,
                 "suspicious_cv": perfect_cv,
                 "structural_risk": bool(structural_warnings),
-                "robustness_verdict": robustness_verdict
+                "robustness_verdict": robustness_verdict,
+                "explainability_verdict": explain_verdict
             },
             "recommendations": recommendations
         }

{ai_critic-1.1.0.dist-info → ai_critic-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-critic
-Version: 1.1.0
+Version: 1.2.0
 Summary: Fast AI evaluator for scikit-learn models
 Author-email: Luiz Seabra <filipedemarco@yahoo.com>
 Requires-Python: >=3.9
@@ -91,6 +91,7 @@ Each pillar contributes signals used later in the **deployment gate**.
 
 ---
 
+
 ### Full Technical & Visual Analysis
 
 To access **all internal diagnostics**, including plots and recommendations, use `view="all"`.

{ai_critic-1.1.0.dist-info → ai_critic-1.2.0.dist-info}/RECORD

@@ -4,14 +4,15 @@ ai_critic/evaluators/__init__.py,sha256=ri6InmL8_LIcO-JZpU_gEFKLO4URdqo3z6rh7fV6
 ai_critic/evaluators/adapters.py,sha256=8Xw9Ccg1iGVNwVQDGVIqhWj5-Sg6evqCZhg21u8EP20,3068
 ai_critic/evaluators/config.py,sha256=gBXaS8Qxl14f40JnvMWgA0Z0SGEtbCuCHpTOPem0H90,1163
 ai_critic/evaluators/data.py,sha256=YAK5NkwCeJOny_UueZ5ALwvEcRDIbEck404eV2oqWnc,1871
+ai_critic/evaluators/explainability.py,sha256=UWbcb5uVI78d1ljfdrWd2DrjlwEz1y9CeVtkukefEfA,1759
 ai_critic/evaluators/performance.py,sha256=1CQx5DueK0XkelYyJnAGRJ3AjQtjsKeW8_1JQZqKVOI,1973
 ai_critic/evaluators/robustness.py,sha256=mfVQ67Z6t6aRvtIq-XQEQYbwvyf8UefM1myeOGVrnAE,1869
-ai_critic/evaluators/scoring.py,sha256=GBkmDa5Q6RZY4hJfzrCbxbBopsOsRjsNtzyoQHqgWHA,1046
-ai_critic/evaluators/summary.py,sha256=O9ZCrph93VV6pFcMIx2a7DizPIccRUqbGcUZ6oDmOLs,3791
+ai_critic/evaluators/scoring.py,sha256=9rgkCXKKm9G1Lfwn5i9HcsJTN5OUjxMycOUzhWkp_2g,1576
+ai_critic/evaluators/summary.py,sha256=H9rU9tXAXqyQ34L6bOOOHrdIapSq71gcjjc8jfyJMq4,5003
 ai_critic/evaluators/validation.py,sha256=rnzRwD78Cugey33gl9geE8JoBURsKEEnqrIOhBZv0LY,904
 ai_critic/sessions/__init__.py,sha256=Yp7mphSPJwt8a4cJgcQNErqwqHVuP_xAJODrs0y0Abw,72
 ai_critic/sessions/store.py,sha256=65m9WXFVFWv4pPzvXV4l8zLHoHWMfCGe6eHh4X-8agY,947
-ai_critic-1.1.0.dist-info/METADATA,sha256=gIxyPwmDrmkqzZdLILBkq6uOl_2UAu5QRIV7Xis2rGc,8114
-ai_critic-1.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ai_critic-1.1.0.dist-info/top_level.txt,sha256=TRyZkm1vyLLcFDg_80yeg5cHvPis_oW1Ti170417jkw,10
-ai_critic-1.1.0.dist-info/RECORD,,
+ai_critic-1.2.0.dist-info/METADATA,sha256=s0XYw_E7ZoVBhF74lyhQsFk_bcyJWY3eo8Yk5E97tZ4,8115
+ai_critic-1.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ai_critic-1.2.0.dist-info/top_level.txt,sha256=TRyZkm1vyLLcFDg_80yeg5cHvPis_oW1Ti170417jkw,10
+ai_critic-1.2.0.dist-info/RECORD,,