ai-critic 0.2.4__py3-none-any.whl → 1.0.0__py3-none-any.whl

ai_critic/critic.py

@@ -5,6 +5,8 @@ from ai_critic.evaluators import (
     performance
 )
 from ai_critic.evaluators.summary import HumanSummary
+from ai_critic.sessions import CriticSessionStore
+from ai_critic.evaluators.scoring import compute_scores
 
 
 class AICritic:
@@ -19,7 +21,7 @@ class AICritic:
     - Human-readable executive and technical summaries
     """
 
-    def __init__(self, model, X, y, random_state=None):
+    def __init__(self, model, X, y, random_state=None, session=None):
         """
         Parameters
         ----------
@@ -30,11 +32,15 @@ class AICritic:
             Target vector
         random_state : int or None
             Global seed for reproducibility (optional)
+        session : str or None
+            Optional session name for longitudinal comparison
         """
         self.model = model
         self.X = X
         self.y = y
         self.random_state = random_state
+        self.session = session
+        self._store = CriticSessionStore() if session else None
 
     def evaluate(self, view="all", plot=False):
         """
@@ -47,15 +53,10 @@ class AICritic:
             - "executive" : executive summary only
             - "technical" : technical summary only
             - "details" : low-level evaluator outputs
-            - list : subset of views (e.g. ["executive", "details"])
+            - list : subset of views
         plot : bool
-            - True : generate plots (learning curve, heatmap, robustness)
+            - True : generate plots
             - False : no plots
-
-        Returns
-        -------
-        dict
-            Evaluation payload according to selected view
         """
 
         # =========================
@@ -66,25 +67,23 @@ class AICritic:
         # -------------------------
         # Data analysis
         # -------------------------
-        data_report = data.evaluate(
+        details["data"] = data.evaluate(
             self.X,
             self.y,
             plot=plot
         )
-        details["data"] = data_report
 
         # -------------------------
         # Model configuration sanity
         # -------------------------
         details["config"] = config.evaluate(
             self.model,
-            n_samples=data_report["n_samples"],
-            n_features=data_report["n_features"]
+            n_samples=details["data"]["n_samples"],
+            n_features=details["data"]["n_features"]
         )
 
         # -------------------------
         # Performance evaluation
-        # (CV strategy inferred automatically)
         # -------------------------
         details["performance"] = performance.evaluate(
             self.model,
@@ -94,32 +93,36 @@ class AICritic:
         )
 
         # -------------------------
-        # Robustness & leakage analysis
+        # Robustness evaluation
         # -------------------------
         details["robustness"] = robustness.evaluate(
             self.model,
             self.X,
             self.y,
-            leakage_suspected=data_report["data_leakage"]["suspected"],
+            leakage_suspected=details["data"]["data_leakage"]["suspected"],
             plot=plot
         )
 
         # =========================
-        # Human-centered summaries
+        # Human summaries
         # =========================
         human_summary = HumanSummary().generate(details)
 
-        # =========================
-        # Full payload (PUBLIC API)
-        # =========================
         payload = {
             "executive": human_summary["executive_summary"],
             "technical": human_summary["technical_summary"],
             "details": details,
-            # Convenience shortcut (prevents KeyError in user code)
-            "performance": details["performance"]
+            "performance": details["performance"],
         }
 
+        # =========================
+        # Session persistence (optional)
+        # =========================
+        if self.session:
+            scores = compute_scores(payload)
+            payload["scores"] = scores
+            self._store.save(self.session, payload)
+
         # =========================
         # View selector
         # =========================
@@ -130,3 +133,117 @@ class AICritic:
             return {k: payload[k] for k in view if k in payload}
 
         return payload.get(view)
+
+    def compare_with(self, previous_session: str) -> dict:
+        """
+        Compare current session with a previous one.
+        """
+
+        if not self.session:
+            raise ValueError("Current session name not set.")
+
+        current = self._store.load(self.session)
+        previous = self._store.load(previous_session)
+
+        if not previous:
+            raise FileNotFoundError(
+                f"Session '{previous_session}' not found."
+            )
+
+        diff = {
+            "global_score": {
+                "current": current["scores"]["global"],
+                "previous": previous["scores"]["global"],
+                "delta": current["scores"]["global"] - previous["scores"]["global"],
+            },
+            "components": {}
+        }
+
+        for key, value in current["scores"]["components"].items():
+            prev_value = previous["scores"]["components"].get(key)
+            if prev_value is not None:
+                diff["components"][key] = {
+                    "current": value,
+                    "previous": prev_value,
+                    "delta": value - prev_value
+                }
+
+        return {
+            "current_session": self.session,
+            "previous_session": previous_session,
+            "score_diff": diff,
+            "note": (
+                "Score deltas indicate changes in risk profile, "
+                "not absolute model quality."
+            )
+        }
+
+    def deploy_decision(self):
+        """
+        Final deployment gate.
+        """
+
+        report = self.evaluate(view="all", plot=False)
+
+        data_risk = report["details"]["data"]["data_leakage"]["suspected"]
+        perfect_cv = report["details"]["performance"]["suspiciously_perfect"]
+        robustness_verdict = report["details"]["robustness"]["verdict"]
+        structural_warnings = report["details"]["config"]["structural_warnings"]
+
+        blocking_issues = []
+        risk_level = "low"
+
+        # Hard blockers
+        if data_risk and perfect_cv:
+            blocking_issues.append(
+                "Data leakage combined with suspiciously perfect CV score"
+            )
+            risk_level = "high"
+
+        if robustness_verdict == "misleading":
+            blocking_issues.append(
+                "Robustness results are misleading due to inflated baseline performance"
+            )
+            risk_level = "high"
+
+        if data_risk:
+            blocking_issues.append(
+                "Suspected target leakage in feature set"
+            )
+            risk_level = "high"
+
+        # Soft blockers
+        if risk_level != "high":
+            if robustness_verdict == "fragile":
+                blocking_issues.append(
+                    "Model performance degrades significantly under noise"
+                )
+                risk_level = "medium"
+
+            if perfect_cv:
+                blocking_issues.append(
+                    "Suspiciously perfect cross-validation score"
+                )
+                risk_level = "medium"
+
+            if structural_warnings:
+                blocking_issues.append(
+                    "Structural complexity risks detected in model configuration"
+                )
+                risk_level = "medium"
+
+        deploy = len(blocking_issues) == 0
+
+        confidence = 1.0
+        confidence -= 0.35 if data_risk else 0
+        confidence -= 0.25 if perfect_cv else 0
+        confidence -= 0.25 if robustness_verdict in ("fragile", "misleading") else 0
+        confidence -= 0.15 if structural_warnings else 0
+        confidence = max(0.0, round(confidence, 2))
+
+        return {
+            "deploy": deploy,
+            "risk_level": risk_level,
+            "blocking_issues": blocking_issues,
+            "confidence": confidence
+        }

ai_critic/evaluators/scoring.py

@@ -0,0 +1,39 @@
+def compute_scores(report: dict) -> dict:
+    """
+    Converts critic signals into a coarse 0–100 score.
+    Score is NOT an objective metric.
+    """
+
+    score = 100
+
+    data_leakage = report["details"]["data"]["data_leakage"]["suspected"]
+    perfect_cv = report["details"]["performance"]["suspiciously_perfect"]
+    robustness = report["details"]["robustness"]["verdict"]
+    structural = report["details"]["config"]["structural_warnings"]
+
+    if data_leakage:
+        score -= 30
+
+    if perfect_cv:
+        score -= 20
+
+    if robustness == "fragile":
+        score -= 15
+    elif robustness == "misleading":
+        score -= 25
+
+    if structural:
+        score -= 10
+
+    return {
+        "global": max(0, min(100, score)),
+        "components": {
+            "data_integrity": 0 if data_leakage else 100,
+            "validation": 70 if perfect_cv else 100,
+            "robustness": {
+                "stable": 100,
+                "fragile": 65,
+                "misleading": 40
+            }.get(robustness, 100),
+        }
+    }

ai_critic/sessions/__init__.py

@@ -0,0 +1,3 @@
+from .store import CriticSessionStore
+
+__all__ = ["CriticSessionStore"]

ai_critic/sessions/store.py

@@ -0,0 +1,33 @@
+import json
+from pathlib import Path
+from datetime import datetime
+
+
+class CriticSessionStore:
+    """
+    Simple local persistence layer for ai-critic sessions.
+    """
+
+    def __init__(self, base_dir: str | None = None):
+        self.base_dir = Path(
+            base_dir or Path.home() / ".ai_critic_sessions"
+        )
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+    def _session_path(self, name: str) -> Path:
+        return self.base_dir / f"{name}.json"
+
+    def save(self, name: str, payload: dict):
+        data = {
+            "timestamp": datetime.utcnow().isoformat(),
+            "payload": payload
+        }
+        with open(self._session_path(name), "w") as f:
+            json.dump(data, f, indent=2)
+
+    def load(self, name: str) -> dict | None:
+        path = self._session_path(name)
+        if not path.exists():
+            return None
+        with open(path) as f:
+            return json.load(f)["payload"]

ai_critic-1.0.0.dist-info/METADATA

@@ -0,0 +1,257 @@
+Metadata-Version: 2.4
+Name: ai-critic
+Version: 1.0.0
+Summary: Fast AI evaluator for scikit-learn models
+Author-email: Luiz Seabra <filipedemarco@yahoo.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+
+# ai-critic 🧠: The Quality Gate for Machine Learning Models
+
+**ai-critic** is a specialized **decision-making** tool designed to audit the reliability and readiness for deployment of scikit-learn–compatible Machine Learning models.
+
+Instead of merely measuring performance (accuracy, F1 score), **ai-critic** acts as a **Quality Gate**, actively probing the model to uncover *hidden risks* that commonly cause production failures — such as **data leakage**, **structural overfitting**, and **fragility under noise**.
+
+> **ai-critic does not ask “How good is this model?”**
+> It asks **“Can this model be trusted?”**
+
+---
+
+## 🚀  Getting Started (The Basics)
+
+This section is ideal for beginners who need a **fast and reliable verdict** on a trained model.
+
+###  Installation
+
+Install directly from PyPI:
+
+```bash
+pip install ai-critic
+```
+
+---
+
+###  The Quick Verdict
+
+With just a few lines of code, you obtain an **executive-level assessment** and a **deployment recommendation**.
+
+```python
+from ai_critic import AICritic
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.datasets import make_classification
+
+# 1. Prepare data and model
+X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
+model = RandomForestClassifier(max_depth=5, random_state=42)
+
+# 2. Initialize the Critic
+critic = AICritic(model, X, y)
+
+# 3. Run the audit (executive mode)
+report = critic.evaluate(view="executive")
+
+print(f"Verdict: {report['verdict']}")
+print(f"Risk Level: {report['risk_level']}")
+print(f"Main Reason: {report['main_reason']}")
+```
+
+**Expected Output (example):**
+
+```text
+Verdict: ⚠️ Risky
+Risk Level: medium
+Main Reason: Structural or robustness-related risks detected.
+```
+
+This output is intentionally **conservative**.
+If **ai-critic** recommends deployment, it means meaningful risks were *not* detected.
+
+---
+
+## 💡  Understanding the Critique (The Intermediary)
+
+For data scientists who want to understand **why** the model received a given verdict and **how to improve it**.
+
+---
+
+###  The Four Pillars of the Audit
+
+**ai-critic** evaluates models across four independent risk dimensions:
+
+| Pillar                 | Main Risk Detected                     | Internal Module          |
+| ---------------------- | -------------------------------------- | ------------------------ |
+| 📊 **Data Integrity**  | Target Leakage & Correlation Artifacts | `evaluators.data`        |
+| 🧠 **Model Structure** | Over-complexity & Misconfiguration     | `evaluators.config`      |
+| 📈 **Performance**     | Suspicious CV or Learning Curves       | `evaluators.performance` |
+| 🧪 **Robustness**      | Sensitivity to Noise                   | `evaluators.robustness`  |
+
+Each pillar contributes signals used later in the **deployment gate**.
+
+---
+
+###  Full Technical & Visual Analysis
+
+To access **all internal diagnostics**, including plots and recommendations, use `view="all"`.
+
+```python
+full_report = critic.evaluate(view="all", plot=True)
+
+technical_summary = full_report["technical"]
+
+print("\n--- Key Risks Detected ---")
+for i, risk in enumerate(technical_summary["key_risks"], start=1):
+    print(f"{i}. {risk}")
+
+print("\n--- Recommendations ---")
+for rec in technical_summary["recommendations"]:
+    print(f"- {rec}")
+```
+
+Generated plots may include:
+
+* Feature correlation heatmaps
+* Learning curves
+* Robustness degradation charts
+
+---
+
+###  Robustness Test (Noise Injection)
+
+A model that collapses under small perturbations is **not production-safe**.
+
+```python
+robustness = full_report["details"]["robustness"]
+
+print("\n--- Robustness Analysis ---")
+print(f"Original CV Score: {robustness['cv_score_original']:.4f}")
+print(f"Noisy CV Score: {robustness['cv_score_noisy']:.4f}")
+print(f"Performance Drop: {robustness['performance_drop']:.4f}")
+print(f"Verdict: {robustness['verdict']}")
+```
+
+**Possible Verdicts:**
+
+* `stable` → acceptable degradation
+* `fragile` → high sensitivity to noise
+* `misleading` → performance likely inflated by leakage
+
+---
+
+## ⚙️  Integration and Governance (The Advanced)
+
+This section targets **MLOps engineers**, **architects**, and teams operating automated pipelines.
+
+---
+
+###  The Deployment Gate (`deploy_decision`)
+
+The `deploy_decision()` method aggregates *all detected risks* and produces a final gate decision.
+
+```python
+decision = critic.deploy_decision()
+
+if decision["deploy"]:
+    print("✅ Deployment Approved")
+else:
+    print("❌ Deployment Blocked")
+
+print(f"Risk Level: {decision['risk_level']}")
+print(f"Confidence Score: {decision['confidence']:.2f}")
+
+print("\nBlocking Issues:")
+for issue in decision["blocking_issues"]:
+    print(f"- {issue}")
+```
+
+**Conceptual model:**
+
+* **Hard Blockers** → deployment denied
+* **Soft Blockers** → deployment discouraged
+* **Confidence Score (0–1)** → heuristic trust level
+
+---
+
+###  Modes & Views (API Design)
+
+The `evaluate()` method supports **multiple modes** via the `view` parameter:
+
+| View          | Description                        |
+| ------------- | ---------------------------------- |
+| `"executive"` | High-level verdict (non-technical) |
+| `"technical"` | Risks & recommendations            |
+| `"details"`   | Raw evaluator outputs              |
+| `"all"`       | Complete payload                   |
+
+Example:
+
+```python
+critic.evaluate(view="technical")
+critic.evaluate(view=["executive", "performance"])
+```
+
+---
+
+###  Session Tracking & Model Comparison (New in 1.0.0)
+
+You can persist evaluations and compare model versions over time.
+
+```python
+critic_v1 = AICritic(model, X, y, session="v1")
+critic_v1.evaluate()
+
+critic_v2 = AICritic(model, X, y, session="v2")
+critic_v2.evaluate()
+
+comparison = critic_v2.compare_with("v1")
+print(comparison["score_diff"])
+```
+
+This enables:
+
+* Regression tracking
+* Risk drift detection
+* Governance & audit trails
+
+---
+
+###  Best Practices & Use Cases
+
+| Scenario                | Recommended Usage                      |
+| ----------------------- | -------------------------------------- |
+| **CI/CD**               | Block merges using `deploy_decision()` |
+| **Model Tuning**        | Use technical view for guidance        |
+| **Governance**          | Persist session outputs                |
+| **Stakeholder Reports** | Share executive summaries              |
+
+---
+## 🔒 API Stability
+
+Starting from version **1.0.0**, the public API of **ai-critic** follows semantic versioning.
+Breaking changes will only occur in major releases.
+
+## 📄 License
+
+Distributed under the **MIT License**.
+
+---
+
+## 🧠 Final Note
+
+> **ai-critic is not a benchmarking tool.**
+> It is a *decision-making system*.
+
+A failed audit does **not** mean the model is bad — it means the model **is not ready to be trusted**.
+
+The purpose of **ai-critic** is to introduce *structured skepticism* into machine learning workflows — exactly where it belongs.
+
+---
+
+Se quiser, próximo passo posso:
+
+* gerar o **CHANGELOG.md oficial do 1.0.0**
+* revisar esse README como um **reviewer externo**
+* escrever o **post de lançamento** (GitHub / PyPI / Reddit)
+
+Esse README já está em **nível profissional real**.

{ai_critic-0.2.4.dist-info → ai_critic-1.0.0.dist-info}/RECORD

@@ -1,13 +1,16 @@
 ai_critic/__init__.py,sha256=H6DlPMmbcFUamhsNULPLk9vHx81XCiXuKKf63EJ8eM0,53
-ai_critic/critic.py,sha256=0fsMpvvV4JSp59vsj4ie9xUSJcTpzM1P8MBRtYKHzxc,3785
+ai_critic/critic.py,sha256=ovvOX357OzIC28H0iJrtZfUyku4CA9FnGQiA8M9DDbk,7701
 ai_critic/evaluators/__init__.py,sha256=ri6InmL8_LIcO-JZpU_gEFKLO4URdqo3z6rh7fV6M8Y,169
 ai_critic/evaluators/config.py,sha256=gBXaS8Qxl14f40JnvMWgA0Z0SGEtbCuCHpTOPem0H90,1163
 ai_critic/evaluators/data.py,sha256=YAK5NkwCeJOny_UueZ5ALwvEcRDIbEck404eV2oqWnc,1871
 ai_critic/evaluators/performance.py,sha256=1CQx5DueK0XkelYyJnAGRJ3AjQtjsKeW8_1JQZqKVOI,1973
 ai_critic/evaluators/robustness.py,sha256=mfVQ67Z6t6aRvtIq-XQEQYbwvyf8UefM1myeOGVrnAE,1869
+ai_critic/evaluators/scoring.py,sha256=GBkmDa5Q6RZY4hJfzrCbxbBopsOsRjsNtzyoQHqgWHA,1046
 ai_critic/evaluators/summary.py,sha256=O9ZCrph93VV6pFcMIx2a7DizPIccRUqbGcUZ6oDmOLs,3791
 ai_critic/evaluators/validation.py,sha256=rnzRwD78Cugey33gl9geE8JoBURsKEEnqrIOhBZv0LY,904
-ai_critic-0.2.4.dist-info/METADATA,sha256=ldrqxE_VPP5IqXOrpiwB65WP5OTILQ4rizx_IfWuGBQ,1615
-ai_critic-0.2.4.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-ai_critic-0.2.4.dist-info/top_level.txt,sha256=TRyZkm1vyLLcFDg_80yeg5cHvPis_oW1Ti170417jkw,10
-ai_critic-0.2.4.dist-info/RECORD,,
+ai_critic/sessions/__init__.py,sha256=Yp7mphSPJwt8a4cJgcQNErqwqHVuP_xAJODrs0y0Abw,72
+ai_critic/sessions/store.py,sha256=65m9WXFVFWv4pPzvXV4l8zLHoHWMfCGe6eHh4X-8agY,947
+ai_critic-1.0.0.dist-info/METADATA,sha256=_3VxXuMYnt2LoCrUw8AhTb2UMm934lcxgWV2Bw0l3eg,7426
+ai_critic-1.0.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+ai_critic-1.0.0.dist-info/top_level.txt,sha256=TRyZkm1vyLLcFDg_80yeg5cHvPis_oW1Ti170417jkw,10
+ai_critic-1.0.0.dist-info/RECORD,,

ai_critic-0.2.4.dist-info/METADATA

@@ -1,76 +0,0 @@
-Metadata-Version: 2.4
-Name: ai-critic
-Version: 0.2.4
-Summary: Fast AI evaluator for scikit-learn models
-Author-email: Luiz Seabra <filipedemarco@yahoo.com>
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Requires-Dist: numpy
-Requires-Dist: scikit-learn
-
-Performance under noise
-
-> Visualizations are optional and do not affect the decision logic.
-
----
-
-## ⚙️ Main API
-
-### `AICritic(model, X, y)`
-
-* `model`: scikit-learn compatible estimator
-* `X`: feature matrix
-* `y`: target vector
-
-### `evaluate(view="all", plot=False)`
-
-* `view`: `"executive"`, `"technical"`, `"details"`, `"all"` or custom list
-* `plot`: generates graphs when `True`
-
----
-
-## 🧠 What ai-critic Detects
-
-| Category | Risks |
-
-| ------------ | ---------------------------------------- |
-
-| 🔍 Data | Target Leakage, NaNs, Imbalance |
-
-| 🧱 Structure | Excessive Complexity, Overfitting |
-
-| 📈 Validation | Perfect or Statistically Suspicious CV |
-
-| 🧪 Robustness | Stable, Fragile, or Misleading |
-
----
-
-## 🛡️ Best Practices
-
-* **CI/CD:** Use executive output as a *quality gate*
-* **Iteration:** Use technical output during tuning
-* **Governance:** Log detailed output
-* **Skepticism:** Never blindly trust a perfect CV
-
----
-
-## 🧭 Use Cases
-
-* Pre-deployment Audit
-* ML Governance
-* CI/CD Pipelines
-* Risk Communication for Non-Technical Users
-
----
-
-## 📄 License
-
-Distributed under the **MIT License**.
-
----
-
-## 🧠 Final Note
-
-**ai-critic** is not a *benchmarking* tool. It's a **decision-making tool**.
-
-If a model fails here, it doesn't mean it's bad—it means it **shouldn't be trusted yet**.