ai-critic 1.2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

ai_critic/learning/recommender.py

@@ -0,0 +1,26 @@
+def recommend_changes(report):
+    recs = []
+
+    config = report["details"]["config"]
+    perf = report["details"]["performance"]
+    data = report["details"]["data"]
+
+    if config["risk_level"] == "high":
+        recs.append(
+            "Reduce model complexity (e.g., lower max_depth, fewer estimators)."
+        )
+
+    if perf["suspiciously_perfect"]:
+        recs.append(
+            "Suspiciously perfect performance detected — verify data leakage."
+        )
+
+    if data["data_leakage"]["suspected"]:
+        recs.append(
+            "Potential target leakage — review feature engineering pipeline."
+        )
+
+    if not recs:
+        recs.append("No critical changes recommended.")
+
+    return recs

ai_critic/learning/trainer.py

@@ -0,0 +1,16 @@
+from .features import extract_features
+
+class CriticTrainer:
+    def __init__(self, critic_model, min_samples=10):
+        self.model = critic_model
+        self.min_samples = min_samples
+        self.X = []
+        self.y = []
+
+    def add_feedback(self, report, success: bool):
+        features = extract_features(report)
+        self.X.append(list(features.values()))
+        self.y.append(int(success))
+
+        if len(self.y) >= self.min_samples:
+            self.model.train(self.X, self.y)

ai_critic/ml/suggester.py

@@ -0,0 +1,63 @@
+# ai_critic/ml/suggester.py
+
+def suggest_fix(event: dict) -> dict:
+    """
+    Lightweight ML-ready suggestion engine.
+    Today: rule-based.
+    Tomorrow: trained on global telemetry.
+    """
+
+    signals = event["signals"]
+    score = event["score"]
+
+    # 🔴 Casos críticos
+    if signals["leakage"] and signals["perfect_cv"]:
+        return {
+            "verdict": "critical",
+            "suggestion": (
+                "Strong evidence of data leakage. "
+                "Audit features highly correlated with the target, "
+                "remove shortcuts and re-run validation."
+            )
+        }
+
+    # 🟠 Robustez fraca
+    if signals["robustness"] == "fragile":
+        return {
+            "verdict": "warning",
+            "suggestion": (
+                "Model is fragile under noise. "
+                "Consider stronger regularization, "
+                "simpler architecture or more data."
+            )
+        }
+
+    # 🟠 Estrutura pesada
+    if signals["structural"] == "high":
+        return {
+            "verdict": "warning",
+            "suggestion": (
+                "Model complexity may be too high for dataset size. "
+                "Reduce depth, number of parameters or features."
+            )
+        }
+
+    # 🟢 Caso saudável
+    if score >= 85:
+        return {
+            "verdict": "ok",
+            "suggestion": (
+                "Model behavior looks consistent. "
+                "No critical risks detected at this stage."
+            )
+        }
+
+    # 🟡 Default
+    return {
+        "verdict": "review",
+        "suggestion": (
+            "No critical failures detected, "
+            "but model could benefit from further validation "
+            "and robustness checks."
+        )
+    }

ai_critic/telemetry/anonymizer.py

@@ -0,0 +1,9 @@
+def anonymize(report: dict) -> dict:
+    return {
+        "model_type": report["meta"]["model_type"],
+        "score": report["score"]["global"],
+        "signals": {
+            "leakage": report["details"]["data"]["data_leakage"]["suspected"],
+            "robustness": report["details"]["robustness"]["verdict"],
+        }
+    }

ai_critic/telemetry/client.py

@@ -0,0 +1,6 @@
+def collect_and_send(report, enabled=True):
+    if not enabled:
+        return
+
+    payload = anonymize(report)
+    send(payload)

ai_critic/telemetry/event.py

@@ -0,0 +1,15 @@
+def build_event(report: dict) -> dict:
+    return {
+        "model_type": report["meta"]["model_type"],
+        "framework": report["meta"]["framework"],
+        "n_samples": report["meta"]["n_samples"],
+        "n_features": report["meta"]["n_features"],
+        "score": report["scores"]["global"],
+        "risk_level": report["executive"]["risk_level"],
+        "signals": {
+            "leakage": report["details"]["data"]["data_leakage"]["suspected"],
+            "perfect_cv": report["details"]["performance"]["suspiciously_perfect"],
+            "robustness": report["details"]["robustness"]["verdict"],
+            "structural": report["details"]["config"]["risk_level"],
+        }
+    }

ai_critic/telemetry/local_store.py

@@ -0,0 +1,9 @@
+import json
+from pathlib import Path
+
+STORE = Path.home() / ".ai_critic" / "telemetry.jsonl"
+STORE.parent.mkdir(exist_ok=True)
+
+def save(event: dict):
+    with open(STORE, "a") as f:
+        f.write(json.dumps(event) + "\n")

ai_critic/telemetry/schema.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+
+@dataclass
+class TelemetryEvent:
+    model_type: str
+    framework: str
+    problem_type: str
+    n_samples: int
+    n_features: int
+    score: int
+    verdict: str

ai_critic/telemetry/sender.py

@@ -0,0 +1,9 @@
+def send(payload: dict):
+    try:
+        requests.post(
+            "https://api.ai-critic.dev/telemetry",
+            json=payload,
+            timeout=1
+        )
+    except Exception:
+        pass  # nunca quebra o usuário

ai_critic-2.0.0.dist-info/METADATA

@@ -0,0 +1,390 @@
+Metadata-Version: 2.4
+Name: ai-critic
+Version: 2.0.0
+Summary: Fast AI evaluator for scikit-learn models
+Author-email: Luiz Seabra <filipedemarco@yahoo.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+
+# ai-critic 🧠
+
+## The Quality Gate for Machine Learning Models
+
+**ai-critic** is a specialized **decision-making system** designed to evaluate whether a machine learning model is **safe, reliable, and trustworthy enough** to be deployed in real-world environments.
+
+Unlike traditional ML evaluation tools that focus almost exclusively on *performance metrics*, **ai-critic** operates as a **Quality Gate** — a final checkpoint that actively probes models to uncover **hidden risks** that frequently cause silent failures in production.
+
+> **ai-critic does not ask *“How accurate is this model?”***
+> It asks ***“Can this model be trusted in the real world?”***
+
+---
+
+## 🎯 What Problem Does ai-critic Solve?
+
+In production, most ML failures are **not accuracy problems**.
+
+They are caused by:
+
+* Data leakage hidden inside features
+* Overfitting disguised as strong validation scores
+* Models that collapse under small noise
+* Models that rely on a single fragile signal
+* Configuration choices that look fine — but are structurally unsafe
+
+These failures usually appear **after deployment**, when it is already expensive or dangerous to fix them.
+
+**ai-critic exists to catch these failures *before* deployment.**
+
+---
+
+## 🚀 Getting Started (The Basics)
+
+This section is intentionally designed for **beginners**, **students**, and **engineers under time pressure**.
+
+If you only want a **fast, conservative verdict**, this is all you need.
+
+---
+
+### Installation
+
+Install directly from PyPI:
+
+```bash
+pip install ai-critic
+```
+
+Python ≥ 3.8 is recommended.
+
+---
+
+### The Quick Verdict
+
+With just a few lines of code, you can obtain:
+
+* An **executive-level verdict**
+* A **risk classification**
+* A **deployment recommendation**
+
+```python
+from ai_critic import AICritic
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.datasets import make_classification
+
+# 1. Prepare data and model
+X, y = make_classification(
+    n_samples=1000,
+    n_features=20,
+    random_state=42
+)
+
+model = RandomForestClassifier(
+    max_depth=5,
+    random_state=42
+)
+
+# 2. Initialize the Critic
+critic = AICritic(model, X, y)
+
+# 3. Run the audit
+report = critic.evaluate(view="executive")
+
+print(f"Verdict: {report['verdict']}")
+print(f"Risk Level: {report['risk_level']}")
+print(f"Deploy Recommended: {report['deploy_recommended']}")
+print(f"Main Reason: {report['main_reason']}")
+```
+
+**Example Output:**
+
+```text
+Verdict: ⚠️ Risky
+Risk Level: medium
+Deploy Recommended: False
+Main Reason: Structural, robustness, or dependency-related risks detected.
+```
+
+This verdict is intentionally **conservative by design**.
+
+> If **ai-critic approves deployment**, it means **no meaningful risks were detected** by multiple independent heuristics.
+
+---
+
+## 🧭 How to Read the Verdict
+
+| Field                | Meaning                 |
+| -------------------- | ----------------------- |
+| `verdict`            | Human-readable summary  |
+| `risk_level`         | low / medium / high     |
+| `deploy_recommended` | Final gate decision     |
+| `main_reason`        | Primary blocking factor |
+
+The goal is clarity, not ambiguity.
+
+---
+
+## 💡 Understanding the Critique (Intermediate Level)
+
+This section is for **data scientists**, **ML engineers**, and **students** who want to understand *why* the model was flagged — and how to improve it.
+
+---
+
+### The Four Pillars of the Audit
+
+**ai-critic** evaluates models across **four independent risk dimensions**.
+
+| Pillar             | What It Detects                  | Why It Matters       |
+| ------------------ | -------------------------------- | -------------------- |
+| 📊 Data Integrity  | Leakage, correlations, shortcuts | Inflated performance |
+| 🧠 Model Structure | Over-complexity, unsafe configs  | Poor generalization  |
+| 📈 Performance     | Suspicious CV behavior           | False confidence     |
+| 🧪 Robustness      | Noise sensitivity                | Production collapse  |
+
+Each pillar produces **signals**, not binary judgments.
+
+Those signals are later aggregated by the **deployment gate**.
+
+---
+
+## 📊 Data Integrity Analysis
+
+This pillar focuses on **the relationship between features and the target**.
+
+It answers questions like:
+
+* Are some features *too predictive*?
+* Are there suspicious correlations?
+* Does performance collapse when a single feature is disturbed?
+
+These are classic symptoms of **data leakage** and **shortcut learning**.
+
+---
+
+## 🧠 Model Structure Analysis
+
+A model can be accurate and still be unsafe.
+
+Structural analysis looks for:
+
+* Excessive depth
+* Over-parameterization
+* Configuration choices that amplify variance
+* Inconsistent bias–variance tradeoffs
+
+This is especially important for:
+
+* Decision trees
+* Boosting models
+* Neural networks with limited data
+
+---
+
+## 📈 Performance Sanity Checks
+
+Rather than optimizing metrics, **ai-critic questions them**.
+
+It checks:
+
+* Cross-validation stability
+* Variance across folds
+* Learning curve consistency
+* Performance under perturbations
+
+A strong score that behaves strangely is treated as **a warning, not a success**.
+
+---
+
+## 🧪 Robustness Testing (Noise Injection)
+
+Production data is **never clean**.
+
+This test injects controlled noise into inputs and measures degradation.
+
+```python
+robustness = report["details"]["robustness"]
+
+print(f"Original CV Score: {robustness['cv_score_original']}")
+print(f"Noisy CV Score: {robustness['cv_score_noisy']}")
+print(f"Performance Drop: {robustness['performance_drop']}")
+print(f"Verdict: {robustness['verdict']}")
+```
+
+Possible outcomes:
+
+* `stable` → acceptable degradation
+* `fragile` → high sensitivity
+* `misleading` → performance likely inflated
+
+---
+
+## 🔍 Explainability & Feature Sensitivity
+
+Accuracy alone hides *why* a model works.
+
+The explainability module performs **feature sensitivity analysis** to detect:
+
+* Feature-level leakage
+* Over-reliance on a single signal
+* Structural shortcuts
+
+---
+
+### How Explainability Works
+
+For each feature:
+
+1. The feature is randomly permuted.
+2. The model is re-evaluated.
+3. Performance drop is measured.
+
+Large drops indicate **critical dependency**.
+
+This approach is:
+
+* Model-agnostic
+* Lightweight
+* Framework-independent
+* Interpretable by humans
+
+---
+
+### Explainability Verdicts
+
+| Verdict                | Meaning                  |
+| ---------------------- | ------------------------ |
+| `stable`               | Balanced feature usage   |
+| `feature_dependency`   | Few features dominate    |
+| `feature_leakage_risk` | Single feature dominates |
+
+These verdicts **directly affect**:
+
+* Deployment decision
+* Confidence score
+* Recommendations
+
+---
+
+## 🧠 Recommendations Engine (New)
+
+**ai-critic does not stop at “deploy or not”.**
+
+It generates **actionable recommendations**, such as:
+
+* “Reduce `max_depth`”
+* “Increase regularization”
+* “Likely feature leakage detected”
+* “Model shows structural overfitting”
+* “High noise sensitivity — retrain with augmentation”
+
+These recommendations are **rule-based + data-driven**, not LLM hallucinations.
+
+---
+
+## ⚙️ Deployment Gate
+
+The final decision is produced by `deploy_decision()`.
+
+```python
+decision = critic.deploy_decision()
+
+print(decision["deploy"])
+print(decision["risk_level"])
+print(decision["confidence"])
+print(decision["blocking_issues"])
+```
+
+Conceptually:
+
+* **Hard blockers** → deployment denied
+* **Soft blockers** → deployment discouraged
+* **Confidence score (0–1)** → heuristic trust
+
+---
+
+## 🔄 Feedback Loop & Learning Critic
+
+**ai-critic improves over time**.
+
+Each evaluation can be stored as feedback:
+
+* Model config
+* Signals
+* Final outcome
+* Human override (optional)
+
+This enables:
+
+* Meta-learning
+* Better future recommendations
+* Context-aware criticism
+
+---
+
+## 🧪 Session Tracking & Comparison
+
+You can compare models over time:
+
+```python
+critic_v1 = AICritic(model, X, y, session="v1")
+critic_v1.evaluate()
+
+critic_v2 = AICritic(model, X, y, session="v2")
+critic_v2.evaluate()
+
+critic_v2.compare_with("v1")
+```
+
+Use cases:
+
+* Regression detection
+* Risk drift
+* Governance audits
+
+---
+
+## ⚙️ Multi-Framework Support
+
+The same API works for:
+
+* scikit-learn
+* PyTorch
+* TensorFlow
+
+Adapters handle training, evaluation, and probing internally.
+
+---
+
+## 🧩 Design Philosophy
+
+**ai-critic is intentionally skeptical.**
+
+It assumes:
+
+* Metrics can lie
+* Data is imperfect
+* Models fail silently
+* Confidence must be earned
+
+This makes it ideal as a **final gate**, not a tuning toy.
+
+---
+
+## 🛡️ What ai-critic Is NOT
+
+* ❌ A hyperparameter optimizer
+* ❌ A leaderboard benchmark tool
+* ❌ A replacement for domain expertise
+* ❌ A magic “approve all” system
+
+---
+
+## 🧠 Final Note
+
+> **ai-critic is not here to make models look good.**
+> It exists to **prevent bad models from looking good enough to deploy**.
+
+A failed audit does **not** mean your model is bad.
+It means your model is **not yet safe to trust**.
+
+That distinction is everything.

ai_critic-2.0.0.dist-info/RECORD

@@ -0,0 +1,37 @@
+ai_critic/__init__.py,sha256=H6DlPMmbcFUamhsNULPLk9vHx81XCiXuKKf63EJ8eM0,53
+ai_critic/cli.py,sha256=4rf9g-CjtYaS9jjLPYWI56Z_6JLHhKh4KSbELYWwsX8,4133
+ai_critic/critic.py,sha256=4rf9g-CjtYaS9jjLPYWI56Z_6JLHhKh4KSbELYWwsX8,4133
+ai_critic/ai_suggestions/predictor.py,sha256=pn20sG1MjXuzXSquu0IqAcSzM2Y3gROHTq8zRSle0RM,155
+ai_critic/ai_suggestions/rules.py,sha256=ZbCtPc5OZgPGzqgYNXf43C2aVdGWKsjhQ-_zdsmtftw,132
+ai_critic/evaluators/__init__.py,sha256=ri6InmL8_LIcO-JZpU_gEFKLO4URdqo3z6rh7fV6M8Y,169
+ai_critic/evaluators/adapters.py,sha256=8Xw9Ccg1iGVNwVQDGVIqhWj5-Sg6evqCZhg21u8EP20,3068
+ai_critic/evaluators/config.py,sha256=eeHOqyU-GOMFdQlhoyJnsaEZYMF1XewyoFKYP7d0o-w,1911
+ai_critic/evaluators/data.py,sha256=YAK5NkwCeJOny_UueZ5ALwvEcRDIbEck404eV2oqWnc,1871
+ai_critic/evaluators/explainability.py,sha256=UWbcb5uVI78d1ljfdrWd2DrjlwEz1y9CeVtkukefEfA,1759
+ai_critic/evaluators/performance.py,sha256=1CQx5DueK0XkelYyJnAGRJ3AjQtjsKeW8_1JQZqKVOI,1973
+ai_critic/evaluators/robustness.py,sha256=mfVQ67Z6t6aRvtIq-XQEQYbwvyf8UefM1myeOGVrnAE,1869
+ai_critic/evaluators/scoring.py,sha256=9rgkCXKKm9G1Lfwn5i9HcsJTN5OUjxMycOUzhWkp_2g,1576
+ai_critic/evaluators/summary.py,sha256=H9rU9tXAXqyQ34L6bOOOHrdIapSq71gcjjc8jfyJMq4,5003
+ai_critic/evaluators/validation.py,sha256=rnzRwD78Cugey33gl9geE8JoBURsKEEnqrIOhBZv0LY,904
+ai_critic/feedback/__init__.py,sha256=JWzTxV8ycpoQPrHuwWzwmuDlpVAL-vzUx2dXy1_7z9c,62
+ai_critic/feedback/store.py,sha256=BpWAM9byj-zfFG6cjxNOvVDoEPP5TUe5DSiAgVQ_8Rg,606
+ai_critic/learning/__init__.py,sha256=umxvyyh8bKdyZP5tRidJUJ-mqeRlTkNcV0HE7_0qHTU,318
+ai_critic/learning/critic_model.py,sha256=KqTOYjk4DsdHjcps-IZO-Udx0KPq6fHVWD9w8OViWyo,708
+ai_critic/learning/features.py,sha256=T5thN96ZBv63gSKxHARc87al2UJP5AIF4_4SAPt1fuE,663
+ai_critic/learning/policy.py,sha256=c2bAtvUu6DK1KDtxTScK-C2xMcQC-HvXmPxpLwggygg,511
+ai_critic/learning/recommender.py,sha256=DxL_C-oqahJJ_u69486JyxBNszYibbru1YBV-TyWgRQ,715
+ai_critic/learning/trainer.py,sha256=UsSS_1QMYQNf4UCGBvbVtWr3n60qC6ZUMP9AmBilKz8,497
+ai_critic/ml/suggester.py,sha256=DKq5NjT7oLBXVw2jNtDLeSjzamV7un2QPqxMHYRIh1Q,1757
+ai_critic/sessions/__init__.py,sha256=Yp7mphSPJwt8a4cJgcQNErqwqHVuP_xAJODrs0y0Abw,72
+ai_critic/sessions/store.py,sha256=65m9WXFVFWv4pPzvXV4l8zLHoHWMfCGe6eHh4X-8agY,947
+ai_critic/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ai_critic/telemetry/anonymizer.py,sha256=Mg40ZZ1U8vbUEar_n3ZtrDwTfLvJB-qnRlmqXEDAsnk,332
+ai_critic/telemetry/client.py,sha256=CuEibfDkI5Y_Y1yV8BT0aXEjFgZurcL5p5Pk7yfpTz4,130
+ai_critic/telemetry/event.py,sha256=9n2tGmrrAyFhx26Yb3wUImO72ANJc06ZM8yPs4Q48Uc,697
+ai_critic/telemetry/local_store.py,sha256=zZ3CgvHYMUhwgpa_VuITI4rlPHCV4yJKxxYjIGoijfA,225
+ai_critic/telemetry/schema.py,sha256=1fbZqmEmgvmM4FNmhI7O2sQfUaGEZty5SzqLYVo3y0g,200
+ai_critic/telemetry/sender.py,sha256=rsxKbmbT2UvoT_oFK8mQxdcqZRfHLRgrkozHJqbsDps,228
+ai_critic-2.0.0.dist-info/METADATA,sha256=MUuBILCf0XNrD0D8kjdOa7HX-ENYPqW4AGfNsBlpajQ,9293
+ai_critic-2.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ai_critic-2.0.0.dist-info/top_level.txt,sha256=TRyZkm1vyLLcFDg_80yeg5cHvPis_oW1Ti170417jkw,10
+ai_critic-2.0.0.dist-info/RECORD,,