modelbrief 0.2.1__py3-none-any.whl

modelbrief/__init__.py

@@ -0,0 +1,5 @@
+"""ModelBrief: reporting for already-trained ML models."""
+from .core.report import ModelBrief
+from .core.result import ReportResult, Section
+__all__ = ["ModelBrief", "ReportResult", "Section"]
+__version__ = "0.2.0"

modelbrief/adapters/base.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+import numpy as np
+
+class BaseAdapter(ABC):
+    def __init__(self, model): self.model=model
+    @abstractmethod
+    def predict(self, X): raise NotImplementedError
+    def predict_proba(self, X):
+        if hasattr(self.model,"predict_proba"): return np.asarray(self.model.predict_proba(X))
+        return None
+    def parameters(self):
+        if hasattr(self.model,"get_params"):
+            try: return self.model.get_params(deep=False)
+            except Exception: pass
+        if hasattr(self.model,"count_params"):
+            try: return {"trainable_or_total_parameters":int(self.model.count_params())}
+            except Exception: pass
+        return {"class":type(self.model).__name__}
+    def feature_importance(self, feature_names=None):
+        values=getattr(self.model,"feature_importances_",None)
+        if values is None and hasattr(self.model,"coef_"):
+            values=np.asarray(self.model.coef_); values=np.mean(np.abs(values),axis=0) if values.ndim>1 else np.abs(values)
+        if values is None: return None
+        values=np.asarray(values).reshape(-1); names=feature_names or [f"feature_{i}" for i in range(len(values))]
+        return sorted(zip(names,values.tolist()),key=lambda x:abs(x[1]),reverse=True)

modelbrief/adapters/computer_vision/image_models.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+def is_image_batch(X):
+    shape=getattr(X,"shape",None)
+    return bool(shape is not None and len(shape)==4 and (shape[1] in (1,3,4) or shape[-1] in (1,3,4)))

modelbrief/adapters/computer_vision/pytorch_vision.py

@@ -0,0 +1,2 @@
+from ..deep_learning.pytorch import PyTorchAdapter
+class PyTorchVisionAdapter(PyTorchAdapter): pass

modelbrief/adapters/computer_vision/tensorflow_vision.py

@@ -0,0 +1,2 @@
+from ..deep_learning.tensorflow import TensorFlowAdapter
+class TensorFlowVisionAdapter(TensorFlowAdapter): pass

modelbrief/adapters/deep_learning/pytorch.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+import numpy as np
+from ..base import BaseAdapter
+class PyTorchAdapter(BaseAdapter):
+    def __init__(self,model,device=None,batch_size=64): super().__init__(model); self.device=device; self.batch_size=batch_size
+    def _tensor(self,X):
+        import torch
+        if isinstance(X,torch.Tensor): return X
+        return torch.as_tensor(np.asarray(X),dtype=torch.float32)
+    def raw_predict(self,X):
+        import torch
+        self.model.eval(); device=self.device or next(self.model.parameters(),torch.empty(0)).device
+        out=[]; t=self._tensor(X)
+        with torch.no_grad():
+            for batch in torch.split(t,self.batch_size):
+                y=self.model(batch.to(device)); y=y.logits if hasattr(y,"logits") else y; out.append(y.detach().cpu())
+        return torch.cat(out).numpy()
+    def predict_proba(self,X):
+        z=self.raw_predict(X)
+        if z.ndim==1 or z.shape[-1]==1: return 1/(1+np.exp(-z.reshape(-1)))
+        z=z-z.max(axis=1,keepdims=True); e=np.exp(z); return e/e.sum(axis=1,keepdims=True)
+    def predict(self,X):
+        p=self.predict_proba(X); return (p>=.5).astype(int) if p.ndim==1 else p.argmax(axis=1)
+    def parameters(self):
+        total=sum(p.numel() for p in self.model.parameters()); trainable=sum(p.numel() for p in self.model.parameters() if p.requires_grad)
+        return {"total_parameters":total,"trainable_parameters":trainable,"architecture":str(self.model)}

modelbrief/adapters/deep_learning/tensorflow.py

@@ -0,0 +1,10 @@
+import numpy as np
+from ..base import BaseAdapter
+class TensorFlowAdapter(BaseAdapter):
+    def raw_predict(self,X): return np.asarray(self.model.predict(X,verbose=0))
+    def predict_proba(self,X):
+        z=self.raw_predict(X)
+        if z.ndim==1 or z.shape[-1]==1: return z.reshape(-1)
+        return z
+    def predict(self,X):
+        p=self.predict_proba(X); return (p>=.5).astype(int) if p.ndim==1 else p.argmax(axis=1)

modelbrief/adapters/nlp/pytorch_nlp.py

@@ -0,0 +1,2 @@
+from ..deep_learning.pytorch import PyTorchAdapter
+class PyTorchNLPAdapter(PyTorchAdapter): pass

modelbrief/adapters/nlp/tensorflow_nlp.py

@@ -0,0 +1,2 @@
+from ..deep_learning.tensorflow import TensorFlowAdapter
+class TensorFlowNLPAdapter(TensorFlowAdapter): pass

modelbrief/adapters/nlp/transformers.py

@@ -0,0 +1,11 @@
+import numpy as np
+from ..base import BaseAdapter
+class TransformersAdapter(BaseAdapter):
+    def predict(self,X):
+        outputs=self.model(X)
+        labels=[]
+        for item in outputs:
+            if isinstance(item,list): item=max(item,key=lambda z:z.get("score",0))
+            labels.append(item.get("label",item) if isinstance(item,dict) else item)
+        return np.asarray(labels,dtype=object)
+    def parameters(self): return {"pipeline_task":getattr(self.model,"task",None),"model_class":type(getattr(self.model,"model",self.model)).__name__}

modelbrief/adapters/tabular/catboost.py

@@ -0,0 +1,4 @@
+from .sklearn import SklearnAdapter
+class CatBoostAdapter(SklearnAdapter):
+    """Adapter for an already-fitted catboost estimator."""
+    pass

modelbrief/adapters/tabular/lightgbm.py

@@ -0,0 +1,4 @@
+from .sklearn import SklearnAdapter
+class LightGBMAdapter(SklearnAdapter):
+    """Adapter for an already-fitted lightgbm estimator."""
+    pass

modelbrief/adapters/tabular/sklearn.py

@@ -0,0 +1,17 @@
+import numpy as np
+
+from ..base import BaseAdapter
+
+
+class SklearnAdapter(BaseAdapter):
+    """Adapter for scikit-learn and compatible estimators."""
+
+    def predict(self, X):
+        return np.asarray(self.model.predict(X))
+
+    def predict_proba(self, X):
+        if hasattr(self.model, "predict_proba"):
+            return np.asarray(self.model.predict_proba(X))
+
+        return None
+

modelbrief/adapters/tabular/xgboost.py

@@ -0,0 +1,4 @@
+from .sklearn import SklearnAdapter
+class XGBoostAdapter(SklearnAdapter):
+    """Adapter for an already-fitted xgboost estimator."""
+    pass

modelbrief/ai/explainer.py

@@ -0,0 +1,24 @@
+import json
+
+
+def explain(result, provider):
+    prompt = (
+    "Explain the ML evaluation results accurately and concisely. "
+    "Return exactly 3 to 5 findings. "
+    "Use clean Markdown with one finding per line beginning with '- '. "
+    "Do not use numbered lists, nested bullets, bullet symbols such as '•', "
+    "or additional '-' characters. "
+    "Focus only on the most important findings, patterns, and limitations. "
+    "Do not repeat all metrics, tables, confusion matrices, or feature "
+    "importance values already shown in the report. "
+    "Do not provide code. "
+    "Base every statement only on the supplied results. "
+    "Do not invent facts or make unsupported claims."
+)
+
+    data = json.dumps(
+        result.to_dict(),
+        default=str,
+    )[:7000]
+
+    return provider.complete(prompt, data)

modelbrief/ai/providers/base.py

@@ -0,0 +1,4 @@
+from abc import ABC,abstractmethod
+class AIProvider(ABC):
+ @abstractmethod
+ def complete(self,system,user): raise NotImplementedError

modelbrief/ai/providers/groq.py

@@ -0,0 +1,45 @@
+import os
+
+from .base import AIProvider
+
+
+class GroqProvider(AIProvider):
+
+    def __init__(self, api_key=None, model=None):
+        key = api_key or os.getenv("GROQ_API_KEY")
+
+        if not key:
+            raise ValueError("GROQ_API_KEY is required when ai=True")
+
+        try:
+            from groq import Groq
+        except ImportError as e:
+            raise ImportError(
+                'Install AI support with: pip install "modelbrief[ai]"'
+            ) from e
+
+        self.client = Groq(api_key=key)
+
+        self.model = model or os.getenv(
+            "MODELBRIEF_GROQ_MODEL",
+            "openai/gpt-oss-20b",
+        )
+
+    def complete(self, system, user):
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {
+                    "role": "system",
+                    "content": system,
+                },
+                {
+                    "role": "user",
+                    "content": user,
+                },
+            ],
+            temperature=0.2,
+            max_completion_tokens=1200,
+        )
+
+        return response.choices[0].message.content

modelbrief/ai/recommender.py

@@ -0,0 +1,23 @@
+import json
+
+
+def recommend(result, provider):
+    prompt = (
+    "Provide 3 to 5 concise, practical, evidence-based recommendations. "
+    "Use clean Markdown with one recommendation per line beginning with '- '. "
+    "Do not use numbered lists, nested bullets, bullet symbols such as '•', "
+    "or additional '-' characters. "
+    "Focus only on recommendations supported by the supplied results. "
+    "Do not repeat metrics or findings already shown in the report. "
+    "Do not claim the model was retrained or improved. "
+    "Do not provide code or tables. "
+    "Prioritize the most useful recommendations and avoid repetition."
+)
+
+    data = json.dumps(
+        result.to_dict(),
+        default=str,
+    )[:7000]
+
+    return provider.complete(prompt, data)
+

modelbrief/analysis/anomaly_detection.py

@@ -0,0 +1,2 @@
+from ..metrics.anomaly_detection import anomaly_metrics
+def analyse(y,pred,scores=None): return anomaly_metrics(y,pred,scores)

modelbrief/analysis/classification.py

@@ -0,0 +1,2 @@
+from ..metrics.classification import classification_metrics
+def analyse(y,pred,proba=None): return classification_metrics(y,pred,proba)

modelbrief/analysis/clustering.py

@@ -0,0 +1,2 @@
+from ..metrics.clustering import clustering_metrics
+def analyse(X,labels): return clustering_metrics(X,labels)

modelbrief/analysis/computer_vision.py

@@ -0,0 +1,2 @@
+from ..metrics.vision import image_classification_metrics
+def analyse(y,pred,proba=None): return image_classification_metrics(y,pred,proba)

modelbrief/analysis/dataset.py

@@ -0,0 +1,12 @@
+import numpy as np
+
+def describe_dataset(X,y=None):
+    if X is None: return {"available":False}
+    shape=getattr(X,"shape",None) or (len(X),)
+    info={"available":True,"rows":int(shape[0]),"shape":tuple(int(i) for i in shape),"type":type(X).__name__}
+    try:
+        a=np.asarray(X); info["missing_values"]=int(np.sum(a!=a)) if a.dtype.kind in "fc" else None
+    except Exception: info["missing_values"]=None
+    if y is not None:
+        ya=np.asarray(y); info["target_shape"]=tuple(ya.shape); info["target_unique"]=int(len(np.unique(ya))) if ya.size and ya.ndim==1 else None
+    return info

modelbrief/analysis/dimensionality_reduction.py

@@ -0,0 +1,2 @@
+def analyse(X,embedded):
+ return {"input_dimensions":getattr(X,"shape",[None,None])[-1],"output_dimensions":getattr(embedded,"shape",[None,None])[-1],"samples":len(embedded)}

modelbrief/analysis/image_segmentation.py

@@ -0,0 +1,2 @@
+from ..metrics.segmentation import segmentation_metrics
+def analyse(y,pred): return segmentation_metrics(y,pred)

modelbrief/analysis/model.py

@@ -0,0 +1,2 @@
+def model_overview(model,adapter,task):
+ return {"model_class":type(model).__name__,"module":type(model).__module__,"task":task,"fitted_model_supplied":True,"retraining_performed":False,"parameters":adapter.parameters()}

modelbrief/analysis/nlp.py

@@ -0,0 +1,2 @@
+def analyse_texts(texts):
+ lengths=[len(str(t).split()) for t in texts]; return {"documents":len(lengths),"mean_tokens":sum(lengths)/len(lengths) if lengths else 0,"max_tokens":max(lengths,default=0)}

modelbrief/analysis/object_detection.py

@@ -0,0 +1,3 @@
+from ..metrics.object_detection import intersection_over_union
+def analyse(true_boxes,pred_boxes):
+ vals=[intersection_over_union(a,b) for a,b in zip(true_boxes,pred_boxes)]; return {"mean_iou":sum(vals)/len(vals) if vals else 0.0,"matched_boxes":len(vals)}

modelbrief/analysis/regression.py

@@ -0,0 +1,2 @@
+from ..metrics.regression import regression_metrics
+def analyse(y,pred): return regression_metrics(y,pred)

modelbrief/analysis/sentiment.py

@@ -0,0 +1 @@
+from .classification import analyse

modelbrief/analysis/text_classification.py

@@ -0,0 +1 @@
+from .classification import analyse

modelbrief/analysis/text_generation.py

@@ -0,0 +1,2 @@
+def analyse(generated):
+ texts=[str(x) for x in generated]; return {"generations":len(texts),"mean_characters":sum(map(len,texts))/len(texts) if texts else 0}

modelbrief/analysis/time_series.py

@@ -0,0 +1,2 @@
+from ..metrics.time_series import time_series_metrics
+def analyse(y,pred): return time_series_metrics(y,pred)

modelbrief/core/context.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any
+
+@dataclass
+class DataSplit:
+    X: Any = None
+    y: Any = None
+
+@dataclass
+class ReportContext:
+    model: Any
+    train: DataSplit = field(default_factory=DataSplit)
+    validation: DataSplit = field(default_factory=DataSplit)
+    test: DataSplit = field(default_factory=DataSplit)
+    task: str | None = None
+    feature_names: list[str] | None = None
+    target_names: list[str] | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    options: dict[str, Any] = field(default_factory=dict)

modelbrief/core/detector.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import numpy as np
+
+from ..adapters.tabular.sklearn import SklearnAdapter
+
+
+def detect_adapter(model, X=None):
+    """Select the correct adapter for the model's framework."""
+
+    module_name = type(model).__module__.lower()
+
+    # Scikit-learn models and pipelines
+    if (
+        module_name == "sklearn.pipeline"
+        or module_name.startswith("sklearn.")
+    ):
+        return SklearnAdapter(model)
+
+    # Genuine Hugging Face Transformers pipelines
+    if (
+        module_name == "transformers.pipelines.base"
+        or module_name.startswith("transformers.pipelines")
+    ):
+        from ..adapters.nlp.transformers import TransformersAdapter
+
+        return TransformersAdapter(model)
+
+    # PyTorch models
+    if module_name.startswith("torch"):
+        from ..adapters.deep_learning.pytorch import PyTorchAdapter
+
+        return PyTorchAdapter(model)
+
+    # TensorFlow and Keras models
+    if module_name.startswith(("tensorflow", "keras")):
+        from ..adapters.deep_learning.tensorflow import TensorFlowAdapter
+
+        return TensorFlowAdapter(model)
+
+    # Default to the scikit-learn adapter
+    return SklearnAdapter(model)
+
+
+def detect_task(model, y=None, requested=None):
+    """Detect the task unless the user explicitly provides one."""
+
+    if requested:
+        return requested.lower().strip().replace(" ", "_")
+
+    # Older scikit-learn versions expose estimator type through
+    # _estimator_type.
+    estimator_type = getattr(model, "_estimator_type", None)
+
+    # Newer scikit-learn versions expose estimator type through
+    # __sklearn_tags__().
+    if estimator_type is None and hasattr(model, "__sklearn_tags__"):
+        try:
+            estimator_type = model.__sklearn_tags__().estimator_type
+        except (AttributeError, TypeError):
+            estimator_type = None
+
+    if estimator_type == "classifier":
+        return "classification"
+
+    if estimator_type == "regressor":
+        return "regression"
+
+    if estimator_type == "clusterer" or hasattr(model, "labels_"):
+        return "clustering"
+
+    if y is None:
+        return "clustering"
+
+    target = np.asarray(y)
+
+    if target.dtype.kind in "OUSb":
+        return "classification"
+
+    unique_values = len(np.unique(target))
+
+    classification_limit = max(
+        20,
+        int(np.sqrt(max(len(target), 1))),
+    )
+
+    if unique_values <= classification_limit:
+        return "classification"
+
+    return "regression"

modelbrief/core/report.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+import base64,io,json
+import numpy as np
+from .context import ReportContext,DataSplit
+from .detector import detect_adapter,detect_task
+from .result import ReportResult
+from ..analysis.dataset import describe_dataset
+from ..analysis.model import model_overview
+
+class ModelBrief:
+ def __init__(self,model,X_train=None,X_val=None,X_test=None,y_train=None,y_val=None,y_test=None,task=None,ai=False,feature_names=None,target_names=None,adapter=None,**options):
+  self.context=ReportContext(model,DataSplit(X_train,y_train),DataSplit(X_val,y_val),DataSplit(X_test,y_test),task,feature_names,target_names,options=options)
+  self.adapter=adapter or detect_adapter(model,X_train if X_train is not None else X_test); self.ai=ai; self._result=None
+#   print(
+#     "Selected adapter:",
+#     type(self.adapter).__name__,
+#     "| Model:",
+#     type(model).__module__,
+#     type(model).__name__,
+# )
+ def _split_analysis(self,name,split,task):
+  if split.X is None: return None
+  pred=np.asarray(self.adapter.predict(split.X)); out={"samples":len(pred)}
+  if split.y is None:
+   if task=="clustering":
+    from ..metrics.clustering import clustering_metrics; out.update(clustering_metrics(split.X,pred))
+   else: out["predictions_generated"]=True
+   return out
+  if task in ("classification","computer_vision","nlp","text_classification","sentiment"):
+   from ..metrics.classification import classification_metrics
+   try: proba=self.adapter.predict_proba(split.X)
+   except Exception: proba=None
+   out.update(classification_metrics(split.y,pred,proba))
+  elif task in ("time_series",):
+   from ..metrics.time_series import time_series_metrics; out.update(time_series_metrics(split.y,pred))
+  elif task in ("anomaly_detection",):
+   from ..metrics.anomaly_detection import anomaly_metrics; out.update(anomaly_metrics(split.y,pred))
+  elif task in ("image_segmentation","segmentation"):
+   from ..metrics.segmentation import segmentation_metrics; out.update(segmentation_metrics(split.y,pred))
+  else:
+   from ..metrics.regression import regression_metrics; out.update(regression_metrics(split.y,pred))
+  return out
+ def analyse(self,force=False):
+  if self._result is not None and not force: return self._result
+  c=self.context; yref=c.train.y if c.train.y is not None else (c.test.y if c.test.y is not None else c.validation.y); task=detect_task(c.model,yref,c.task); r=ReportResult(metadata={"task":task,"library_version":"0.2.0"})
+  r.add("MODEL OVERVIEW",model_overview(c.model,self.adapter,task)); r.add("DATASET",{"train":describe_dataset(c.train.X,c.train.y),"validation":describe_dataset(c.validation.X,c.validation.y),"test":describe_dataset(c.test.X,c.test.y)}); r.add("TASK",{"detected_or_requested":task}); r.add("PARAMETERS",self.adapter.parameters())
+  fi=self.adapter.feature_importance(c.feature_names)
+  r.add("FEATURE IMPORTANCE",{"available":fi is not None,"ranking":fi[:50] if fi else []})
+  perf={};
+  for name,split in (("train",c.train),("validation",c.validation),("test",c.test)):
+   try:
+    value=self._split_analysis(name,split,task)
+    if value is not None: perf[name]=value
+   except Exception as e: r.warnings.append(f"{name} analysis skipped: {type(e).__name__}: {e}")
+  r.add("MODEL PERFORMANCE",perf)
+  cm=next((v.get("confusion_matrix") for v in (perf.get("test",{}),perf.get("validation",{}),perf.get("train",{})) if "confusion_matrix" in v),None); r.add("CONFUSION MATRIX",{"available":cm is not None,"matrix":cm})
+  r.add("ERROR ANALYSIS",self._errors(task,c.test if c.test.X is not None else c.validation if c.validation.X is not None else c.train))
+  self._figures(r,fi,cm,task,c)
+  if self.ai:
+   from ..ai.providers.groq import GroqProvider
+   from ..ai.explainer import explain
+   from ..ai.recommender import recommend
+   provider=GroqProvider(); r.add("AI EXPLANATION",{"text":explain(r,provider)}); r.recommendations.append(recommend(r,provider))
+  self._result=r; return r
+ def _errors(self,task,split):
+  if split.X is None or split.y is None: return {"available":False}
+  try:
+   pred=np.asarray(self.adapter.predict(split.X)); y=np.asarray(split.y)
+   if task in ("classification","computer_vision","nlp","text_classification","sentiment","anomaly_detection"):
+    idx=np.flatnonzero(pred!=y)[:25]; return {"available":True,"error_count":int(np.sum(pred!=y)),"examples":[{"index":int(i),"actual":str(y[i]),"predicted":str(pred[i])} for i in idx]}
+   err=np.abs(y-pred); idx=np.argsort(err.reshape(-1))[::-1][:25]; return {"available":True,"mean_absolute_error":float(np.mean(err)),"largest":[{"index":int(i),"actual":float(y.reshape(-1)[i]),"predicted":float(pred.reshape(-1)[i]),"absolute_error":float(err.reshape(-1)[i])} for i in idx]}
+  except Exception as e: return {"available":False,"reason":str(e)}
+ def _add_fig(self,r,fig,title):
+  b=io.BytesIO(); fig.savefig(b,format="png",dpi=130,bbox_inches="tight"); r.figures.append({"title":title,"data":base64.b64encode(b.getvalue()).decode()});
+  import matplotlib.pyplot as plt; plt.close(fig)
+ def _figures(self,r,fi,cm,task,c):
+  try:
+   if fi:
+    from ..visualization.feature_importance import plot_feature_importance; self._add_fig(r,plot_feature_importance(fi),"Feature importance")
+   if cm:
+    from ..visualization.confusion_matrix import plot_confusion_matrix; self._add_fig(r,plot_confusion_matrix(cm,c.target_names),"Confusion matrix")
+   split=c.test if c.test.X is not None else c.validation
+   if task in ("regression","time_series") and split.X is not None and split.y is not None:
+    from ..visualization.residuals import plot_residuals; self._add_fig(r,plot_residuals(split.y,self.adapter.predict(split.X)),"Residuals")
+  except Exception as e: r.warnings.append(f"Visualisation skipped: {e}")
+ def show(self):
+  from ..output.console import render_console
+  text=render_console(self.analyse()); print(text); return text
+ def html(self,path="modelbrief_report.html"):
+  from ..output.html import render_html; return render_html(self.analyse(),path)
+ def pdf(self,path="modelbrief_report.pdf"):
+  from ..output.pdf import render_pdf; return render_pdf(self.analyse(),path)
+ @property
+ def result(self): return self.analyse()

modelbrief/core/result.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any
+
+@dataclass
+class Section:
+    title: str
+    content: dict[str, Any] = field(default_factory=dict)
+    narrative: str | None = None
+
+@dataclass
+class ReportResult:
+    metadata: dict[str, Any] = field(default_factory=dict)
+    sections: list[Section] = field(default_factory=list)
+    figures: list[dict[str, Any]] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    recommendations: list[str] = field(default_factory=list)
+    def add(self, title: str, content: dict[str, Any], narrative: str | None = None):
+        self.sections.append(Section(title, content, narrative)); return self
+    def section(self, title: str):
+        return next((s for s in self.sections if s.title == title), None)
+    def to_dict(self):
+        return {"metadata":self.metadata,"sections":[{"title":s.title,"content":s.content,"narrative":s.narrative} for s in self.sections],"figures":self.figures,"warnings":self.warnings,"recommendations":self.recommendations}

modelbrief/metrics/anomaly_detection.py

@@ -0,0 +1,3 @@
+def anomaly_metrics(y,pred,scores=None):
+ from .classification import classification_metrics
+ return classification_metrics(y,pred,scores)

modelbrief/metrics/classification.py

@@ -0,0 +1,10 @@
+import numpy as np
+
+def classification_metrics(y,pred,proba=None):
+ from sklearn.metrics import accuracy_score,balanced_accuracy_score,precision_recall_fscore_support,confusion_matrix,log_loss,roc_auc_score
+ y=np.asarray(y); pred=np.asarray(pred); pr,rc,f1,_=precision_recall_fscore_support(y,pred,average="weighted",zero_division=0)
+ out={"accuracy":float(accuracy_score(y,pred)),"balanced_accuracy":float(balanced_accuracy_score(y,pred)),"precision_weighted":float(pr),"recall_weighted":float(rc),"f1_weighted":float(f1),"confusion_matrix":confusion_matrix(y,pred).tolist()}
+ if proba is not None:
+  try: out["log_loss"]=float(log_loss(y,proba)); out["roc_auc"]=float(roc_auc_score(y,proba,multi_class="ovr") if np.asarray(proba).ndim>1 and np.asarray(proba).shape[1]>2 else roc_auc_score(y,np.asarray(proba)[:,1] if np.asarray(proba).ndim>1 else proba))
+  except Exception: pass
+ return out

modelbrief/metrics/clustering.py

@@ -0,0 +1,6 @@
+def clustering_metrics(X,labels):
+ from sklearn.metrics import silhouette_score,calinski_harabasz_score,davies_bouldin_score
+ n=len(set(labels)); out={"clusters":n}
+ if n>1 and n<len(labels):
+  out.update(silhouette=float(silhouette_score(X,labels)),calinski_harabasz=float(calinski_harabasz_score(X,labels)),davies_bouldin=float(davies_bouldin_score(X,labels)))
+ return out

modelbrief/metrics/nlp.py

@@ -0,0 +1,2 @@
+def token_accuracy(y,pred):
+ total=sum(len(a) for a in y); correct=sum(sum(x==z for x,z in zip(a,b)) for a,b in zip(y,pred)); return correct/total if total else 0.0

modelbrief/metrics/object_detection.py

@@ -0,0 +1,2 @@
+def intersection_over_union(a,b):
+ x1=max(a[0],b[0]); y1=max(a[1],b[1]); x2=min(a[2],b[2]); y2=min(a[3],b[3]); inter=max(0,x2-x1)*max(0,y2-y1); union=max(0,a[2]-a[0])*max(0,a[3]-a[1])+max(0,b[2]-b[0])*max(0,b[3]-b[1])-inter; return inter/union if union else 0.0

modelbrief/metrics/regression.py

@@ -0,0 +1,6 @@
+import numpy as np
+
+def regression_metrics(y,pred):
+ from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score,median_absolute_error,explained_variance_score
+ y=np.asarray(y); pred=np.asarray(pred); mse=mean_squared_error(y,pred)
+ return {"mae":float(mean_absolute_error(y,pred)),"mse":float(mse),"rmse":float(np.sqrt(mse)),"r2":float(r2_score(y,pred)),"median_ae":float(median_absolute_error(y,pred)),"explained_variance":float(explained_variance_score(y,pred))}

modelbrief/metrics/segmentation.py

@@ -0,0 +1,3 @@
+import numpy as np
+def segmentation_metrics(y,pred):
+ y=np.asarray(y).astype(bool); p=np.asarray(pred).astype(bool); inter=np.logical_and(y,p).sum(); union=np.logical_or(y,p).sum(); return {"iou":float(inter/union) if union else 1.0,"dice":float(2*inter/(y.sum()+p.sum())) if y.sum()+p.sum() else 1.0}

modelbrief/metrics/time_series.py

@@ -0,0 +1,7 @@
+import numpy as np
+
+def time_series_metrics(y,pred):
+ from .regression import regression_metrics
+ out=regression_metrics(y,pred); y=np.asarray(y); p=np.asarray(pred); nz=np.abs(y)>1e-12
+ out["mape"]=float(np.mean(np.abs((y[nz]-p[nz])/y[nz]))*100) if nz.any() else None
+ out["smape"]=float(np.mean(2*np.abs(p-y)/(np.abs(y)+np.abs(p)+1e-12))*100); return out

modelbrief/metrics/vision.py

@@ -0,0 +1 @@
+from .classification import classification_metrics as image_classification_metrics