murthylytics 0.1.0__py3-none-any.whl

murthylytics/core/optional.py

@@ -0,0 +1,32 @@
+"""Helper for importing optional dependencies with a friendly error."""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+from typing import Optional
+
+from .exceptions import MissingDependencyError
+
+
+def optional_import(module: str, *, extra: Optional[str] = None) -> ModuleType:
+    """Import ``module`` or raise a helpful :class:`MissingDependencyError`.
+
+    Parameters
+    ----------
+    module:
+        Importable module name, e.g. ``"matplotlib.pyplot"``.
+    extra:
+        The Murthylytics extra that installs it, e.g. ``"viz"``. Used to build
+        the ``pip install 'murthylytics[viz]'`` hint.
+    """
+    try:
+        return importlib.import_module(module)
+    except ImportError as exc:  # pragma: no cover - exercised via feature tests
+        top = module.split(".")[0]
+        raise MissingDependencyError(top, extra) from exc
+
+
+def has_module(module: str) -> bool:
+    """Return True if ``module`` can be imported, without importing it fully."""
+    return importlib.util.find_spec(module) is not None

murthylytics/core/types.py

@@ -0,0 +1,50 @@
+"""Shared type definitions and enumerations used across Murthylytics."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    import os
+
+    PathLike = Union[str, "os.PathLike[str]"]
+else:
+    PathLike = Union[str, "object"]
+
+
+class ColumnRole(str, Enum):
+    """Semantic role inferred for a dataframe column."""
+
+    NUMERIC = "numeric"
+    CATEGORICAL = "categorical"
+    BOOLEAN = "boolean"
+    DATETIME = "datetime"
+    TEXT = "text"
+    CONSTANT = "constant"
+    HIGH_CARDINALITY = "high_cardinality"
+    ID_LIKE = "id_like"
+    UNKNOWN = "unknown"
+
+
+class TaskType(str, Enum):
+    """Machine-learning task inferred from the dataset and target."""
+
+    REGRESSION = "regression"
+    BINARY_CLASSIFICATION = "binary_classification"
+    MULTICLASS_CLASSIFICATION = "multiclass_classification"
+    CLUSTERING = "clustering"
+    TIME_SERIES = "time_series"
+    ANOMALY_DETECTION = "anomaly_detection"
+    NLP = "nlp"
+    RECOMMENDATION = "recommendation"
+    TABULAR = "tabular"
+    UNKNOWN = "unknown"
+
+
+class Severity(str, Enum):
+    """Severity level for a reported data-quality issue."""
+
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"

murthylytics/eda/__init__.py

@@ -0,0 +1 @@
+"""Murthylytics eda module (roadmap — see project README for status)."""

murthylytics/exploration/__init__.py

@@ -0,0 +1,36 @@
+"""Data exploration: display helpers and intelligent inspection."""
+
+from __future__ import annotations
+
+from .display import (
+    bottom,
+    columns,
+    describe,
+    duplicates,
+    info,
+    memory,
+    missing,
+    shape,
+    summary,
+    top,
+    types,
+)
+from .inspect import ColumnProfile, InspectionReport, Issue, inspect
+
+__all__ = [
+    "top",
+    "bottom",
+    "shape",
+    "columns",
+    "types",
+    "missing",
+    "duplicates",
+    "memory",
+    "summary",
+    "describe",
+    "info",
+    "inspect",
+    "InspectionReport",
+    "ColumnProfile",
+    "Issue",
+]

murthylytics/exploration/display.py

@@ -0,0 +1,99 @@
+"""Lightweight dataframe exploration helpers.
+
+These are pure functions over a :class:`pandas.DataFrame`; the facade binds them
+to the active session so users can call ``mlt.summary()`` with no arguments.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+import pandas as pd
+
+from ..io.optimize import memory_report
+
+
+def top(df: pd.DataFrame, n: int = 5) -> pd.DataFrame:
+    """Return the first ``n`` rows."""
+    return df.head(n)
+
+
+def bottom(df: pd.DataFrame, n: int = 5) -> pd.DataFrame:
+    """Return the last ``n`` rows."""
+    return df.tail(n)
+
+
+def shape(df: pd.DataFrame) -> Dict[str, int]:
+    """Return row/column counts as a dict."""
+    rows, cols = df.shape
+    return {"rows": int(rows), "columns": int(cols)}
+
+
+def columns(df: pd.DataFrame) -> list:
+    """Return the list of column names."""
+    return list(df.columns)
+
+
+def types(df: pd.DataFrame) -> Dict[str, str]:
+    """Return a mapping of column name to dtype string."""
+    return {c: str(t) for c, t in df.dtypes.items()}
+
+
+def missing(df: pd.DataFrame) -> pd.DataFrame:
+    """Return per-column missing counts and percentages, sorted worst-first."""
+    count = df.isna().sum()
+    pct = (count / len(df) * 100).round(2) if len(df) else count.astype(float)
+    out = pd.DataFrame({"missing_count": count, "missing_pct": pct})
+    return out[out["missing_count"] > 0].sort_values("missing_count", ascending=False)
+
+
+def duplicates(df: pd.DataFrame) -> Dict[str, Any]:
+    """Return duplicate-row statistics."""
+    dup_mask = df.duplicated()
+    total = int(dup_mask.sum())
+    return {
+        "duplicate_rows": total,
+        "duplicate_pct": round(total / len(df) * 100, 2) if len(df) else 0.0,
+        "unique_rows": int(len(df) - total),
+    }
+
+
+def memory(df: pd.DataFrame) -> Dict[str, float]:
+    """Return a per-dtype memory footprint report in MB."""
+    return memory_report(df)
+
+
+def summary(df: pd.DataFrame) -> Dict[str, Any]:
+    """Return a compact, JSON-serialisable overview of the dataframe."""
+    miss = int(df.isna().sum().sum())
+    cells = int(df.size) or 1
+    return {
+        "shape": shape(df),
+        "columns": len(df.columns),
+        "memory_mb": round(memory_report(df)["total"], 3),
+        "missing_cells": miss,
+        "missing_pct": round(miss / cells * 100, 2),
+        "duplicate_rows": duplicates(df)["duplicate_rows"],
+        "numeric_columns": len(df.select_dtypes(include="number").columns),
+        "categorical_columns": len(df.select_dtypes(include=["object", "category"]).columns),
+        "datetime_columns": len(df.select_dtypes(include="datetime").columns),
+        "boolean_columns": len(df.select_dtypes(include="bool").columns),
+    }
+
+
+def describe(df: pd.DataFrame, include: str = "all") -> pd.DataFrame:
+    """Descriptive statistics wrapper around :meth:`pandas.DataFrame.describe`."""
+    try:
+        return df.describe(include=include)
+    except ValueError:
+        return df.describe()
+
+
+def info(df: pd.DataFrame) -> Dict[str, Any]:
+    """Structured equivalent of :meth:`pandas.DataFrame.info` as a dict."""
+    return {
+        "shape": shape(df),
+        "dtypes": types(df),
+        "non_null_counts": {c: int(df[c].notna().sum()) for c in df.columns},
+        "memory_mb": round(memory_report(df)["total"], 3),
+    }

murthylytics/exploration/inspect.py

@@ -0,0 +1,337 @@
+"""Intelligent dataset inspection.
+
+:func:`inspect` walks a dataframe once and produces an :class:`InspectionReport`
+describing dimensions, column roles, missingness, duplicates, likely target
+column(s), class imbalance, strong correlations, data-quality issues, potential
+data-leakage risks and concrete recommendations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from ..core.config import DEFAULT_CONFIG, Config
+from ..core.types import ColumnRole, Severity
+
+
+@dataclass
+class ColumnProfile:
+    """Per-column facts inferred during inspection."""
+
+    name: str
+    dtype: str
+    role: ColumnRole
+    missing_count: int
+    missing_pct: float
+    unique_count: int
+    unique_ratio: float
+    sample_values: List[Any] = field(default_factory=list)
+
+
+@dataclass
+class Issue:
+    """A single data-quality finding."""
+
+    severity: Severity
+    column: Optional[str]
+    message: str
+
+
+@dataclass
+class InspectionReport:
+    """Structured result of :func:`inspect`."""
+
+    n_rows: int
+    n_columns: int
+    memory_mb: float
+    duplicate_rows: int
+    total_missing_cells: int
+    missing_pct: float
+    columns: List[ColumnProfile] = field(default_factory=list)
+    roles: Dict[str, List[str]] = field(default_factory=dict)
+    potential_targets: List[str] = field(default_factory=list)
+    class_imbalance: Dict[str, Any] = field(default_factory=dict)
+    strong_correlations: List[Tuple[str, str, float]] = field(default_factory=list)
+    leakage_risks: List[str] = field(default_factory=list)
+    issues: List[Issue] = field(default_factory=list)
+    recommendations: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a fully JSON-serialisable representation."""
+        d = asdict(self)
+        # Enums -> their string values.
+        for col in d["columns"]:
+            col["role"] = col["role"].value if hasattr(col["role"], "value") else str(col["role"])
+        for iss in d["issues"]:
+            iss["severity"] = (
+                iss["severity"].value if hasattr(iss["severity"], "value") else str(iss["severity"])
+            )
+        return d
+
+    def __repr__(self) -> str:  # pragma: no cover - cosmetic
+        lines = [
+            "InspectionReport",
+            f"  shape           : {self.n_rows} rows x {self.n_columns} cols",
+            f"  memory          : {self.memory_mb:.3f} MB",
+            f"  missing cells   : {self.total_missing_cells} ({self.missing_pct:.2f}%)",
+            f"  duplicate rows  : {self.duplicate_rows}",
+            f"  potential target: {', '.join(self.potential_targets) or 'none detected'}",
+            f"  issues          : {len(self.issues)}",
+            f"  recommendations : {len(self.recommendations)}",
+        ]
+        return "\n".join(lines)
+
+
+def _classify_column(s: pd.Series, n_rows: int, config: Config) -> Tuple[ColumnRole, int, float]:
+    """Infer a semantic :class:`ColumnRole` for a single series."""
+    nunique = int(s.nunique(dropna=True))
+    unique_ratio = nunique / n_rows if n_rows else 0.0
+
+    if nunique <= 1:
+        return ColumnRole.CONSTANT, nunique, unique_ratio
+
+    if pd.api.types.is_bool_dtype(s):
+        return ColumnRole.BOOLEAN, nunique, unique_ratio
+    if pd.api.types.is_datetime64_any_dtype(s):
+        return ColumnRole.DATETIME, nunique, unique_ratio
+
+    if pd.api.types.is_numeric_dtype(s):
+        # A near-unique integer column is more likely an identifier than a feature.
+        if pd.api.types.is_integer_dtype(s) and unique_ratio >= config.id_unique_ratio:
+            return ColumnRole.ID_LIKE, nunique, unique_ratio
+        return ColumnRole.NUMERIC, nunique, unique_ratio
+
+    # Object / string / category.
+    if unique_ratio >= config.id_unique_ratio:
+        # Distinguish free-text from identifiers by average token length.
+        avg_len = s.dropna().astype(str).str.len().mean() if nunique else 0
+        if avg_len and avg_len > 40:
+            return ColumnRole.TEXT, nunique, unique_ratio
+        return ColumnRole.ID_LIKE, nunique, unique_ratio
+    if unique_ratio >= config.high_cardinality_ratio:
+        return ColumnRole.HIGH_CARDINALITY, nunique, unique_ratio
+    # Long strings with moderate cardinality read as text (reviews, descriptions).
+    avg_len = s.dropna().astype(str).str.len().mean() if nunique else 0
+    if avg_len and avg_len > 60:
+        return ColumnRole.TEXT, nunique, unique_ratio
+    return ColumnRole.CATEGORICAL, nunique, unique_ratio
+
+
+def _detect_targets(profiles: List[ColumnProfile], df: pd.DataFrame) -> List[str]:
+    """Heuristically rank columns by their likelihood of being the target.
+
+    Signals: conventional names, being the last column, low-cardinality
+    categoricals (classification) and complete numeric columns (regression).
+    """
+    name_hints = {"target", "label", "class", "y", "outcome", "output", "result", "churn"}
+    scored: List[Tuple[float, str]] = []
+    for i, p in enumerate(profiles):
+        if p.role in (ColumnRole.ID_LIKE, ColumnRole.CONSTANT, ColumnRole.TEXT):
+            continue
+        score = 0.0
+        lname = p.name.lower()
+        if lname in name_hints:
+            score += 5.0
+        elif any(h in lname for h in name_hints if len(h) >= 4):
+            # Substring matching only for longer hints, so "y" doesn't match "city".
+            score += 3.0
+        if i == len(profiles) - 1:  # last column convention
+            score += 1.5
+        if p.role in (ColumnRole.CATEGORICAL, ColumnRole.BOOLEAN) and 2 <= p.unique_count <= 20:
+            score += 2.0
+        if p.missing_count == 0:
+            score += 0.5
+        if score > 0:
+            scored.append((score, p.name))
+    scored.sort(reverse=True)
+    return [name for _, name in scored[:3]]
+
+
+def _class_imbalance(df: pd.DataFrame, target: str, config: Config) -> Dict[str, Any]:
+    """Compute class-frequency imbalance for a candidate target column."""
+    counts = df[target].value_counts(dropna=True)
+    if counts.empty:
+        return {}
+    minority, majority = int(counts.min()), int(counts.max())
+    ratio = minority / majority if majority else 0.0
+    return {
+        "target": target,
+        "n_classes": int(counts.size),
+        "class_counts": {str(k): int(v) for k, v in counts.head(20).items()},
+        "imbalance_ratio": round(ratio, 4),
+        "is_imbalanced": bool(ratio < config.class_imbalance_ratio and counts.size > 1),
+    }
+
+
+def _strong_correlations(
+    df: pd.DataFrame, config: Config
+) -> List[Tuple[str, str, float]]:
+    """Return numeric column pairs with |correlation| above the threshold."""
+    num = df.select_dtypes(include="number")
+    if num.shape[1] < 2:
+        return []
+    corr = num.corr(numeric_only=True).abs()
+    pairs: List[Tuple[str, str, float]] = []
+    cols = corr.columns
+    for i in range(len(cols)):
+        for j in range(i + 1, len(cols)):
+            val = corr.iloc[i, j]
+            if pd.notna(val) and val >= config.correlation_threshold:
+                pairs.append((cols[i], cols[j], round(float(val), 4)))
+    pairs.sort(key=lambda t: t[2], reverse=True)
+    return pairs
+
+
+def inspect(df: pd.DataFrame, *, config: Config = DEFAULT_CONFIG) -> InspectionReport:
+    """Run a full intelligent inspection of ``df`` and return a report."""
+    n_rows, n_cols = df.shape
+    total_missing = int(df.isna().sum().sum())
+    cells = df.size or 1
+
+    profiles: List[ColumnProfile] = []
+    roles: Dict[str, List[str]] = {r.value: [] for r in ColumnRole}
+    issues: List[Issue] = []
+
+    for col in df.columns:
+        s = df[col]
+        role, nunique, uratio = _classify_column(s, n_rows, config)
+        miss = int(s.isna().sum())
+        profiles.append(
+            ColumnProfile(
+                name=str(col),
+                dtype=str(s.dtype),
+                role=role,
+                missing_count=miss,
+                missing_pct=round(miss / n_rows * 100, 2) if n_rows else 0.0,
+                unique_count=nunique,
+                unique_ratio=round(uratio, 4),
+                sample_values=[_safe(v) for v in s.dropna().unique()[:5]],
+            )
+        )
+        roles[role.value].append(str(col))
+
+        # Per-column data-quality issues.
+        if role == ColumnRole.CONSTANT:
+            issues.append(Issue(Severity.WARNING, str(col), "Column is constant (no variance)."))
+        if n_rows and miss / n_rows >= config.missing_drop_column_ratio:
+            issues.append(
+                Issue(
+                    Severity.CRITICAL,
+                    str(col),
+                    f"{miss / n_rows * 100:.1f}% missing — consider dropping.",
+                )
+            )
+        if role == ColumnRole.ID_LIKE:
+            issues.append(
+                Issue(Severity.INFO, str(col), "Looks like an identifier — likely not predictive.")
+            )
+        if _has_infinite(s):
+            issues.append(Issue(Severity.WARNING, str(col), "Contains infinite values."))
+
+    # Dataset-level findings.
+    duplicate_rows = int(df.duplicated().sum())
+    if duplicate_rows:
+        issues.append(
+            Issue(Severity.WARNING, None, f"{duplicate_rows} duplicate rows detected.")
+        )
+
+    potential_targets = _detect_targets(profiles, df)
+    class_imbalance: Dict[str, Any] = {}
+    if potential_targets:
+        ci = _class_imbalance(df, potential_targets[0], config)
+        if ci.get("is_imbalanced"):
+            issues.append(
+                Issue(
+                    Severity.WARNING,
+                    potential_targets[0],
+                    f"Class imbalance (ratio={ci['imbalance_ratio']}). Consider resampling.",
+                )
+            )
+        class_imbalance = ci
+
+    strong = _strong_correlations(df, config)
+    leakage_risks: List[str] = []
+    if potential_targets:
+        tgt = potential_targets[0]
+        for a, b, v in strong:
+            if tgt in (a, b) and v >= 0.98:
+                other = b if a == tgt else a
+                leakage_risks.append(other)
+                issues.append(
+                    Issue(
+                        Severity.CRITICAL,
+                        other,
+                        f"Almost perfectly correlated with target '{tgt}' (r={v}); "
+                        "possible data leakage.",
+                    )
+                )
+
+    report = InspectionReport(
+        n_rows=int(n_rows),
+        n_columns=int(n_cols),
+        memory_mb=round(float(df.memory_usage(deep=True).sum()) / 1e6, 3),
+        duplicate_rows=duplicate_rows,
+        total_missing_cells=total_missing,
+        missing_pct=round(total_missing / cells * 100, 2),
+        columns=profiles,
+        roles={k: v for k, v in roles.items() if v},
+        potential_targets=potential_targets,
+        class_imbalance=class_imbalance,
+        strong_correlations=strong,
+        leakage_risks=leakage_risks,
+        issues=issues,
+    )
+    report.recommendations = _recommendations(report)
+    return report
+
+
+def _recommendations(r: InspectionReport) -> List[str]:
+    """Derive actionable recommendations from the assembled report."""
+    recs: List[str] = []
+    if r.duplicate_rows:
+        recs.append(f"Remove {r.duplicate_rows} duplicate rows with `mlt.clean_data()`.")
+    if r.total_missing_cells:
+        recs.append("Impute or drop missing values before modelling.")
+    if r.roles.get(ColumnRole.ID_LIKE.value):
+        recs.append(
+            "Drop identifier columns "
+            f"({', '.join(r.roles[ColumnRole.ID_LIKE.value])}) — they add no signal."
+        )
+    if r.roles.get(ColumnRole.CONSTANT.value):
+        recs.append("Drop constant columns — they carry no information.")
+    if r.roles.get(ColumnRole.HIGH_CARDINALITY.value):
+        recs.append("Use target/frequency encoding for high-cardinality categoricals.")
+    if r.leakage_risks:
+        recs.append(
+            f"Investigate potential leakage from {', '.join(r.leakage_risks)} before training."
+        )
+    if r.class_imbalance.get("is_imbalanced"):
+        recs.append("Address class imbalance (e.g. SMOTE, class weights) during preprocessing.")
+    if r.strong_correlations:
+        recs.append("Consider dropping one of each strongly correlated numeric pair.")
+    if not recs:
+        recs.append("No major issues detected — data looks clean.")
+    return recs
+
+
+def _safe(value: Any) -> Any:
+    """Coerce numpy/pandas scalars to plain Python for JSON-friendliness."""
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        return float(value)
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    return value
+
+
+def _has_infinite(s: pd.Series) -> bool:
+    if not pd.api.types.is_float_dtype(s):
+        return False
+    return bool(np.isinf(s.to_numpy(dtype="float64", na_value=np.nan)).any())

murthylytics/features/__init__.py

@@ -0,0 +1 @@
+"""Murthylytics features module (roadmap — see project README for status)."""

murthylytics/io/__init__.py

@@ -0,0 +1,28 @@
+"""Data input/output: smart readers, detection and memory optimisation."""
+
+from __future__ import annotations
+
+from .detect import (
+    coerce_datetimes,
+    detect_datetime_columns,
+    detect_delimiter,
+    detect_encoding,
+)
+from .optimize import estimate_savings, memory_report, memory_usage_bytes, optimize_memory
+from .readers import read_csv, read_excel, read_json, read_parquet, read_sql
+
+__all__ = [
+    "read_csv",
+    "read_excel",
+    "read_json",
+    "read_parquet",
+    "read_sql",
+    "detect_encoding",
+    "detect_delimiter",
+    "detect_datetime_columns",
+    "coerce_datetimes",
+    "optimize_memory",
+    "memory_report",
+    "memory_usage_bytes",
+    "estimate_savings",
+]