dissectml 0.1.0__py3-none-any.whl

dissectml/__init__.py

@@ -0,0 +1,155 @@
+"""
+InsightML — The missing middle layer between EDA and AutoML.
+
+Unified pipeline from deep data understanding to model comparison,
+in as few as 3 function calls.
+
+Quick start::
+
+    import dissectml as iml
+
+    # Deep EDA (v0.1+)
+    eda = iml.explore(df)
+    eda.overview.show()
+    eda.correlations.heatmap()
+
+    # Model battle (v0.2+)
+    models = iml.battle(df, target="price")
+
+    # Full pipeline (v0.4+)
+    report = iml.analyze(df, target="price")
+    report.export("report.html")
+"""
+
+from __future__ import annotations
+
+from dissectml._compat import to_pandas
+from dissectml._config import InsightMLConfig, config_context, get_config, set_config
+from dissectml._version import __version__
+from dissectml.battle import battle
+from dissectml.compare import ModelComparator
+from dissectml.datasets import load_housing, load_titanic
+from dissectml.eda import explore
+from dissectml.exceptions import InsightMLError
+from dissectml.intelligence import analyze_intelligence
+from dissectml.report import AnalysisReport
+
+__all__ = [
+    "__version__",
+    # Public API
+    "explore",
+    "battle",
+    "analyze_intelligence",
+    "analyze",
+    "ModelComparator",
+    "AnalysisReport",
+    # Datasets
+    "load_titanic",
+    "load_housing",
+    # Compat
+    "to_pandas",
+    # Config
+    "InsightMLConfig",
+    "get_config",
+    "set_config",
+    "config_context",
+    # Exceptions
+    "InsightMLError",
+]
+
+
+def analyze(
+    df,
+    target: str,
+    *,
+    task: str | None = None,
+    run_battle: bool = True,
+    battle_families: list[str] | None = None,
+    battle_models: list[str] | None = None,
+    battle_exclude: list[str] | None = None,
+    cv: int | None = None,
+    n_jobs: int | None = None,
+    datetime_col: str | None = None,
+) -> AnalysisReport:
+    """Full pipeline: EDA → Intelligence → Battle → Compare → Report.
+
+    Runs all five stages and returns an :class:`~dissectml.report.AnalysisReport`
+    that can be inspected interactively or exported to HTML.
+
+    Args:
+        df: Input DataFrame (features + target column).
+        target: Name of the target column.
+        task: ``"classification"`` or ``"regression"``. Inferred if None.
+        run_battle: If False, skip Stages 3-4 (EDA + Intelligence only).
+        battle_families: Filter models by family for the battle stage.
+        battle_models: Explicit model names for the battle stage.
+        battle_exclude: Model names to exclude from battle.
+        cv: CV folds for the battle stage.
+        n_jobs: Parallel workers for the battle stage.
+        datetime_col: Optional datetime column for temporal leakage detection.
+
+    Returns:
+        :class:`~dissectml.report.AnalysisReport`
+
+    Example::
+
+        import dissectml as iml
+        report = iml.analyze(df, target="survived")
+        report.summary()
+        report.export("report.html")
+    """
+    import pandas as pd
+
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(f"df must be a pandas DataFrame, got {type(df).__name__}")
+    if target not in df.columns:
+        raise KeyError(f"Target column '{target}' not found in DataFrame.")
+
+    feature_cols = [c for c in df.columns if c != target]
+    n_samples = len(df)
+    n_features = len(feature_cols)
+
+    # Infer task if not provided
+    if task is None or task == "auto":
+        from dissectml.battle.runner import _infer_task
+        task = _infer_task(df[target])
+
+    # Stage 1: EDA
+    eda_result = explore(df, target=target)
+
+    # Stage 2: Intelligence
+    intel_result = analyze_intelligence(
+        df, target=target, task=task,
+        datetime_col=datetime_col,
+        eda_result=eda_result,
+    )
+
+    if not run_battle:
+        return AnalysisReport(
+            task=task, target=target,
+            n_samples=n_samples, n_features=n_features,
+            eda=eda_result, intelligence=intel_result,
+        )
+
+    # Stage 3: Battle
+    battle_result = battle(
+        df, target=target, task=task,
+        families=battle_families,
+        models=battle_models,
+        exclude=battle_exclude,
+        cv=cv,
+        n_jobs=n_jobs,
+        eda_result=eda_result,
+    )
+
+    # Stage 4: Compare
+    X = df.drop(columns=[target])
+    y = df[target]
+    comparator = ModelComparator(battle_result, X=X, y=y)
+
+    return AnalysisReport(
+        task=task, target=target,
+        n_samples=n_samples, n_features=n_features,
+        eda=eda_result, intelligence=intel_result,
+        models=battle_result, compare=comparator,
+    )

dissectml/_compat.py

@@ -0,0 +1,111 @@
+"""Compatibility layer — accept Polars DataFrames, file paths, and dicts in public API.
+
+Usage::
+
+    from dissectml._compat import to_pandas
+
+    df = to_pandas(user_input)  # works for pd.DataFrame, pl.DataFrame, Path, str, dict, list
+
+All public-facing functions that accept data should call ``to_pandas`` first.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+
+def to_pandas(data: Any, **read_csv_kwargs: Any) -> pd.DataFrame:
+    """Convert various input types to a pandas DataFrame.
+
+    Supported inputs:
+
+    * ``pandas.DataFrame`` — returned unchanged.
+    * ``polars.DataFrame`` or ``polars.LazyFrame`` — converted via ``.to_pandas()``.
+    * ``str`` / ``pathlib.Path`` — read as CSV (pass extra kwargs to ``pd.read_csv``).
+    * ``dict`` — passed to ``pd.DataFrame.from_dict``.
+    * ``list[dict]`` — passed to ``pd.DataFrame(data)``.
+    * ``numpy.ndarray`` — passed to ``pd.DataFrame(data)``.
+
+    Args:
+        data: Input data.
+        **read_csv_kwargs: Forwarded to :func:`pandas.read_csv` when *data* is a path.
+
+    Returns:
+        pandas DataFrame.
+
+    Raises:
+        TypeError: If *data* is an unsupported type.
+    """
+    if isinstance(data, pd.DataFrame):
+        return data
+
+    # --- Polars (optional dep) ---
+    try:
+        import polars as pl  # noqa: F401 (only imported if available)
+
+        if isinstance(data, pl.DataFrame):
+            return data.to_pandas()
+        if isinstance(data, pl.LazyFrame):
+            return data.collect().to_pandas()
+    except ImportError:
+        pass  # Polars not installed — skip
+
+    # --- File path ---
+    if isinstance(data, (str, Path)):
+        path = Path(data)
+        suffix = path.suffix.lower()
+        if suffix == ".csv":
+            return pd.read_csv(path, **read_csv_kwargs)
+        if suffix in (".parquet", ".pq"):
+            return pd.read_parquet(path, **read_csv_kwargs)
+        if suffix in (".xlsx", ".xls"):
+            return pd.read_excel(path, **read_csv_kwargs)
+        if suffix == ".json":
+            return pd.read_json(path, **read_csv_kwargs)
+        # Fallback: try CSV
+        return pd.read_csv(path, **read_csv_kwargs)
+
+    # --- dict / list / array-like ---
+    if isinstance(data, dict):
+        return pd.DataFrame(data)
+
+    if isinstance(data, list):
+        return pd.DataFrame(data)
+
+    # --- numpy ndarray ---
+    try:
+        import numpy as np
+
+        if isinstance(data, np.ndarray):
+            return pd.DataFrame(data)
+    except ImportError:
+        pass
+
+    raise TypeError(
+        f"Unsupported data type: {type(data).__name__}. "
+        "Expected pandas DataFrame, polars DataFrame, file path (str/Path), "
+        "dict, or list of dicts."
+    )
+
+
+def is_polars_available() -> bool:
+    """Return True if polars is importable."""
+    try:
+        import polars  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def get_pandas_version() -> tuple[int, ...]:
+    """Return pandas version as a tuple of ints, e.g. (2, 1, 0)."""
+    parts = pd.__version__.split(".")
+    result = []
+    for p in parts:
+        # Strip any non-numeric suffix like "1rc1"
+        num = "".join(c for c in p if c.isdigit())
+        result.append(int(num) if num else 0)
+    return tuple(result)

dissectml/_config.py

@@ -0,0 +1,110 @@
+"""Global configuration for InsightML with context manager support."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import threading
+from collections.abc import Generator
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class InsightMLConfig:
+    """Configuration dataclass for InsightML.
+
+    Three override levels (highest to lowest priority):
+        per-call kwargs > config_context() > set_config() > defaults
+    """
+
+    # --- EDA ---
+    categorical_threshold: int = 50          # nunique <= this -> CATEGORICAL
+    high_cardinality_threshold: int = 100    # nunique > this -> HIGH_CARDINALITY
+    text_min_avg_length: int = 30            # avg str length > this -> TEXT
+    significance_level: float = 0.05        # p-value threshold for all tests
+    iqr_multiplier: float = 1.5             # outlier IQR fence multiplier
+    zscore_threshold: float = 3.0           # outlier z-score cutoff
+    isolation_forest_contamination: float = 0.05
+    max_k_clusters: int = 10               # max K for auto K-Means
+    max_bivariate_pairs: int = 30          # pair limit for bivariate analysis
+    correlation_methods: list[str] = field(
+        default_factory=lambda: ["pearson", "spearman", "cramers_v"]
+    )
+
+    # --- Battle ---
+    cv_folds: int = 5
+    timeout_per_model: int = 300            # seconds per model during training
+    n_jobs: int = -1                        # joblib parallelism (-1 = all cores)
+    random_state: int = 42
+
+    # --- Scale ---
+    large_dataset_threshold: int = 100_000  # rows; triggers auto-sampling
+    sample_size: int = 50_000              # subsample size for expensive ops
+
+    # --- Report ---
+    report_theme: str = "default"
+    plotly_template: str = "plotly_white"
+
+    # --- General ---
+    verbosity: int = 1                      # 0=silent, 1=progress, 2=debug
+
+    def copy_with(self, **kwargs: Any) -> InsightMLConfig:
+        """Return a copy of this config with the given fields overridden."""
+        cfg = copy.copy(self)
+        for key, value in kwargs.items():
+            if not hasattr(cfg, key):
+                raise ValueError(f"Unknown config key: {key!r}")
+            setattr(cfg, key, value)
+        return cfg
+
+
+# ---------------------------------------------------------------------------
+# Global config state (thread-local for context manager support)
+# ---------------------------------------------------------------------------
+
+_global_config = InsightMLConfig()
+_thread_local = threading.local()
+
+
+def get_config() -> InsightMLConfig:
+    """Return the currently active configuration.
+
+    Returns thread-local config if inside config_context(), else the global config.
+    """
+    return getattr(_thread_local, "config", _global_config)
+
+
+def set_config(**kwargs: Any) -> None:
+    """Update the global default configuration.
+
+    Example::
+
+        iml.set_config(cv_folds=10, verbosity=0)
+    """
+    global _global_config
+    for key, value in kwargs.items():
+        if not hasattr(_global_config, key):
+            raise ValueError(f"Unknown config key: {key!r}")
+        setattr(_global_config, key, value)
+
+
+@contextlib.contextmanager
+def config_context(**kwargs: Any) -> Generator[InsightMLConfig, None, None]:
+    """Temporarily override configuration within a with-block.
+
+    Example::
+
+        with iml.config_context(cv_folds=3):
+            result = iml.battle(df, target="y")
+    """
+    old_config = getattr(_thread_local, "config", None)
+    base = old_config if old_config is not None else _global_config
+    _thread_local.config = base.copy_with(**kwargs)
+    try:
+        yield _thread_local.config
+    finally:
+        if old_config is None:
+            del _thread_local.config
+        else:
+            _thread_local.config = old_config

dissectml/_io.py

@@ -0,0 +1,63 @@
+"""File I/O helpers — load DataFrames from CSV, Excel, Parquet, or JSON."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from dissectml.exceptions import DependencyError, UnsupportedFormatError
+
+# Supported extensions and their pandas readers
+_READERS: dict[str, Any] = {
+    ".csv": pd.read_csv,
+    ".tsv": lambda p, **kw: pd.read_csv(p, sep="\t", **kw),
+    ".xlsx": pd.read_excel,
+    ".xls": pd.read_excel,
+    ".parquet": pd.read_parquet,
+    ".json": pd.read_json,
+}
+
+SUPPORTED_EXTENSIONS = list(_READERS.keys())
+
+
+def read_data(path: str | Path, **kwargs: Any) -> pd.DataFrame:
+    """Load a DataFrame from a file path.
+
+    Supports CSV, TSV, Excel (.xlsx/.xls), Parquet, and JSON.
+    Extra kwargs are passed directly to the underlying pandas reader.
+
+    Args:
+        path: Path to the data file.
+        **kwargs: Additional keyword arguments forwarded to the pandas reader.
+
+    Returns:
+        Loaded DataFrame.
+
+    Raises:
+        UnsupportedFormatError: If the file extension is not supported.
+        DependencyError: If a required optional reader package is missing.
+        FileNotFoundError: If the file does not exist.
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Data file not found: {path}")
+
+    ext = path.suffix.lower()
+    if ext not in _READERS:
+        raise UnsupportedFormatError(
+            f"Unsupported file format: '{ext}'. "
+            f"Supported formats: {SUPPORTED_EXTENSIONS}"
+        )
+
+    reader = _READERS[ext]
+    try:
+        return reader(path, **kwargs)
+    except ImportError as exc:
+        # e.g., openpyxl not installed for .xlsx, pyarrow missing for .parquet
+        missing = str(exc).split("'")[1] if "'" in str(exc) else str(exc)
+        raise DependencyError(
+            f"Reading '{ext}' files requires '{missing}'. "
+            f"Install it with: pip install {missing}"
+        ) from exc

dissectml/_lazy.py

@@ -0,0 +1,61 @@
+"""Optional dependency guard for InsightML extras."""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+
+from dissectml.exceptions import OptionalDependencyError
+
+# Maps package import name -> (install name, extra group)
+_EXTRA_MAP: dict[str, tuple[str, str]] = {
+    "xgboost": ("xgboost", "boost"),
+    "lightgbm": ("lightgbm", "boost"),
+    "catboost": ("catboost", "boost"),
+    "shap": ("shap", "explain"),
+    "weasyprint": ("weasyprint", "report"),
+    "kaleido": ("kaleido", "report"),
+    "polars": ("polars", "scale"),
+    "optuna": ("optuna", "scale"),
+    "openpyxl": ("openpyxl", "pip install openpyxl"),
+    "pyarrow": ("pyarrow", "pip install pyarrow"),
+}
+
+
+def require(package_name: str, extra: str | None = None) -> ModuleType:
+    """Import an optional package, raising a helpful error if missing.
+
+    Args:
+        package_name: The import name of the package (e.g., "xgboost").
+        extra: The dissectml extra group to suggest (e.g., "boost").
+               If None, inferred from _EXTRA_MAP or shown as bare install.
+
+    Returns:
+        The imported module.
+
+    Raises:
+        OptionalDependencyError: If the package is not installed.
+    """
+    try:
+        return importlib.import_module(package_name)
+    except ImportError:
+        if extra is None:
+            entry = _EXTRA_MAP.get(package_name)
+            extra = entry[1] if entry else package_name
+        if extra.startswith("pip install"):
+            install_hint = extra
+        else:
+            install_hint = f"pip install dissectml[{extra}]"
+        raise OptionalDependencyError(
+            f"Optional dependency '{package_name}' is not installed. "
+            f"Install it with: {install_hint}"
+        ) from None
+
+
+def is_available(package_name: str) -> bool:
+    """Check if an optional package is importable (no error raised)."""
+    try:
+        importlib.import_module(package_name)
+        return True
+    except ImportError:
+        return False

dissectml/_sampling.py

@@ -0,0 +1,72 @@
+"""Smart sampling strategies for large datasets."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from dissectml._config import InsightMLConfig
+
+
+def smart_sample(
+    df: pd.DataFrame,
+    target: str | None = None,
+    config: InsightMLConfig | None = None,
+    *,
+    force: bool = False,
+) -> pd.DataFrame:
+    """Return a representative subsample if the DataFrame exceeds the size threshold.
+
+    Sampling strategy (in priority order):
+    1. Stratified — when target is provided (preserves class distribution)
+    2. Temporal — when a datetime column is detected (preserves time ordering)
+    3. Random — fallback with fixed random_state for reproducibility
+
+    Args:
+        df: Input DataFrame.
+        target: Target column name (enables stratified sampling if given).
+        config: InsightML configuration. Uses global config if None.
+        force: If True, always sample even below threshold.
+
+    Returns:
+        The original DataFrame if small enough, otherwise a subsample.
+    """
+    if config is None:
+        from dissectml._config import get_config
+        config = get_config()
+
+    n = len(df)
+    if not force and n <= config.large_dataset_threshold:
+        return df
+
+    sample_size = min(config.sample_size, n)
+
+    # --- Stratified sampling (classification target present) ---
+    if target is not None and target in df.columns:
+        target_col = df[target]
+        if target_col.dtype in ("object", "category", "bool") or (
+            target_col.nunique() <= 50
+        ):
+            try:
+                return df.groupby(target, group_keys=False).apply(
+                    lambda g: g.sample(
+                        frac=sample_size / n,
+                        random_state=config.random_state,
+                    )
+                ).reset_index(drop=True)
+            except Exception:
+                pass  # Fall through to temporal or random
+
+    # --- Temporal sampling (datetime column present) ---
+    datetime_cols = df.select_dtypes(include=["datetime64"]).columns.tolist()
+    if datetime_cols:
+        dt_col = datetime_cols[0]
+        sorted_df = df.sort_values(dt_col)
+        # Sample evenly spaced indices to preserve time distribution
+        step = max(1, n // sample_size)
+        indices = list(range(0, n, step))[:sample_size]
+        return sorted_df.iloc[indices].reset_index(drop=True)
+
+    # --- Random sampling (fallback) ---
+    return df.sample(n=sample_size, random_state=config.random_state).reset_index(
+        drop=True
+    )

dissectml/_types.py

@@ -0,0 +1,114 @@
+"""Enums, TypedDicts, and type aliases used across InsightML."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, TypedDict
+
+import numpy as np
+import pandas as pd
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+class TaskType(str, Enum):
+    CLASSIFICATION = "classification"
+    REGRESSION = "regression"
+    AUTO = "auto"
+
+
+class ColumnType(str, Enum):
+    NUMERIC = "numeric"
+    CATEGORICAL = "categorical"
+    DATETIME = "datetime"
+    TEXT = "text"
+    BOOLEAN = "boolean"
+    HIGH_CARDINALITY = "high_cardinality"
+    CONSTANT = "constant"
+    UNIQUE_ID = "unique_id"
+
+
+class MissingnessType(str, Enum):
+    MCAR = "MCAR"    # Missing Completely At Random
+    MAR = "MAR"      # Missing At Random (depends on observed data)
+    MNAR = "MNAR"    # Missing Not At Random (depends on missing value itself)
+    UNKNOWN = "unknown"
+
+
+class TuningMode(str, Enum):
+    QUICK = "quick"    # Default hyperparameters only (no search)
+    TUNED = "tuned"    # RandomizedSearchCV on top-N models
+    CUSTOM = "custom"  # User-provided param grids
+
+
+# ---------------------------------------------------------------------------
+# TypedDicts
+# ---------------------------------------------------------------------------
+
+class ColumnProfile(TypedDict, total=False):
+    """Per-column statistics computed by DataOverview."""
+    name: str
+    dtype: str
+    inferred_type: str          # ColumnType value
+    count: int
+    unique: int
+    missing_count: int
+    missing_pct: float
+    memory_bytes: int
+    # Numeric fields
+    mean: float
+    median: float
+    std: float
+    variance: float
+    min: float
+    max: float
+    range: float
+    iqr: float
+    q1: float
+    q3: float
+    skewness: float
+    kurtosis: float
+    # Categorical fields
+    top_value: Any
+    top_freq: int
+    cardinality_ratio: float
+    value_counts: dict[str, int]
+    # DateTime fields
+    dt_min: str
+    dt_max: str
+    range_days: float
+    inferred_frequency: str | None
+
+
+class DataSchema(TypedDict):
+    """Schema inferred from the dataset."""
+    column_types: dict[str, ColumnType]
+    numeric_cols: list[str]
+    categorical_cols: list[str]
+    datetime_cols: list[str]
+    text_cols: list[str]
+    boolean_cols: list[str]
+    high_cardinality_cols: list[str]
+    constant_cols: list[str]
+    unique_id_cols: list[str]
+    target_col: str | None
+    task: TaskType
+
+
+class LeakageWarning(TypedDict):
+    """A potential data leakage warning for a feature."""
+    column: str
+    score: float
+    method: str          # "high_correlation" | "mutual_information" | "temporal" | "derived"
+    severity: str        # "critical" | "warning" | "info"
+    explanation: str
+
+
+# ---------------------------------------------------------------------------
+# Type aliases
+# ---------------------------------------------------------------------------
+
+DataFrame = pd.DataFrame
+Series = pd.Series
+Array = np.ndarray

dissectml/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"