data-advisor 0.1.0__py3-none-any.whl

data_advisor/scanner.py

@@ -0,0 +1,655 @@
+"""
+scanner.py - Core dataset inspection, profiling, and statistical testing engine.
+
+Designed for robust execution against unknown, messy real-world datasets during
+timed data analysis examinations.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+
+def load_dataset(source: Union[str, Path, pd.DataFrame]) -> pd.DataFrame:
+    """
+    Safely load a dataset from a file path or return an existing DataFrame copy.
+    Supports CSV, TSV, Excel, Parquet, JSON, and Feather formats.
+    """
+    if isinstance(source, pd.DataFrame):
+        return source.copy()
+
+    path = Path(source)
+    if not path.exists():
+        raise FileNotFoundError(f"Dataset path does not exist: {path}")
+
+    ext = path.suffix.lower()
+    if ext in [".csv", ".txt"]:
+        # Try default comma, fallback to tab or semicolon if 1 column detected
+        try:
+            df = pd.read_csv(path)
+            if len(df.columns) == 1:
+                df_tab = pd.read_csv(path, sep="\t")
+                if len(df_tab.columns) > 1:
+                    df = df_tab
+        except Exception:
+            df = pd.read_csv(path, sep=None, engine="python")
+    elif ext == ".tsv":
+        df = pd.read_csv(path, sep="\t")
+    elif ext in [".xlsx", ".xls"]:
+        df = pd.read_excel(path)
+    elif ext == ".parquet":
+        df = pd.read_parquet(path)
+    elif ext == ".json":
+        df = pd.read_json(path)
+    elif ext == ".feather":
+        df = pd.read_feather(path)
+    else:
+        # Fallback to read_csv
+        df = pd.read_csv(path)
+
+    return df
+
+
+class DataScanner:
+    """
+    Exhaustive data profiler and statistical testing engine.
+    Analyzes structure, data quality, distributions, outliers, cardinality,
+    and runs relevant bivariate hypothesis tests.
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        target: Optional[str] = None,
+        id_cols: Optional[List[str]] = None,
+    ) -> None:
+        self.df = df.copy()
+        self.target = target
+        self.id_cols = set(id_cols or [])
+        self._cache: Dict[str, Any] = {}
+
+    # -------------------------------------------------------------------------
+    # 1. Structure & Overview
+    # -------------------------------------------------------------------------
+    def scan_overview(self) -> Dict[str, Any]:
+        """Compute basic dataset dimensions, memory footprint, and row duplicates."""
+        n_rows, n_cols = self.df.shape
+        mem_bytes = self.df.memory_usage(deep=True).sum()
+        mem_mb = round(mem_bytes / (1024 * 1024), 2)
+        dup_count = int(self.df.duplicated().sum())
+        dup_pct = round((dup_count / n_rows * 100) if n_rows > 0 else 0.0, 2)
+
+        dtypes_summary: Dict[str, int] = {}
+        for dt in self.df.dtypes:
+            dt_name = str(dt)
+            dtypes_summary[dt_name] = dtypes_summary.get(dt_name, 0) + 1
+
+        return {
+            "n_rows": n_rows,
+            "n_cols": n_cols,
+            "memory_mb": mem_mb,
+            "duplicate_rows": dup_count,
+            "duplicate_pct": dup_pct,
+            "dtypes_summary": dtypes_summary,
+            "column_names": list(self.df.columns),
+        }
+
+    # -------------------------------------------------------------------------
+    # 2. Missing Values & Data Quality
+    # -------------------------------------------------------------------------
+    def scan_missing_values(self) -> Dict[str, Dict[str, Any]]:
+        """Identify missing value counts, percentages, and imputation strategy candidates."""
+        n_rows = len(self.df)
+        missing_report: Dict[str, Dict[str, Any]] = {}
+
+        for col in self.df.columns:
+            null_count = int(self.df[col].isna().sum())
+            null_pct = round((null_count / n_rows * 100) if n_rows > 0 else 0.0, 2)
+            dtype_str = str(self.df[col].dtype)
+            is_numeric = pd.api.types.is_numeric_dtype(self.df[col])
+
+            missing_report[col] = {
+                "null_count": null_count,
+                "null_pct": null_pct,
+                "dtype": dtype_str,
+                "is_numeric": is_numeric,
+                "all_null": null_count == n_rows,
+            }
+
+        return missing_report
+
+    # -------------------------------------------------------------------------
+    # 3. Data Type Mismatches & Semantic Roles
+    # -------------------------------------------------------------------------
+    def scan_data_types(self) -> Dict[str, Any]:
+        """
+        Flag data type mismatches:
+        - Numeric IDs stored as numbers
+        - Dates stored as strings
+        - Low-cardinality integers that are categoricals
+        - Constant / zero variance columns
+        - High-cardinality text/ID columns
+        """
+        n_rows = len(self.df)
+        potential_dates: List[str] = []
+        potential_ids: List[str] = []
+        low_card_numerics: List[Dict[str, Any]] = []
+        constant_columns: List[str] = []
+        high_card_objects: List[str] = []
+
+        id_name_patterns = ("id", "key", "code", "uuid", "guid", "hash", "ssn", "token")
+
+        for col in self.df.columns:
+            series = self.df[col]
+            non_null = series.dropna()
+            n_non_null = len(non_null)
+            nunique = int(non_null.nunique())
+            col_lower = str(col).lower()
+
+            # Constant column
+            if nunique <= 1:
+                constant_columns.append(col)
+                continue
+
+            # Check potential dates in object/string columns
+            if pd.api.types.is_object_dtype(series) or pd.api.types.is_string_dtype(series):
+                # High cardinality string / hash
+                if n_non_null > 0 and (nunique / n_non_null) > 0.85 and nunique > 30:
+                    high_card_objects.append(col)
+
+                # Heuristic date parsing on sample
+                if any(k in col_lower for k in ("date", "time", "timestamp", "year", "dob", "created", "updated")):
+                    potential_dates.append(col)
+                elif n_non_null > 0:
+                    sample = non_null.head(20).astype(str)
+                    parsed_success = 0
+                    for val in sample:
+                        if len(val) >= 6 and (("-" in val) or ("/" in val) or (":" in val)):
+                            try:
+                                pd.to_datetime(val, errors="raise")
+                                parsed_success += 1
+                            except Exception:
+                                break
+                    if parsed_success >= max(5, int(len(sample) * 0.7)):
+                        potential_dates.append(col)
+
+            # Check potential IDs
+            if col in self.id_cols:
+                potential_ids.append(col)
+            elif any(col_lower.endswith(pat) or col_lower.startswith(pat) or f"_{pat}" in col_lower for pat in id_name_patterns):
+                if nunique > 20 and (n_non_null > 0 and (nunique / n_non_null) > 0.6):
+                    potential_ids.append(col)
+            elif pd.api.types.is_integer_dtype(series) and n_non_null > 50:
+                if nunique == n_non_null:
+                    potential_ids.append(col)
+
+            # Low cardinality numeric acting as categorical
+            if pd.api.types.is_numeric_dtype(series) and col not in potential_ids:
+                if 2 <= nunique <= 6 and nunique < (n_rows * 0.05):
+                    low_card_numerics.append({
+                        "column": col,
+                        "unique_count": nunique,
+                        "unique_values": sorted(non_null.unique().tolist()),
+                    })
+
+        return {
+            "potential_dates": potential_dates,
+            "potential_ids": list(set(potential_ids)),
+            "low_card_numerics": low_card_numerics,
+            "constant_columns": constant_columns,
+            "high_card_objects": high_card_objects,
+        }
+
+    # -------------------------------------------------------------------------
+    # 4. Distributions, Outliers & Skewness
+    # -------------------------------------------------------------------------
+    def scan_distributions(self) -> Dict[str, Dict[str, Any]]:
+        """
+        Calculate descriptive statistics, skewness, kurtosis, and outlier counts
+        using both IQR criteria (1.5x and 3x) and Z-score criteria (|z| > 3).
+        """
+        num_cols = self.df.select_dtypes(include=[np.number]).columns
+        dist_report: Dict[str, Dict[str, Any]] = {}
+
+        for col in num_cols:
+            series = self.df[col].dropna()
+            n_count = len(series)
+
+            if n_count < 3 or series.nunique() <= 1:
+                # Insufficient variance to compute skewness/outliers
+                dist_report[col] = {
+                    "count": n_count,
+                    "mean": float(series.mean()) if n_count > 0 else 0.0,
+                    "std": 0.0,
+                    "median": float(series.median()) if n_count > 0 else 0.0,
+                    "min": float(series.min()) if n_count > 0 else 0.0,
+                    "max": float(series.max()) if n_count > 0 else 0.0,
+                    "skewness": 0.0,
+                    "kurtosis": 0.0,
+                    "is_non_negative": bool(series.min() >= 0) if n_count > 0 else True,
+                    "iqr_outliers": 0,
+                    "iqr_outlier_pct": 0.0,
+                    "extreme_outliers": 0,
+                    "z_outliers": 0,
+                    "z_outlier_pct": 0.0,
+                    "lower_fence": float(series.min()) if n_count > 0 else 0.0,
+                    "upper_fence": float(series.max()) if n_count > 0 else 0.0,
+                }
+                continue
+
+            mean_val = float(series.mean())
+            std_val = float(series.std(ddof=1))
+            median_val = float(series.median())
+            min_val = float(series.min())
+            max_val = float(series.max())
+
+            # Skewness & Kurtosis
+            skew_val = float(stats.skew(series, bias=False))
+            kurt_val = float(stats.kurtosis(series, bias=False))
+
+            # IQR Outliers
+            q1 = float(np.percentile(series, 25))
+            q3 = float(np.percentile(series, 75))
+            iqr = q3 - q1
+
+            lower_fence = q1 - 1.5 * iqr
+            upper_fence = q3 + 1.5 * iqr
+            extreme_lower = q1 - 3.0 * iqr
+            extreme_upper = q3 + 3.0 * iqr
+
+            iqr_outliers = int(((series < lower_fence) | (series > upper_fence)).sum())
+            iqr_outlier_pct = round((iqr_outliers / n_count) * 100, 2)
+
+            extreme_outliers = int(((series < extreme_lower) | (series > extreme_upper)).sum())
+
+            # Z-score Outliers
+            if std_val > 0:
+                z_scores = np.abs((series - mean_val) / std_val)
+                z_outliers = int((z_scores > 3.0).sum())
+                z_outlier_pct = round((z_outliers / n_count) * 100, 2)
+            else:
+                z_outliers = 0
+                z_outlier_pct = 0.0
+
+            dist_report[col] = {
+                "count": n_count,
+                "mean": round(mean_val, 4),
+                "std": round(std_val, 4),
+                "median": round(median_val, 4),
+                "min": round(min_val, 4),
+                "max": round(max_val, 4),
+                "skewness": round(skew_val, 3),
+                "kurtosis": round(kurt_val, 3),
+                "is_non_negative": bool(min_val >= 0),
+                "q1": round(q1, 4),
+                "q3": round(q3, 4),
+                "iqr": round(iqr, 4),
+                "lower_fence": round(lower_fence, 4),
+                "upper_fence": round(upper_fence, 4),
+                "iqr_outliers": iqr_outliers,
+                "iqr_outlier_pct": iqr_outlier_pct,
+                "extreme_outliers": extreme_outliers,
+                "z_outliers": z_outliers,
+                "z_outlier_pct": z_outlier_pct,
+            }
+
+        return dist_report
+
+    # -------------------------------------------------------------------------
+    # 5. Categorical Processing & Cardinality
+    # -------------------------------------------------------------------------
+    def scan_categoricals(self) -> Dict[str, Dict[str, Any]]:
+        """
+        Analyze cardinality and distribution for categorical, boolean, and
+        object features.
+        """
+        cat_cols = self.df.select_dtypes(include=["object", "string", "category", "bool"]).columns.tolist()
+        cat_report: Dict[str, Dict[str, Any]] = {}
+        n_rows = len(self.df)
+
+        for col in cat_cols:
+            series = self.df[col]
+            non_null = series.dropna()
+            n_non_null = len(non_null)
+            nunique = int(non_null.nunique())
+
+            # Top value and frequency
+            vc = non_null.value_counts()
+            top_val = str(vc.index[0]) if not vc.empty else None
+            top_pct = round((float(vc.iloc[0]) / n_non_null * 100) if n_non_null > 0 else 0.0, 2)
+
+            # Sample values up to 8
+            sample_values = [str(x) for x in vc.index[:8]]
+
+            cat_report[col] = {
+                "cardinality": nunique,
+                "cardinality_pct": round((nunique / n_rows * 100) if n_rows > 0 else 0.0, 2),
+                "top_value": top_val,
+                "top_value_pct": top_pct,
+                "sample_values": sample_values,
+                "is_binary": nunique == 2,
+                "is_low_cardinality": 2 <= nunique <= 10,
+                "is_high_cardinality": nunique > 10,
+            }
+
+        return cat_report
+
+    # -------------------------------------------------------------------------
+    # 6. Target Variable Analysis
+    # -------------------------------------------------------------------------
+    def scan_target(self) -> Optional[Dict[str, Any]]:
+        """
+        Inspect the target variable to determine problem type (Regression vs
+        Classification) and class balance.
+        """
+        target_col = self.target
+        if not target_col:
+            # Auto-detect target candidates
+            candidates = ["target", "label", "class", "churn", "price", "salary",
+                          "default", "income", "outcome", "survived", "status", "y"]
+            for cand in candidates:
+                for col in self.df.columns:
+                    if col.lower() == cand:
+                        target_col = col
+                        break
+                if target_col:
+                    break
+
+        if not target_col or target_col not in self.df.columns:
+            return None
+
+        series = self.df[target_col].dropna()
+        n_total = len(series)
+        nunique = int(series.nunique())
+        is_numeric = pd.api.types.is_numeric_dtype(series)
+
+        if is_numeric and nunique > 15:
+            # Regression problem
+            return {
+                "target_column": target_col,
+                "problem_type": "regression",
+                "is_continuous": True,
+                "n_samples": n_total,
+                "mean": round(float(series.mean()), 4),
+                "std": round(float(series.std()), 4),
+                "skewness": round(float(stats.skew(series)), 3),
+                "min": round(float(series.min()), 4),
+                "max": round(float(series.max()), 4),
+            }
+        else:
+            # Classification problem
+            vc = series.value_counts()
+            class_distribution: Dict[str, Dict[str, Any]] = {}
+            for cls_val, cnt in vc.items():
+                pct = round(float(cnt) / n_total * 100, 2)
+                class_distribution[str(cls_val)] = {"count": int(cnt), "percentage": pct}
+
+            min_cnt = vc.min()
+            max_cnt = vc.max()
+            minority_pct = round((min_cnt / n_total) * 100, 2)
+            imbalance_ratio = round(max_cnt / min_cnt, 2) if min_cnt > 0 else float("inf")
+            is_imbalanced = bool(minority_pct < 20.0 or imbalance_ratio >= 3.0)
+            classification_type = "binary" if nunique == 2 else "multiclass"
+
+            return {
+                "target_column": target_col,
+                "problem_type": "classification",
+                "classification_type": classification_type,
+                "n_classes": nunique,
+                "class_distribution": class_distribution,
+                "minority_percentage": minority_pct,
+                "imbalance_ratio": imbalance_ratio,
+                "is_imbalanced": is_imbalanced,
+            }
+
+    # -------------------------------------------------------------------------
+    # 7. Hypothesis Testing Engine
+    # -------------------------------------------------------------------------
+    def run_hypothesis_tests(self) -> List[Dict[str, Any]]:
+        """
+        Run relevant statistical hypothesis tests:
+        - 2 numeric: Pearson & Spearman correlation
+        - 2 categorical: Chi-Square Test of Independence
+        - 1 numeric + 1 binary categorical: Two-sample t-test / Mann-Whitney U test
+        - 1 numeric + 1 multi-class categorical: One-way ANOVA / Kruskal-Wallis test
+
+        Prioritizes Feature vs Target relationships if target is provided.
+        """
+        tests: List[Dict[str, Any]] = []
+        target_info = self.scan_target()
+        target_col = target_info["target_column"] if target_info else None
+        # Filter out metadata/date/id/constant columns from hypothesis tests
+        dt_info = self.scan_data_types()
+        excluded_test_cols = set(
+            list(self.id_cols)
+            + list(dt_info.get("potential_ids", []))
+            + list(dt_info.get("potential_dates", []))
+            + list(dt_info.get("constant_columns", []))
+            + list(dt_info.get("high_card_objects", []))
+        )
+        num_cols = [c for c in self.df.select_dtypes(include=[np.number]).columns if c not in excluded_test_cols]
+        cat_cols = [
+            c for c in self.df.select_dtypes(include=["object", "string", "category", "bool"]).columns
+            if c not in excluded_test_cols and 2 <= self.df[c].nunique() <= 20
+        ]
+
+        if target_col:
+            # Target-centric testing: test each feature against target
+            is_target_numeric = target_info.get("problem_type") == "regression"
+            t_series = self.df[target_col]
+
+            # 1. Target vs Numeric features
+            for col in num_cols:
+                if col == target_col:
+                    continue
+                pair_df = self.df[[col, target_col]].dropna()
+                if len(pair_df) < 5 or pair_df[col].nunique() <= 1:
+                    continue
+
+                if is_target_numeric:
+                    # 2 numeric columns: Pearson & Spearman
+                    r_val, p_val = stats.pearsonr(pair_df[col], pair_df[target_col])
+                    rho_val, rho_pval = stats.spearmanr(pair_df[col], pair_df[target_col])
+                    tests.append({
+                        "feature_1": col,
+                        "feature_2": target_col,
+                        "test_type": "Pearson & Spearman Correlation",
+                        "h0": f"There is no linear/monotonic correlation between {col} and {target_col}.",
+                        "h1": f"There is a statistically significant correlation between {col} and {target_col}.",
+                        "statistic": round(float(r_val), 4),
+                        "p_value": float(p_val),
+                        "secondary_statistic": round(float(rho_val), 4),
+                        "secondary_p_value": float(rho_pval),
+                        "is_significant": bool(p_val < 0.05 or rho_pval < 0.05),
+                        "conclusion": (
+                            f"Significant association (r={r_val:.3f}, p={p_val:.2e})"
+                            if p_val < 0.05 else f"No significant linear association (p={p_val:.3f})"
+                        ),
+                    })
+                else:
+                    # Numeric feature vs Categorical target
+                    classes = pair_df[target_col].unique()
+                    if len(classes) == 2:
+                        g1 = pair_df[pair_df[target_col] == classes[0]][col]
+                        g2 = pair_df[pair_df[target_col] == classes[1]][col]
+                        if len(g1) >= 2 and len(g2) >= 2:
+                            t_stat, t_pval = stats.ttest_ind(g1, g2, equal_var=False)
+                            u_stat, u_pval = stats.mannwhitneyu(g1, g2, alternative="two-sided")
+                            tests.append({
+                                "feature_1": col,
+                                "feature_2": target_col,
+                                "test_type": "Two-sample Welch's t-test / Mann-Whitney U",
+                                "h0": f"Mean/distribution of {col} is equal across {target_col} classes ({classes[0]} vs {classes[1]}).",
+                                "h1": f"Mean/distribution of {col} differs significantly across {target_col} classes.",
+                                "statistic": round(float(t_stat), 4),
+                                "p_value": float(t_pval),
+                                "secondary_statistic": round(float(u_stat), 4),
+                                "secondary_p_value": float(u_pval),
+                                "is_significant": bool(t_pval < 0.05 or u_pval < 0.05),
+                                "conclusion": (
+                                    f"Distributions differ significantly across target classes (t={t_stat:.3f}, p={t_pval:.2e})"
+                                    if t_pval < 0.05 else f"No significant difference across target classes (p={t_pval:.3f})"
+                                ),
+                            })
+                    elif len(classes) > 2:
+                        groups = [pair_df[pair_df[target_col] == c][col] for c in classes if len(pair_df[pair_df[target_col] == c]) >= 2]
+                        if len(groups) >= 2:
+                            f_stat, f_pval = stats.f_oneway(*groups)
+                            kw_stat, kw_pval = stats.kruskal(*groups)
+                            tests.append({
+                                "feature_1": col,
+                                "feature_2": target_col,
+                                "test_type": "One-Way ANOVA / Kruskal-Wallis",
+                                "h0": f"Mean of {col} is identical across all levels of {target_col}.",
+                                "h1": f"At least one {target_col} group mean of {col} is different.",
+                                "statistic": round(float(f_stat), 4),
+                                "p_value": float(f_pval),
+                                "secondary_statistic": round(float(kw_stat), 4),
+                                "secondary_p_value": float(kw_pval),
+                                "is_significant": bool(f_pval < 0.05 or kw_pval < 0.05),
+                                "conclusion": (
+                                    f"Significant difference between group means (F={f_stat:.3f}, p={f_pval:.2e})"
+                                    if f_pval < 0.05 else f"No significant difference across groups (p={f_pval:.3f})"
+                                ),
+                            })
+
+            # 2. Target vs Categorical features
+            for col in cat_cols:
+                if col == target_col:
+                    continue
+                pair_df = self.df[[col, target_col]].dropna()
+                if len(pair_df) < 10 or pair_df[col].nunique() <= 1:
+                    continue
+
+                if not is_target_numeric:
+                    # 2 categorical columns: Chi-square test of independence
+                    cont_tab = pd.crosstab(pair_df[col], pair_df[target_col])
+                    if cont_tab.shape[0] > 1 and cont_tab.shape[1] > 1:
+                        chi2, p_val, dof, _ = stats.chi2_contingency(cont_tab)
+                        # Cramer's V
+                        n_obs = cont_tab.sum().sum()
+                        min_dim = min(cont_tab.shape) - 1
+                        cramer_v = float(np.sqrt(chi2 / (n_obs * min_dim))) if (n_obs * min_dim) > 0 else 0.0
+
+                        tests.append({
+                            "feature_1": col,
+                            "feature_2": target_col,
+                            "test_type": "Chi-Square Test of Independence",
+                            "h0": f"{col} and {target_col} are independent.",
+                            "h1": f"{col} and {target_col} are significantly dependent/associated.",
+                            "statistic": round(float(chi2), 4),
+                            "p_value": float(p_val),
+                            "secondary_statistic": round(cramer_v, 4),
+                            "secondary_name": "Cramer's V",
+                            "is_significant": bool(p_val < 0.05),
+                            "conclusion": (
+                                f"Significant association (Chi2={chi2:.2f}, p={p_val:.2e}, Cramer's V={cramer_v:.3f})"
+                                if p_val < 0.05 else f"Independent/no significant association (p={p_val:.3f})"
+                            ),
+                        })
+                else:
+                    # Categorical feature vs Numeric target (ANOVA / t-test)
+                    classes = pair_df[col].unique()
+                    if len(classes) == 2:
+                        g1 = pair_df[pair_df[col] == classes[0]][target_col]
+                        g2 = pair_df[pair_df[col] == classes[1]][target_col]
+                        if len(g1) >= 2 and len(g2) >= 2:
+                            t_stat, t_pval = stats.ttest_ind(g1, g2, equal_var=False)
+                            tests.append({
+                                "feature_1": col,
+                                "feature_2": target_col,
+                                "test_type": "Two-sample Welch's t-test",
+                                "h0": f"Mean target {target_col} is equal across {col} categories.",
+                                "h1": f"Mean target {target_col} differs between {col} categories.",
+                                "statistic": round(float(t_stat), 4),
+                                "p_value": float(t_pval),
+                                "is_significant": bool(t_pval < 0.05),
+                                "conclusion": (
+                                    f"Significant difference in {target_col} (t={t_stat:.3f}, p={t_pval:.2e})"
+                                    if t_pval < 0.05 else f"No significant difference in {target_col} (p={t_pval:.3f})"
+                                ),
+                            })
+                    elif 2 < len(classes) <= 10:
+                        groups = [pair_df[pair_df[col] == c][target_col] for c in classes if len(pair_df[pair_df[col] == c]) >= 2]
+                        if len(groups) >= 2:
+                            f_stat, f_pval = stats.f_oneway(*groups)
+                            tests.append({
+                                "feature_1": col,
+                                "feature_2": target_col,
+                                "test_type": "One-Way ANOVA",
+                                "h0": f"Mean {target_col} is equal across all {col} groups.",
+                                "h1": f"Mean {target_col} differs across {col} groups.",
+                                "statistic": round(float(f_stat), 4),
+                                "p_value": float(f_pval),
+                                "is_significant": bool(f_pval < 0.05),
+                                "conclusion": (
+                                    f"Significant difference across {col} groups (F={f_stat:.3f}, p={f_pval:.2e})"
+                                    if f_pval < 0.05 else f"No significant difference (p={f_pval:.3f})"
+                                ),
+                            })
+
+        else:
+            # No target specified: test top correlated numeric pairs
+            if len(num_cols) >= 2:
+                corr_matrix = self.df[num_cols].corr()
+                checked_pairs = set()
+                # Find top correlated pairs
+                pairs: List[Tuple[str, str, float]] = []
+                for i in range(len(num_cols)):
+                    for j in range(i + 1, len(num_cols)):
+                        c1, c2 = num_cols[i], num_cols[j]
+                        val = corr_matrix.loc[c1, c2]
+                        if not np.isnan(val):
+                            pairs.append((c1, c2, abs(val)))
+
+                pairs.sort(key=lambda x: x[2], reverse=True)
+                # Take top 5 pairs
+                for c1, c2, _ in pairs[:5]:
+                    pair_df = self.df[[c1, c2]].dropna()
+                    if len(pair_df) >= 5:
+                        r_val, p_val = stats.pearsonr(pair_df[c1], pair_df[c2])
+                        tests.append({
+                            "feature_1": c1,
+                            "feature_2": c2,
+                            "test_type": "Pearson Correlation",
+                            "h0": f"No linear correlation between {c1} and {c2}.",
+                            "h1": f"Significant linear correlation between {c1} and {c2}.",
+                            "statistic": round(float(r_val), 4),
+                            "p_value": float(p_val),
+                            "is_significant": bool(p_val < 0.05),
+                            "conclusion": (
+                                f"Significant correlation (r={r_val:.3f}, p={p_val:.2e})"
+                                if p_val < 0.05 else f"No significant correlation (p={p_val:.3f})"
+                            ),
+                        })
+
+        return tests
+
+    # -------------------------------------------------------------------------
+    # 8. Complete Scan Aggregation
+    # -------------------------------------------------------------------------
+    def scan(self) -> Dict[str, Any]:
+        """Execute all scanners and return a unified diagnostic dictionary."""
+        overview = self.scan_overview()
+        missing = self.scan_missing_values()
+        dtypes = self.scan_data_types()
+        distributions = self.scan_distributions()
+        categoricals = self.scan_categoricals()
+        target_info = self.scan_target()
+        hypothesis_tests = self.run_hypothesis_tests()
+
+        return {
+            "overview": overview,
+            "missing_values": missing,
+            "data_types": dtypes,
+            "distributions": distributions,
+            "categoricals": categoricals,
+            "target": target_info,
+            "hypothesis_tests": hypothesis_tests,
+        }