hashprep 0.1.0a0__tar.gz

hashprep-0.1.0a0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 HashPrep
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hashprep-0.1.0a0/MANIFEST.in

@@ -0,0 +1,13 @@
+include README.md LICENSE
+recursive-include hashprep *.py
+
+prune datasets
+prune examples
+prune tests
+prune docs
+prune web
+prune scripts
+exclude *.pyc
+exclude __pycache__/*
+exclude test.json
+exclude todo.md

hashprep-0.1.0a0/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: hashprep
+Version: 0.1.0a0
+Summary: A library for dataset quality checks, preprocessing, and report generation
+Author-email: "Aftaab Siddiqui (MaskedSyntax)" <aftaab@aftaab.xyz>
+License: MIT License
+        
+        Copyright (c) 2025 HashPrep
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/cachevector/hashprep
+Project-URL: Repository, https://github.com/cachevector/hashprep
+Project-URL: Documentation, https://github.com/cachevector/hashprep
+Project-URL: Issues, https://github.com/cachevector/hashprep/issues
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.3.0
+Requires-Dist: fastapi>=0.116.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: numpy>=2.2.6
+Requires-Dist: pandas>=2.3.2
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: scikit-learn>=1.7.2
+Requires-Dist: scipy>=1.15.3
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: weasyprint>=66.0
+Dynamic: license-file
+
+<div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/hashprep-wobg.svg" width="100">
+    <img alt="HashPrep Logo" src="docs/assets/hashprep-dark.svg" width="100">
+  </picture>
+
+  <h1>HashPrep</h1>
+  <p>
+    <b> Dataset Profiler & Debugger for Machine Learning </b>
+  </p>
+
+  <p align="center">
+    <!-- Distribution -->
+    <!-- <img src="https://img.shields.io/pypi/v/hashprep?color=blue&label=PyPI" /> -->
+    <img src="https://img.shields.io/badge/PyPI-Coming%20Soon-blue" />
+    <!-- License -->
+    <img src="https://img.shields.io/badge/License-MIT-green" />
+    <img src="https://img.shields.io/badge/CLI-Supported-orange" />
+  </p>
+  <p>
+    <!-- Features -->
+    <img src="https://img.shields.io/badge/Feature-Dataset%20Quality%20Assurance-critical" />
+    <img src="https://img.shields.io/badge/Feature-Preprocessing%20%2B%20Profiling-blueviolet" />
+    <img src="https://img.shields.io/badge/Feature-Report%20Generation-3f4f75" />
+    <img src="https://img.shields.io/badge/Feature-Quick%20Fixes-success" />
+  </p>
+</div>
+
+> [!WARNING]  
+> This repository is under active development and may not be stable.
+
+## Overview
+
+**HashPrep** is a Python library for intelligent dataset profiling and debugging that acts as a comprehensive pre-training quality assurance tool for machine learning projects.
+Think of it as **"Pandas Profiling + PyLint for datasets"**, designed specifically for machine learning workflows.
+
+It catches critical dataset issues before they derail your ML pipeline, explains the problems, and suggests context-aware fixes.  
+If you want, HashPrep can even apply those fixes for you automatically.
+
+
+---
+
+## Features
+
+Key features include:
+
+- **Intelligent Profiling**: Detect missing values, skewed distributions, outliers, and data type inconsistencies.
+- **ML-Specific Checks**: Identify data leakage, dataset drift, class imbalance, and high-cardinality features.
+- **Automated Preparation**: Get suggestions for encoding, imputation, scaling, and transformations, and optionally apply them automatically.
+- **Rich Reporting**: Generate statistical summaries and exportable reports for collaboration.
+- **Production-Ready Pipelines**: Output reproducible cleaning and preprocessing code that integrates seamlessly with ML workflows.
+
+HashPrep turns dataset debugging into a guided, automated process - saving time, improving model reliability, and standardizing best practices across teams.
+
+---
+
+## License
+
+This project is licensed under the [**MIT License**](./LICENSE).
+
+---
+
+## Contributing
+
+We welcome contributions from the community to make HashPrep better!
+
+Before you get started, please:
+
+- Review our [CONTRIBUTING.md](./CONTRIBUTING.md) for detailed guidelines and setup instructions
+- Write clean, well-documented code
+- Follow best practices for the stack or component you’re working on
+- Open a pull request (PR) with a clear description of your changes and motivation

hashprep-0.1.0a0/README.md

@@ -0,0 +1,72 @@
+<div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/hashprep-wobg.svg" width="100">
+    <img alt="HashPrep Logo" src="docs/assets/hashprep-dark.svg" width="100">
+  </picture>
+
+  <h1>HashPrep</h1>
+  <p>
+    <b> Dataset Profiler & Debugger for Machine Learning </b>
+  </p>
+
+  <p align="center">
+    <!-- Distribution -->
+    <!-- <img src="https://img.shields.io/pypi/v/hashprep?color=blue&label=PyPI" /> -->
+    <img src="https://img.shields.io/badge/PyPI-Coming%20Soon-blue" />
+    <!-- License -->
+    <img src="https://img.shields.io/badge/License-MIT-green" />
+    <img src="https://img.shields.io/badge/CLI-Supported-orange" />
+  </p>
+  <p>
+    <!-- Features -->
+    <img src="https://img.shields.io/badge/Feature-Dataset%20Quality%20Assurance-critical" />
+    <img src="https://img.shields.io/badge/Feature-Preprocessing%20%2B%20Profiling-blueviolet" />
+    <img src="https://img.shields.io/badge/Feature-Report%20Generation-3f4f75" />
+    <img src="https://img.shields.io/badge/Feature-Quick%20Fixes-success" />
+  </p>
+</div>
+
+> [!WARNING]  
+> This repository is under active development and may not be stable.
+
+## Overview
+
+**HashPrep** is a Python library for intelligent dataset profiling and debugging that acts as a comprehensive pre-training quality assurance tool for machine learning projects.
+Think of it as **"Pandas Profiling + PyLint for datasets"**, designed specifically for machine learning workflows.
+
+It catches critical dataset issues before they derail your ML pipeline, explains the problems, and suggests context-aware fixes.  
+If you want, HashPrep can even apply those fixes for you automatically.
+
+
+---
+
+## Features
+
+Key features include:
+
+- **Intelligent Profiling**: Detect missing values, skewed distributions, outliers, and data type inconsistencies.
+- **ML-Specific Checks**: Identify data leakage, dataset drift, class imbalance, and high-cardinality features.
+- **Automated Preparation**: Get suggestions for encoding, imputation, scaling, and transformations, and optionally apply them automatically.
+- **Rich Reporting**: Generate statistical summaries and exportable reports for collaboration.
+- **Production-Ready Pipelines**: Output reproducible cleaning and preprocessing code that integrates seamlessly with ML workflows.
+
+HashPrep turns dataset debugging into a guided, automated process - saving time, improving model reliability, and standardizing best practices across teams.
+
+---
+
+## License
+
+This project is licensed under the [**MIT License**](./LICENSE).
+
+---
+
+## Contributing
+
+We welcome contributions from the community to make HashPrep better!
+
+Before you get started, please:
+
+- Review our [CONTRIBUTING.md](./CONTRIBUTING.md) for detailed guidelines and setup instructions
+- Write clean, well-documented code
+- Follow best practices for the stack or component you’re working on
+- Open a pull request (PR) with a clear description of your changes and motivation

hashprep-0.1.0a0/hashprep/__init__.py

@@ -0,0 +1,3 @@
+from .analyzer import DatasetAnalyzer
+
+__version__ = "0.1.0-alpha"

hashprep-0.1.0a0/hashprep/analyzer.py

@@ -0,0 +1,70 @@
+from typing import Dict, List, Optional
+import pandas as pd
+
+from .checks import run_checks
+from .summaries import (
+    get_dataset_preview,
+    summarize_dataset_info,
+    summarize_variable_types,
+    add_reproduction_info,
+    summarize_variables,
+    summarize_interactions,
+    summarize_missing_values,
+)
+
+class DatasetAnalyzer:
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        target_col: Optional[str] = None,
+        selected_checks: Optional[List[str]] = None,
+    ):
+        self.df = df
+        self.target_col = target_col
+        self.selected_checks = selected_checks
+        self.issues = []
+        self.summaries = {}
+        self.all_checks = [
+            "data_leakage", "high_missing_values", "empty_columns", "single_value_columns",
+            "target_leakage_patterns", "class_imbalance", "high_cardinality", "duplicates",
+            "mixed_data_types", "outliers", "feature_correlation", "categorical_correlation",
+            "mixed_correlation", "dataset_missingness", "high_zero_counts",
+            "extreme_text_lengths", "datetime_skew", "missing_patterns",
+        ]
+
+    def analyze(self) -> Dict:
+        """Run all summaries and checks, return summary"""
+        self.summaries.update(get_dataset_preview(self.df))
+        self.summaries.update(summarize_dataset_info(self.df))
+        self.summaries["variable_types"] = summarize_variable_types(self.df)
+        self.summaries["reproduction_info"] = add_reproduction_info(self.df)
+        self.summaries["variables"] = summarize_variables(self.df)
+        self.summaries.update(summarize_interactions(self.df))
+        self.summaries.update(summarize_missing_values(self.df))
+
+        checks_to_run = self.all_checks if self.selected_checks is None else [
+            check for check in self.selected_checks if check in self.all_checks
+        ]
+        self.issues = run_checks(self, checks_to_run)
+
+        return self._generate_summary()
+
+    def _generate_summary(self):
+        critical_issues = [i for i in self.issues if i.severity == "critical"]
+        warning_issues = [i for i in self.issues if i.severity == "warning"]
+        return {
+            "critical_count": len(critical_issues),
+            "warning_count": len(warning_issues),
+            "total_issues": len(self.issues),
+            "issues": [
+                {
+                    "category": issue.category,
+                    "severity": issue.severity,
+                    "column": issue.column,
+                    "description": issue.description,
+                    "impact_score": issue.impact_score,
+                    "quick_fix": issue.quick_fix,
+                } for issue in self.issues
+            ],
+            "summaries": self.summaries,
+        }

hashprep-0.1.0a0/hashprep/checks/__init__.py

@@ -0,0 +1,34 @@
+from .core import Issues
+from .leakage import _check_data_leakage, _check_target_leakage_patterns
+from .missing_values import _check_high_missing_values, _check_empty_columns, _check_dataset_missingness, _check_missing_patterns
+from .columns import _check_single_value_columns, _check_high_cardinality, _check_duplicates, _check_mixed_data_types
+from .outliers import _check_outliers, _check_high_zero_counts, _check_extreme_text_lengths, _check_datetime_skew
+from .correlations import _check_feature_correlation, _check_categorical_correlation, _check_mixed_correlation
+from .imbalance import _check_class_imbalance
+
+CHECKS = {
+    "data_leakage": _check_data_leakage,
+    "high_missing_values": _check_high_missing_values,
+    "empty_columns": _check_empty_columns,
+    "single_value_columns": _check_single_value_columns,
+    "target_leakage_patterns": _check_target_leakage_patterns,
+    "class_imbalance": _check_class_imbalance,
+    "high_cardinality": _check_high_cardinality,
+    "duplicates": _check_duplicates,
+    "mixed_data_types": _check_mixed_data_types,
+    "outliers": _check_outliers,
+    "feature_correlation": _check_feature_correlation,
+    "categorical_correlation": _check_categorical_correlation,
+    "mixed_correlation": _check_mixed_correlation,
+    "dataset_missingness": _check_dataset_missingness,
+    "high_zero_counts": _check_high_zero_counts,
+    "extreme_text_lengths": _check_extreme_text_lengths,
+    "datetime_skew": _check_datetime_skew,
+    "missing_patterns": _check_missing_patterns,
+}
+
+def run_checks(analyzer, checks_to_run):
+    issues = []
+    for check in checks_to_run:
+        issues.extend(CHECKS[check](analyzer))
+    return issues

hashprep-0.1.0a0/hashprep/checks/columns.py

@@ -0,0 +1,91 @@
+from .core import Issues
+
+def _check_single_value_columns(analyzer):
+    issues = []
+    for col in analyzer.df.columns:
+        if analyzer.df[col].nunique(dropna=True) == 1:
+            impact = "low" if col != analyzer.target_col else "high"
+            severity = "warning" if col != analyzer.target_col else "critical"
+            quick_fix = (
+                "Options: \n- Drop column: Not informative for modeling (Pros: Simplifies model; Cons: None).\n- Verify data: Ensure single value isn't an error (Pros: Validates data; Cons: Time-consuming)."
+                if col != analyzer.target_col
+                else "Options: \n- Redefine target: Replace with a more variable target (Pros: Enables modeling; Cons: Requires new data).\n- Stop analysis: Constant target prevents meaningful prediction (Pros: Avoids invalid model; Cons: Halts analysis)."
+            )
+            issues.append(
+                Issues(
+                    category="single_value",
+                    severity=severity,
+                    column=col,
+                    description=f"Column '{col}' contains only one unique value",
+                    impact_score=impact,
+                    quick_fix=quick_fix,
+                )
+            )
+    return issues
+
+def _check_high_cardinality(analyzer, threshold: int = 100, critical_threshold: float = 0.9):
+    issues = []
+    categorical_cols = analyzer.df.select_dtypes(include="object").columns.tolist()
+    for col in categorical_cols:
+        unique_count = int(analyzer.df[col].nunique())
+        unique_ratio = float(unique_count / len(analyzer.df))
+        if unique_count > threshold:
+            severity = "critical" if unique_ratio > critical_threshold else "warning"
+            impact = "high" if severity == "critical" else "medium"
+            quick_fix = (
+                "Options: \n- Drop column: Avoids overfitting from unique identifiers (Pros: Simplifies model; Cons: Loses potential info).\n- Engineer feature: Extract patterns (e.g., titles from names) (Pros: Retains useful info; Cons: Requires domain knowledge).\n- Use hashing: Reduce dimensionality (Pros: Scalable; Cons: May lose interpretability)."
+                if severity == "critical"
+                else "Options: \n- Group rare categories: Reduce cardinality (Pros: Simplifies feature; Cons: May lose nuance).\n- Use feature hashing: Map to lower dimensions (Pros: Scalable; Cons: Less interpretable).\n- Retain and test: Evaluate feature importance (Pros: Data-driven; Cons: Risk of overfitting)."
+            )
+            issues.append(
+                Issues(
+                    category="high_cardinality",
+                    severity=severity,
+                    column=col,
+                    description=f"Column '{col}' has {unique_count} unique values ({unique_ratio:.1%} of rows)",
+                    impact_score=impact,
+                    quick_fix=quick_fix,
+                )
+            )
+    return issues
+
+def _check_duplicates(analyzer):
+    issues = []
+    duplicate_rows = int(analyzer.df.duplicated().sum())
+    if duplicate_rows > 0:
+        duplicate_ratio = float(duplicate_rows / len(analyzer.df))
+        severity = "critical" if duplicate_ratio > 0.1 else "warning"
+        impact = "high" if severity == "critical" else "medium"
+        quick_fix = (
+            "Options: \n- Drop duplicates: Ensures data integrity (Pros: Cleaner data; Cons: May lose valid repeats).\n- Verify duplicates: Check if intentional (e.g., time-series) (Pros: Validates data; Cons: Time-consuming)."
+            if severity == "critical"
+            else "Options: \n- Drop duplicates: Simplifies dataset (Pros: Cleaner data; Cons: May lose valid repeats).\n- Keep duplicates: If meaningful (e.g., repeated events) (Pros: Retains info; Cons: May bias model).\n- Test impact: Evaluate model performance with/without duplicates (Pros: Data-driven; Cons: Requires computation)."
+        )
+        issues.append(
+            Issues(
+                category="duplicates",
+                severity=severity,
+                column="__all__",
+                description=f"Dataset contains {duplicate_rows} duplicate rows ({duplicate_ratio:.1%} of rows)",
+                impact_score=impact,
+                quick_fix=quick_fix,
+            )
+        )
+    return issues
+
+def _check_mixed_data_types(analyzer):
+    issues = []
+    for col in analyzer.df.columns:
+        types = analyzer.df[col].dropna().map(type).nunique()
+        if types > 1:
+            issues.append(
+                Issues(
+                    category="mixed_types",
+                    severity="warning",
+                    column=col,
+                    description=f"Column '{col}' contains mixed data types",
+                    impact_score="low",
+                    quick_fix="Options: \n- Cast to single type: Ensure consistency (Pros: Simplifies processing; Cons: May lose nuance).\n- Split column: Separate types into new features (Pros: Preserves info; Cons: Adds complexity).\n- Investigate source: Check data collection errors (Pros: Improves quality; Cons: Time-consuming).",
+                )
+            )
+    return issues

hashprep-0.1.0a0/hashprep/checks/core.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass
+class Issues:
+    category: str
+    severity: str  # critical or warning
+    column: str
+    description: str
+    impact_score: str  # high, medium, low
+    quick_fix: str

hashprep-0.1.0a0/hashprep/checks/correlations.py

@@ -0,0 +1,114 @@
+from .core import Issues
+import pandas as pd
+from scipy.stats import chi2_contingency, f_oneway
+import numpy as np
+
+def _check_feature_correlation(analyzer, threshold: float = 0.95, critical_threshold: float = 0.98):
+    issues = []
+    numeric_df = analyzer.df.select_dtypes(include="number")
+    if numeric_df.empty:
+        return issues
+    corr_matrix = numeric_df.corr().abs()
+    upper = corr_matrix.where(np.tril(np.ones(corr_matrix.shape)).astype(bool))
+    correlated_pairs = [
+        (col, row, float(val))
+        for row in upper.index
+        for col, val in upper[row].dropna().items()
+        if val > threshold and col != row
+    ]
+    for col1, col2, corr in correlated_pairs:
+        severity = "critical" if corr > critical_threshold else "warning"
+        impact = "high" if severity == "critical" else "medium"
+        quick_fix = (
+            "Options: \n- Drop one feature: Reduces multicollinearity (Pros: Simplifies model; Cons: Loses info).\n- Combine features: Create composite feature (e.g., PCA) (Pros: Retains info; Cons: Less interpretable).\n- Retain and test: Use robust models (e.g., trees) (Pros: Keeps info; Cons: May affect sensitive models)."
+            if severity == "critical"
+            else "Options: \n- Drop one feature: If less predictive (Pros: Simplifies model; Cons: Loses info).\n- Retain and test: Evaluate with robust models (Pros: Keeps info; Cons: Risk of multicollinearity).\n- Engineer feature: Combine or transform features (Pros: Reduces redundancy; Cons: Adds complexity)."
+        )
+        issues.append(
+            Issues(
+                category="feature_correlation",
+                severity=severity,
+                column=f"{col1},{col2}",
+                description=f"Columns '{col1}' and '{col2}' are highly correlated ({corr:.2f})",
+                impact_score=impact,
+                quick_fix=quick_fix,
+            )
+        )
+    return issues
+
+def _check_categorical_correlation(analyzer, threshold: float = 0.8, critical_threshold: float = 0.95):
+    issues = []
+    categorical = analyzer.df.select_dtypes(include="object").columns.tolist()
+    for i, c1 in enumerate(categorical):
+        for c2 in categorical[i + 1 :]:
+            try:
+                table = pd.crosstab(analyzer.df[c1], analyzer.df[c2])
+                chi2, _, _, _ = chi2_contingency(table)
+                n = table.sum().sum()
+                phi2 = chi2 / n
+                r, k = table.shape
+                cramers_v = np.sqrt(phi2 / min(k - 1, r - 1))
+                if cramers_v > threshold:
+                    severity = "critical" if cramers_v > critical_threshold else "warning"
+                    impact = "high" if severity == "critical" else "medium"
+                    quick_fix = (
+                        "Options: \n- Drop one feature: Avoids overfitting from high redundancy (Pros: Simplifies model; Cons: Loses info).\n- Engineer feature: Extract common patterns (e.g., group categories) (Pros: Retains info; Cons: Requires domain knowledge).\n- Retain and test: Use robust models (e.g., trees) (Pros: Keeps info; Cons: May affect sensitive models)."
+                        if severity == "critical"
+                        else "Options: \n- Drop one feature: If less predictive (Pros: Simplifies model; Cons: Loses info).\n- Retain and test: Evaluate with robust models (Pros: Keeps info; Cons: Risk of redundancy).\n- Engineer feature: Group categories or encode differently (Pros: Reduces redundancy; Cons: Adds complexity)."
+                    )
+                    issues.append(
+                        Issues(
+                            category="feature_correlation",
+                            severity=severity,
+                            column=f"{c1},{c2}",
+                            description=f"Columns '{c1}' and '{c2}' are highly associated (Cramer's V: {float(cramers_v):.2f})",
+                            impact_score=impact,
+                            quick_fix=quick_fix,
+                        )
+                    )
+            except Exception:
+                continue
+    return issues
+
+def _check_mixed_correlation(analyzer, p_threshold: float = 0.05, critical_p_threshold: float = 0.001):
+    issues = []
+    cat_cols = analyzer.df.select_dtypes(
+        include=["object", "category"]
+    ).columns.tolist()
+    num_cols = analyzer.df.select_dtypes(include=["int64", "float64"]).columns.tolist()
+    for cat in cat_cols:
+        for num in num_cols:
+            groups = [
+                analyzer.df.loc[analyzer.df[cat] == level, num].dropna().to_numpy()
+                for level in analyzer.df[cat].dropna().unique()
+                if len(analyzer.df.loc[analyzer.df[cat] == level, num].dropna()) > 1
+            ]
+            if len(groups) < 2 or all(np.var(g, ddof=1) == 0 for g in groups):
+                continue
+            try:
+                f_stat, p_val = f_oneway(*groups)
+                if p_val < p_threshold:
+                    severity = (
+                        "critical"
+                        if p_val < critical_p_threshold and f_stat > 20.0
+                        else "warning"
+                    )
+                    impact = "high" if severity == "critical" else "medium"
+                    quick_fix = (
+                        "Options: \n- Drop one feature: Avoids redundancy (Pros: Simplifies model; Cons: Loses info).\n- Engineer feature: Transform categorical or numeric feature (Pros: Retains info; Cons: Adds complexity).\n- Retain and test: Use robust models (e.g., trees) (Pros: Keeps info; Cons: May affect sensitive models)."
+                        if severity == "critical"
+                        else "Options: \n- Drop one feature: If less predictive (Pros: Simplifies model; Cons: Loses info).\n- Retain and test: Evaluate with robust models (Pros: Keeps info; Cons: Risk of redundancy).\n- Engineer feature: Transform or encode differently (Pros: Reduces redundancy; Cons: Adds complexity)."
+                    )
+                    issues.append(
+                        Issues(
+                            category="feature_correlation",
+                            severity=severity,
+                            column=f"{cat},{num}",
+                            description=f"Columns '{cat}' and '{num}' show strong association (F: {float(f_stat):.2f}, p: {float(p_val):.4f})",
+                            impact_score=impact,
+                            quick_fix=quick_fix,
+                        )
+                    )
+            except Exception:
+                continue
+    return issues

hashprep-0.1.0a0/hashprep/checks/imbalance.py

@@ -0,0 +1,18 @@
+from .core import Issues
+
+def _check_class_imbalance(analyzer, threshold: float = 0.9):
+    issues = []
+    if analyzer.target_col and analyzer.target_col in analyzer.df.columns:
+        counts = analyzer.df[analyzer.target_col].value_counts(normalize=True)
+        if counts.max() > threshold:
+            issues.append(
+                Issues(
+                    category="class_imbalance",
+                    severity="warning",
+                    column=analyzer.target_col,
+                    description=f"Target '{analyzer.target_col}' is imbalanced ({float(counts.max()):.1%} in one class)",
+                    impact_score="medium",
+                    quick_fix="Options: \n- Resample data: Use oversampling (e.g., SMOTE) or undersampling (Pros: Balances classes; Cons: May introduce bias or lose data).\n- Use class weights: Adjust model weights for imbalance (Pros: Simple; Cons: Model-dependent).\n- Stratified sampling: Ensure balanced splits in training (Pros: Improves evaluation; Cons: Requires careful implementation).",
+                )
+            )
+    return issues