datapilot-kit 0.3.0rc1__py3-none-any.whl

datapilot/__init__.py

@@ -0,0 +1,11 @@
+"""
+Datapilot
+
+An intelligent Python library for exploratory data analysis.
+"""
+
+__version__ = "0.3.0rc1"
+
+from .api import analyze
+
+__all__ = ["analyze"]

datapilot/analysis/__init__.py

@@ -0,0 +1,3 @@
+"""
+Analysis modules for Datapilot.
+"""

datapilot/analysis/correlation.py

@@ -0,0 +1,58 @@
+"""
+Correlation analysis for Datapilot.
+"""
+
+from typing import SupportsFloat, cast
+
+import pandas as pd
+
+from .models import CorrelationSummary
+
+
+def generate_correlation_summary(
+    dataframe: pd.DataFrame,
+) -> CorrelationSummary:
+    """
+    Generate Pearson correlation analysis for numeric columns.
+    """
+
+    numeric_dataframe = dataframe.select_dtypes(
+        include="number",
+    )
+
+    correlation_matrix = numeric_dataframe.corr()
+
+    strong_positive_pairs: dict[str, float] = {}
+    strong_negative_pairs: dict[str, float] = {}
+
+    columns = list(correlation_matrix.columns)
+
+    for i, left in enumerate(columns):
+        for right in columns[i + 1:]:
+
+            correlation = cast(
+                SupportsFloat,
+                correlation_matrix.loc[left, right],
+            )
+
+            value = float(correlation)
+
+            pair = f"{left} ↔ {right}"
+
+            if value >= 0.70:
+                strong_positive_pairs[pair] = round(
+                    value,
+                    3,
+                )
+
+            elif value <= -0.70:
+                strong_negative_pairs[pair] = round(
+                    value,
+                    3,
+                )
+
+    return CorrelationSummary(
+        correlation_matrix=correlation_matrix,
+        strong_positive_pairs=strong_positive_pairs,
+        strong_negative_pairs=strong_negative_pairs,
+    )

datapilot/analysis/datatype.py

@@ -0,0 +1,48 @@
+"""
+Data type analysis for Datapilot.
+"""
+
+import pandas as pd
+
+from .models import DataTypeSummary
+
+
+def generate_data_type_summary(
+    dataframe: pd.DataFrame,
+) -> DataTypeSummary:
+    """
+    Generate data type statistics for a dataset.
+
+    Parameters
+    ----------
+    dataframe : pandas.DataFrame
+        Dataset to analyze.
+
+    Returns
+    -------
+    DataTypeSummary
+        Summary of dataset column types.
+    """
+
+    numeric_columns = dataframe.select_dtypes(
+        include=["number"]
+    ).columns.tolist()
+
+    categorical_columns = dataframe.select_dtypes(
+    include=["object", "string", "category"]
+    ).columns.tolist()
+
+    boolean_columns = dataframe.select_dtypes(
+        include=["bool"]
+    ).columns.tolist()
+
+    datetime_columns = dataframe.select_dtypes(
+        include=["datetime", "datetimetz"]
+    ).columns.tolist()
+
+    return DataTypeSummary(
+        numeric_columns=numeric_columns,
+        categorical_columns=categorical_columns,
+        boolean_columns=boolean_columns,
+        datetime_columns=datetime_columns,
+    )

datapilot/analysis/duplicate.py

@@ -0,0 +1,40 @@
+"""
+Duplicate row analysis for Datapilot.
+"""
+
+import pandas as pd
+
+from .models import DuplicateSummary
+
+
+def generate_duplicate_summary(
+    dataframe: pd.DataFrame,
+) -> DuplicateSummary:
+    """
+    Generate duplicate row statistics for a dataset.
+
+    Parameters
+    ----------
+    dataframe : pandas.DataFrame
+        Dataset to analyze.
+
+    Returns
+    -------
+    DuplicateSummary
+        Summary of duplicate rows in the dataset.
+    """
+
+    total_duplicates = int(dataframe.duplicated().sum())
+
+    total_rows = len(dataframe)
+
+    duplicate_percentage = (
+        round((total_duplicates / total_rows) * 100, 2)
+        if total_rows > 0
+        else 0.0
+    )
+
+    return DuplicateSummary(
+        total_duplicates=total_duplicates,
+        duplicate_percentage=duplicate_percentage,
+    )

datapilot/analysis/health.py

@@ -0,0 +1,163 @@
+"""
+Dataset health assessment for Datapilot.
+"""
+
+import pandas as pd
+
+from .datatype import generate_data_type_summary
+from .duplicate import generate_duplicate_summary
+from .missing import generate_missing_value_summary
+from .models import DatasetHealth
+
+
+def generate_dataset_health(
+    dataframe: pd.DataFrame,
+) -> DatasetHealth:
+    """
+    Generate an overall health assessment for a dataset.
+    """
+
+    missing = generate_missing_value_summary(dataframe)
+    duplicates = generate_duplicate_summary(dataframe)
+    data_types = generate_data_type_summary(dataframe)
+
+    score = 100.0
+
+    # -----------------------------
+    # Missing Value Penalty (40 pts)
+    # -----------------------------
+    score -= (missing.missing_percentage / 100) * 40
+
+    # -----------------------------
+    # Duplicate Penalty (35 pts)
+    # -----------------------------
+    score -= (duplicates.duplicate_percentage / 100) * 35
+
+    # -----------------------------
+    # Structure Penalty (25 pts)
+    # -----------------------------
+    total_columns = len(dataframe.columns)
+
+    recognized_columns = (
+        len(data_types.numeric_columns)
+        + len(data_types.categorical_columns)
+        + len(data_types.boolean_columns)
+        + len(data_types.datetime_columns)
+    )
+
+    unrecognized_columns = max(
+        total_columns - recognized_columns,
+        0,
+    )
+
+    if total_columns > 0:
+        score -= (
+            unrecognized_columns / total_columns
+        ) * 25
+
+    score = max(0, min(100, round(score)))
+
+    # -----------------------------
+    # Grade
+    # -----------------------------
+    if score >= 95:
+        grade = "A+"
+        status = "Excellent"
+
+    elif score >= 90:
+        grade = "A"
+        status = "Healthy"
+
+    elif score >= 80:
+        grade = "B"
+        status = "Good"
+
+    elif score >= 70:
+        grade = "C"
+        status = "Fair"
+
+    elif score >= 60:
+        grade = "D"
+        status = "Poor"
+
+    else:
+        grade = "F"
+        status = "Critical"
+
+    # -----------------------------
+    # ML Readiness
+    # -----------------------------
+    ml_ready = (
+        score >= 85
+        and missing.missing_percentage <= 20
+        and duplicates.duplicate_percentage <= 5
+    )
+
+    # -----------------------------
+    # Strengths
+    # -----------------------------
+    strengths = []
+
+    if missing.total_missing == 0:
+        strengths.append("No missing values detected.")
+
+    if duplicates.total_duplicates == 0:
+        strengths.append("No duplicate rows detected.")
+
+    if unrecognized_columns == 0:
+        strengths.append("All columns have recognized data types.")
+
+    # -----------------------------
+    # Weaknesses
+    # -----------------------------
+    weaknesses = []
+
+    if missing.total_missing > 0:
+        weaknesses.append(
+            f"{missing.total_missing} missing values detected."
+        )
+
+    if duplicates.total_duplicates > 0:
+        weaknesses.append(
+            f"{duplicates.total_duplicates} duplicate rows detected."
+        )
+
+    if unrecognized_columns > 0:
+        weaknesses.append(
+            f"{unrecognized_columns} columns have unsupported data types."
+        )
+
+    # -----------------------------
+    # Recommendations
+    # -----------------------------
+    recommendations = []
+
+    if missing.missing_percentage > 0:
+        recommendations.append(
+            "Handle missing values before further analysis."
+        )
+
+    if duplicates.total_duplicates > 0:
+        recommendations.append(
+            "Remove duplicate rows."
+        )
+
+    if unrecognized_columns > 0:
+        recommendations.append(
+            "Review unsupported column data types."
+        )
+
+    if not recommendations:
+        recommendations.append(
+            "Dataset is ready for analysis."
+        )
+
+    return DatasetHealth(
+        score=score,
+        grade=grade,
+        status=status,
+        ml_ready=ml_ready,
+        strengths=strengths,
+        weaknesses=weaknesses,
+        recommendations=recommendations,
+    )

datapilot/analysis/insights.py

@@ -0,0 +1,101 @@
+"""
+Insight generation for Datapilot.
+"""
+
+import pandas as pd
+
+from .correlation import generate_correlation_summary
+from .duplicate import generate_duplicate_summary
+from .missing import generate_missing_value_summary
+from .models import InsightSummary
+from .outliers import generate_outlier_summary
+
+
+def generate_insight_summary(
+    dataframe: pd.DataFrame,
+) -> InsightSummary:
+    """
+    Generate actionable dataset insights.
+    """
+
+    insights: list[str] = []
+
+    recommendations: list[str] = []
+
+    missing = generate_missing_value_summary(
+        dataframe
+    )
+
+    duplicates = generate_duplicate_summary(
+        dataframe
+    )
+
+    outliers = generate_outlier_summary(
+        dataframe
+    )
+
+    correlation = generate_correlation_summary(
+        dataframe
+    )
+
+    if missing.total_missing > 0:
+
+        insights.append(
+            f"Dataset contains "
+            f"{missing.total_missing} missing values."
+        )
+
+        recommendations.append(
+            "Handle missing values before "
+            "training machine learning models."
+        )
+
+    if duplicates.total_duplicates > 0:
+
+        insights.append(
+            f"Dataset contains "
+            f"{duplicates.total_duplicates} duplicate rows."
+        )
+
+        recommendations.append(
+            "Remove duplicate rows."
+        )
+
+    if outliers.total_outliers > 0:
+
+        insights.append(
+            f"Detected "
+            f"{outliers.total_outliers} statistical outliers."
+        )
+
+        recommendations.append(
+            "Review outliers before modelling."
+        )
+
+    if correlation.strong_positive_pairs:
+
+        insights.append(
+            "Highly correlated numeric features detected."
+        )
+
+        recommendations.append(
+            "Review correlated features to "
+            "reduce multicollinearity."
+        )
+
+    if correlation.strong_negative_pairs:
+
+        insights.append(
+            "Strong negative correlations detected."
+        )
+
+    if not insights:
+
+        insights.append(
+            "No significant data quality issues detected."
+        )
+
+    return InsightSummary(
+        insights=insights,
+        recommendations=recommendations,
+    )

datapilot/analysis/missing.py

@@ -0,0 +1,56 @@
+"""
+Missing value analysis for Datapilot.
+"""
+
+import pandas as pd
+
+from .models import MissingValueSummary
+
+
+def generate_missing_value_summary(
+    dataframe: pd.DataFrame,
+) -> MissingValueSummary:
+    """
+    Generate missing value statistics for a dataset.
+
+    Parameters
+    ----------
+    dataframe : pandas.DataFrame
+        Dataset to analyze.
+
+    Returns
+    -------
+    MissingValueSummary
+        Summary of missing values in the dataset.
+    """
+
+    total_missing = int(dataframe.isna().sum().sum())
+
+    total_cells = dataframe.shape[0] * dataframe.shape[1]
+
+    missing_percentage = (
+        round((total_missing / total_cells) * 100, 2)
+        if total_cells > 0
+        else 0.0
+    )
+
+    missing_per_column = dataframe.isna().sum()
+
+    columns_with_missing = {
+        str(column): int(count)
+        for column, count in missing_per_column.items()
+        if count > 0
+    }
+
+    columns_without_missing = [
+        str(column)
+        for column, count in missing_per_column.items()
+        if count == 0
+    ]
+
+    return MissingValueSummary(
+        total_missing=total_missing,
+        missing_percentage=missing_percentage,
+        columns_with_missing=columns_with_missing,
+        columns_without_missing=columns_without_missing,
+    )

datapilot/analysis/models.py

@@ -0,0 +1,108 @@
+"""
+Data models used by Datapilot analysis modules.
+"""
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+
+@dataclass(slots=True)
+class DatasetSummary:
+    """
+    Summary information about a dataset.
+    """
+
+    rows: int
+    columns: int
+    memory_usage_mb: float
+    column_names: list[str]
+
+
+@dataclass(slots=True)
+class MissingValueSummary:
+    """
+    Summary information about missing values in a dataset.
+    """
+
+    total_missing: int
+    missing_percentage: float
+    columns_with_missing: dict[str, int]
+    columns_without_missing: list[str]
+
+
+@dataclass(slots=True)
+class DuplicateSummary:
+    """
+    Summary information about duplicate rows in a dataset.
+    """
+
+    total_duplicates: int
+    duplicate_percentage: float
+
+
+@dataclass(slots=True)
+class DataTypeSummary:
+    """
+    Summary information about dataset column types.
+    """
+
+    numeric_columns: list[str]
+    categorical_columns: list[str]
+    boolean_columns: list[str]
+    datetime_columns: list[str]
+
+
+@dataclass(slots=True)
+class DatasetHealth:
+    """
+    Overall health assessment of a dataset.
+    """
+
+    score: int
+    grade: str
+    status: str
+    ml_ready: bool
+    strengths: list[str]
+    weaknesses: list[str]
+    recommendations: list[str]
+
+
+@dataclass(slots=True)
+class StatisticsSummary:
+    """
+    Statistical summary of numeric columns.
+    """
+
+    column_statistics: dict[str, dict[str, float]]
+
+
+@dataclass(slots=True)
+class OutlierSummary:
+    """
+    Summary of outlier detection.
+    """
+
+    total_outliers: int
+    outlier_percentage: float
+    columns_with_outliers: dict[str, int]
+    columns_without_outliers: list[str]
+
+@dataclass(slots=True)
+class CorrelationSummary:
+    """
+    Summary of correlation analysis.
+    """
+
+    correlation_matrix: pd.DataFrame
+    strong_positive_pairs: dict[str, float]
+    strong_negative_pairs: dict[str, float]
+
+@dataclass(slots=True)
+class InsightSummary:
+    """
+    Summary of generated dataset insights.
+    """
+
+    insights: list[str]
+    recommendations: list[str]    

datapilot/analysis/outliers.py

@@ -0,0 +1,85 @@
+"""
+Outlier detection for Datapilot.
+"""
+
+import pandas as pd
+
+from .models import OutlierSummary
+
+
+def generate_outlier_summary(
+    dataframe: pd.DataFrame,
+) -> OutlierSummary:
+    """
+    Detect outliers using the IQR method.
+    """
+
+    numeric_dataframe = dataframe.select_dtypes(
+        include="number",
+    )
+
+    total_outliers = 0
+
+    columns_with_outliers: dict[str, int] = {}
+
+    columns_without_outliers: list[str] = []
+
+    for column in numeric_dataframe.columns:
+
+        series = numeric_dataframe[column].dropna()
+
+        if series.empty:
+            columns_without_outliers.append(
+                str(column)
+            )
+            continue
+
+        q1 = series.quantile(0.25)
+        q3 = series.quantile(0.75)
+
+        iqr = q3 - q1
+
+        lower_bound = q1 - (1.5 * iqr)
+        upper_bound = q3 + (1.5 * iqr)
+
+        outlier_count = int(
+            (
+                (series < lower_bound)
+                | (series > upper_bound)
+            ).sum()
+        )
+
+        if outlier_count > 0:
+
+            columns_with_outliers[
+                str(column)
+            ] = outlier_count
+
+            total_outliers += outlier_count
+
+        else:
+
+            columns_without_outliers.append(
+                str(column)
+            )
+
+    total_numeric_values = int(
+        numeric_dataframe.count().sum()
+    )
+
+    outlier_percentage = (
+        round(
+            (total_outliers / total_numeric_values)
+            * 100,
+            2,
+        )
+        if total_numeric_values > 0
+        else 0.0
+    )
+
+    return OutlierSummary(
+        total_outliers=total_outliers,
+        outlier_percentage=outlier_percentage,
+        columns_with_outliers=columns_with_outliers,
+        columns_without_outliers=columns_without_outliers,
+    )