featuresmith-core 0.1.0__py3-none-any.whl

featuresmith/profiling/correlation.py

@@ -0,0 +1,67 @@
+"""Correlation summary module."""
+
+from __future__ import annotations
+
+import math
+
+import pandas as pd
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import CorrelationSummary
+
+
+def compute_correlations(
+    dataset: Dataset,
+    numeric_columns: list[str],
+    max_correlation_columns: int = 100,
+) -> CorrelationSummary:
+    """Compute pairwise Pearson correlation matrix for numeric columns.
+
+    Args:
+        dataset: The normalized dataset.
+        numeric_columns: List of numeric column names.
+        max_correlation_columns: Maximum number of numeric columns to correlate.
+
+    Returns:
+        A CorrelationSummary object.
+    """
+    if not numeric_columns:
+        return CorrelationSummary(pearson={})
+
+    # Cap columns to prevent combinatorial blowup
+    cols = numeric_columns[:max_correlation_columns]
+    df = dataset.dataframe
+
+    pearson: dict[str, dict[str, float | None]] = {c: {} for c in cols}
+
+    if dataset.backend == "pandas":
+        # Pandas
+        corr_df = df[cols].corr(method="pearson")
+        for col1 in cols:
+            for col2 in cols:
+                val = corr_df.loc[col1, col2]
+                if pd.isna(val) or math.isnan(val) or math.isinf(val):
+                    pearson[col1][col2] = None
+                else:
+                    pearson[col1][col2] = float(val)
+    else:
+        # Polars: compute correlations in parallel using expressions
+        exprs = []
+        for i, c1 in enumerate(cols):
+            for c2 in cols[i:]:
+                exprs.append(pl.corr(c1, c2).alias(f"{c1}__{c2}"))
+
+        if exprs:
+            res = df.select(exprs)
+            for i, c1 in enumerate(cols):
+                for c2 in cols[i:]:
+                    val = res.get_column(f"{c1}__{c2}")[0]
+                    if val is None or math.isnan(val) or math.isinf(val):
+                        f_val = None
+                    else:
+                        f_val = float(val)
+                    pearson[c1][c2] = f_val
+                    pearson[c2][c1] = f_val
+
+    return CorrelationSummary(pearson=pearson)

featuresmith/profiling/datetime.py

@@ -0,0 +1,105 @@
+"""Datetime column profiling module."""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+
+import pandas as pd
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import DatetimeProfile
+
+
+def profile_datetime_column(dataset: Dataset, col_name: str) -> DatetimeProfile:
+    """Profile a single datetime column.
+
+    Args:
+        dataset: The normalized dataset.
+        col_name: Name of the column.
+
+    Returns:
+        A DatetimeProfile object.
+    """
+    df = dataset.dataframe
+
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        missing_count = int(series.isna().sum())
+
+        min_val = series.min()
+        max_val = series.max()
+
+        # Check if min_val/max_val are null (NaT)
+        if pd.isna(min_val) or pd.isna(max_val):
+            return DatetimeProfile(
+                column_name=col_name,
+                minimum=None,
+                maximum=None,
+                range_days=None,
+                missing_count=missing_count,
+                earliest_record=None,
+                latest_record=None,
+            )
+
+        # Handle conversion to standard datetime if they are Timestamp objects
+        if hasattr(min_val, "to_pydatetime"):
+            min_dt = min_val.to_pydatetime()
+        else:
+            min_dt = min_val
+
+        if hasattr(max_val, "to_pydatetime"):
+            max_dt = max_val.to_pydatetime()
+        else:
+            max_dt = max_val
+
+        # If min_dt/max_dt are date (not datetime) objects, convert them to datetimes to subtract safely
+        if isinstance(min_dt, date) and not isinstance(min_dt, datetime):
+            min_dt = datetime.combine(min_dt, datetime.min.time())
+        if isinstance(max_dt, date) and not isinstance(max_dt, datetime):
+            max_dt = datetime.combine(max_dt, datetime.min.time())
+
+        # ISO format
+        minimum = min_val.isoformat()
+        maximum = max_val.isoformat()
+        range_days = float((max_dt - min_dt).total_seconds() / 86400.0)
+
+    else:
+        # Polars
+        missing_count = int(df.select(pl.col(col_name).null_count())[0, 0])
+        min_val = df.select(pl.col(col_name).min())[0, 0]
+        max_val = df.select(pl.col(col_name).max())[0, 0]
+
+        if min_val is None or max_val is None:
+            return DatetimeProfile(
+                column_name=col_name,
+                minimum=None,
+                maximum=None,
+                range_days=None,
+                missing_count=missing_count,
+                earliest_record=None,
+                latest_record=None,
+            )
+
+        # Polars returns datetime.date or datetime.datetime objects directly
+        min_dt = min_val
+        max_dt = max_val
+
+        if isinstance(min_dt, date) and not isinstance(min_dt, datetime):
+            min_dt = datetime.combine(min_dt, datetime.min.time())
+        if isinstance(max_dt, date) and not isinstance(max_dt, datetime):
+            max_dt = datetime.combine(max_dt, datetime.min.time())
+
+        minimum = min_val.isoformat()
+        maximum = max_val.isoformat()
+        range_days = float((max_dt - min_dt).total_seconds() / 86400.0)
+
+    return DatetimeProfile(
+        column_name=col_name,
+        minimum=minimum,
+        maximum=maximum,
+        range_days=range_days,
+        missing_count=missing_count,
+        earliest_record=minimum,
+        latest_record=maximum,
+    )

featuresmith/profiling/duplicates.py

@@ -0,0 +1,48 @@
+"""Duplicate analysis module."""
+
+from __future__ import annotations
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import DuplicateSummary
+
+
+def analyze_duplicates(
+    dataset: Dataset, constant_columns: list[str], fully_empty_columns: list[str]
+) -> DuplicateSummary:
+    """Analyze duplicate rows in the dataset.
+
+    Args:
+        dataset: The normalized dataset.
+        constant_columns: Pre-identified constant columns.
+        fully_empty_columns: Pre-identified fully empty columns.
+
+    Returns:
+        A DuplicateSummary object.
+    """
+    df = dataset.dataframe
+    row_count = dataset.row_count
+
+    if row_count == 0:
+        return DuplicateSummary(
+            duplicate_rows_count=0,
+            duplicate_percentage=0.0,
+            constant_columns=constant_columns,
+            fully_empty_columns=fully_empty_columns,
+        )
+
+    if dataset.backend == "pandas":
+        duplicate_rows_count = int(df.duplicated().sum())
+    else:
+        # Polars: duplicate rows count is total rows minus unique rows
+        # This is equivalent to pandas duplicated().sum()
+        unique_rows_count = df.unique().height
+        duplicate_rows_count = row_count - unique_rows_count
+
+    duplicate_percentage = float((duplicate_rows_count / row_count) * 100.0)
+
+    return DuplicateSummary(
+        duplicate_rows_count=duplicate_rows_count,
+        duplicate_percentage=duplicate_percentage,
+        constant_columns=list(constant_columns),
+        fully_empty_columns=list(fully_empty_columns),
+    )

featuresmith/profiling/missing.py

@@ -0,0 +1,70 @@
+"""Missing value analysis module."""
+
+from __future__ import annotations
+
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import MissingValueSummary
+
+
+def analyze_missing_values(dataset: Dataset) -> MissingValueSummary:
+    """Analyze missing values in the dataset.
+
+    Args:
+        dataset: The normalized dataset.
+
+    Returns:
+        A MissingValueSummary object.
+    """
+    df = dataset.dataframe
+    row_count = dataset.row_count
+    col_count = dataset.column_count
+    total_cells = row_count * col_count
+
+    column_missing_counts: dict[str, int] = {}
+    column_missing_percentages: dict[str, float] = {}
+    total_missing = 0
+
+    if col_count == 0 or row_count == 0:
+        # Edge case: empty dataset
+        for col_name in dataset.schema.names:
+            column_missing_counts[col_name] = row_count
+            column_missing_percentages[col_name] = 100.0 if row_count > 0 else 0.0
+        return MissingValueSummary(
+            column_missing_counts=column_missing_counts,
+            column_missing_percentages=column_missing_percentages,
+            total_missing=row_count * col_count,
+            dataset_missing_percentage=100.0 if total_cells > 0 else 0.0,
+        )
+
+    if dataset.backend == "pandas":
+        # Compute missing counts for all columns at once in pandas
+        missing_series = df.isna().sum()
+        for col_name in dataset.schema.names:
+            count = int(missing_series[col_name])
+            column_missing_counts[col_name] = count
+            column_missing_percentages[col_name] = float((count / row_count) * 100.0)
+            total_missing += count
+    else:
+        # Polars
+        # Compute null counts for all columns in a single select
+        null_counts = df.select(
+            [pl.col(col).null_count().alias(col) for col in dataset.schema.names]
+        )
+        for col_name in dataset.schema.names:
+            count = int(null_counts.get_column(col_name)[0])
+            column_missing_counts[col_name] = count
+            column_missing_percentages[col_name] = float((count / row_count) * 100.0)
+            total_missing += count
+
+    dataset_missing_percentage = (
+        float((total_missing / total_cells) * 100.0) if total_cells > 0 else 0.0
+    )
+
+    return MissingValueSummary(
+        column_missing_counts=column_missing_counts,
+        column_missing_percentages=column_missing_percentages,
+        total_missing=total_missing,
+        dataset_missing_percentage=dataset_missing_percentage,
+    )

featuresmith/profiling/numeric.py

@@ -0,0 +1,248 @@
+"""Numeric column profiling module."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import polars as pl
+
+from featuresmith.core.dataset import Dataset
+from featuresmith.core.profile_result import NumericProfile
+
+
+def _to_float(val: Any) -> float | None:
+    """Safely convert a value to a Python float, returning None if NaN/Inf."""
+    if val is None:
+        return None
+    # Check for pandas/numpy nulls
+    if isinstance(val, (float, int)) and (math.isnan(val) or math.isinf(val)):
+        return None
+    try:
+        # If it's a series or list, we grab first element
+        if hasattr(val, "item"):
+            val = val.item()
+        elif hasattr(val, "iloc"):
+            val = val.iloc[0]
+
+        f = float(val)
+        if math.isnan(f) or math.isinf(f):
+            return None
+        return f
+    except (ValueError, TypeError):
+        return None
+
+
+def _to_int(val: Any) -> int:
+    """Convert a value to a Python int."""
+    if val is None:
+        return 0
+    try:
+        if hasattr(val, "item"):
+            val = val.item()
+        return int(val)
+    except (ValueError, TypeError):
+        return 0
+
+
+def profile_numeric_column(dataset: Dataset, col_name: str) -> NumericProfile:
+    """Profile a single numeric column.
+
+    Args:
+        dataset: The normalized dataset.
+        col_name: Name of the column.
+
+    Returns:
+        A NumericProfile object.
+    """
+    df = dataset.dataframe
+    row_count = dataset.row_count
+
+    if dataset.backend == "pandas":
+        series = df[col_name]
+        non_null_series = series.dropna()
+        count = int(non_null_series.count())
+        missing_count = int(series.isna().sum())
+        missing_percentage = (
+            float((missing_count / row_count) * 100.0) if row_count > 0 else 0.0
+        )
+        unique_count = int(non_null_series.nunique())
+
+        if count == 0:
+            return NumericProfile(
+                column_name=col_name,
+                count=0,
+                missing_count=missing_count,
+                missing_percentage=missing_percentage,
+                unique_count=0,
+                mean=None,
+                median=None,
+                mode=None,
+                minimum=None,
+                maximum=None,
+                range=None,
+                variance=None,
+                std_dev=None,
+                q1=None,
+                q2=None,
+                q3=None,
+                iqr=None,
+                sum=None,
+                zero_count=0,
+                negative_count=0,
+                positive_count=0,
+                skewness=None,
+                kurtosis=None,
+            )
+
+        mean = _to_float(non_null_series.mean())
+        median = _to_float(non_null_series.median())
+        minimum = _to_float(non_null_series.min())
+        maximum = _to_float(non_null_series.max())
+
+        rng = None
+        if minimum is not None and maximum is not None:
+            rng = maximum - minimum
+
+        variance = _to_float(non_null_series.var(ddof=1))
+        std_dev = _to_float(non_null_series.std(ddof=1))
+
+        q1 = _to_float(non_null_series.quantile(0.25))
+        q2 = _to_float(non_null_series.quantile(0.50))
+        q3 = _to_float(non_null_series.quantile(0.75))
+
+        iqr = None
+        if q1 is not None and q3 is not None:
+            iqr = q3 - q1
+
+        sm = _to_float(non_null_series.sum())
+        zero_count = int((non_null_series == 0).sum())
+        negative_count = int((non_null_series < 0).sum())
+        positive_count = int((non_null_series > 0).sum())
+
+        # Mode
+        modes = non_null_series.mode()
+        mode = _to_float(modes.iloc[0]) if not modes.empty else None
+
+        skewness = _to_float(non_null_series.skew())
+        kurtosis = _to_float(non_null_series.kurt())
+
+    else:
+        # Polars
+        col = pl.col(col_name)
+        non_null_col = col.drop_nulls()
+
+        # Compute everything in a single select statement
+        res = df.select(
+            [
+                col.len().alias("total_len"),
+                col.null_count().alias("missing_count"),
+                non_null_col.n_unique().alias("unique_count"),
+                non_null_col.mean().alias("mean"),
+                non_null_col.median().alias("median"),
+                non_null_col.min().alias("min"),
+                non_null_col.max().alias("max"),
+                non_null_col.var(ddof=1).alias("var"),
+                non_null_col.std(ddof=1).alias("std"),
+                non_null_col.quantile(0.25).alias("q1"),
+                non_null_col.quantile(0.50).alias("q2"),
+                non_null_col.quantile(0.75).alias("q3"),
+                non_null_col.sum().alias("sum"),
+                (non_null_col == 0).sum().alias("zero_count"),
+                (non_null_col < 0).sum().alias("negative_count"),
+                (non_null_col > 0).sum().alias("positive_count"),
+                non_null_col.skew().alias("skew"),
+                non_null_col.kurtosis().alias("kurt"),
+                non_null_col.mode().sort().first().alias("mode"),
+            ]
+        )
+
+        total_len = _to_int(res.get_column("total_len")[0])
+        missing_count = _to_int(res.get_column("missing_count")[0])
+        missing_percentage = (
+            float((missing_count / total_len) * 100.0) if total_len > 0 else 0.0
+        )
+        unique_count = _to_int(res.get_column("unique_count")[0])
+        count = total_len - missing_count
+
+        if count == 0:
+            return NumericProfile(
+                column_name=col_name,
+                count=0,
+                missing_count=missing_count,
+                missing_percentage=missing_percentage,
+                unique_count=0,
+                mean=None,
+                median=None,
+                mode=None,
+                minimum=None,
+                maximum=None,
+                range=None,
+                variance=None,
+                std_dev=None,
+                q1=None,
+                q2=None,
+                q3=None,
+                iqr=None,
+                sum=None,
+                zero_count=0,
+                negative_count=0,
+                positive_count=0,
+                skewness=None,
+                kurtosis=None,
+            )
+
+        mean = _to_float(res.get_column("mean")[0])
+        median = _to_float(res.get_column("median")[0])
+        minimum = _to_float(res.get_column("min")[0])
+        maximum = _to_float(res.get_column("max")[0])
+
+        rng = None
+        if minimum is not None and maximum is not None:
+            rng = maximum - minimum
+
+        variance = _to_float(res.get_column("var")[0])
+        std_dev = _to_float(res.get_column("std")[0])
+
+        q1 = _to_float(res.get_column("q1")[0])
+        q2 = _to_float(res.get_column("q2")[0])
+        q3 = _to_float(res.get_column("q3")[0])
+
+        iqr = None
+        if q1 is not None and q3 is not None:
+            iqr = q3 - q1
+
+        sm = _to_float(res.get_column("sum")[0])
+        zero_count = _to_int(res.get_column("zero_count")[0])
+        negative_count = _to_int(res.get_column("negative_count")[0])
+        positive_count = _to_int(res.get_column("positive_count")[0])
+
+        mode = _to_float(res.get_column("mode")[0])
+        skewness = _to_float(res.get_column("skew")[0])
+        kurtosis = _to_float(res.get_column("kurt")[0])
+
+    return NumericProfile(
+        column_name=col_name,
+        count=count,
+        missing_count=missing_count,
+        missing_percentage=missing_percentage,
+        unique_count=unique_count,
+        mean=mean,
+        median=median,
+        mode=mode,
+        minimum=minimum,
+        maximum=maximum,
+        range=rng,
+        variance=variance,
+        std_dev=std_dev,
+        q1=q1,
+        q2=q2,
+        q3=q3,
+        iqr=iqr,
+        sum=sm,
+        zero_count=zero_count,
+        negative_count=negative_count,
+        positive_count=positive_count,
+        skewness=skewness,
+        kurtosis=kurtosis,
+    )