se-dat 0.1.0__tar.gz

se_dat-0.1.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: se-dat
+Version: 0.1.0
+Summary: Simple Exploratory Data Analysis tool: column profiling, correlation analysis, and encoding suggestions.
+Author-email: Haim Feldman <haimfeld@gmail.com>
+License: MIT
+Keywords: eda,data-analysis,pandas,correlation,encoding
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=1.5
+Requires-Dist: scipy>=1.9
+Requires-Dist: matplotlib>=3.6
+Requires-Dist: seaborn>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+
+# se-dat
+
+Simple Exploratory Data Analysis tool. Profiles your columns, measures
+associations across numeric/categorical data, and suggests encodings so you can
+move from raw data to a model-ready frame in one call.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+## Features
+
+1. **Column type profiling** — per column: inferred type (`numeric`, `boolean`,
+   `categorical`, `string`, `datetime`, `id`), missing %, unique count,
+   cardinality ratio, and a confidence flag when the inference is ambiguous
+   (numeric-looking strings, `0/1` vs `yes`/`no` booleans, etc.).
+2. **Correlation analysis** — Pearson/Spearman matrices + heatmap
+   (numeric-numeric), Cramér's V (categorical-categorical), correlation ratio
+   eta (numeric-categorical), plus multicollinearity flags above a threshold.
+3. **Encoding suggestions + auto-transform** — detects binary-like strings
+   (`yes`/`no`, `true`/`false`, `0/1`), suggests one-hot for low-cardinality
+   categoricals, and target/ordinal encoding for high-cardinality ones (with a
+   dimensionality warning). Apply suggestions individually or all at once.
+
+## Usage
+
+```python
+import pandas as pd
+import sedat
+
+df = pd.DataFrame({
+    "id": range(500),
+    "age": ...,
+    "income": ["45000", "67000", ...],       # numeric stored as string
+    "smoker": ["yes", "no", ...],            # binary-like
+    "region": ["north", "south", ...],
+    "target": ...,                            # numeric target
+})
+
+report = sedat.EDAReport.create(df, target=df["target"])
+print(report.profile.summary)                 # column type profiling
+print(report.correlations.flagged_pairs)      # multicollinearity warnings
+print(report.encoding_summary)                # suggested encodings
+
+model_ready = report.apply_all_encodings()    # apply every suggestion
+```
+
+### Granular APIs
+
+```python
+profile = sedat.profile_dataframe(df)
+pearson = sedat.numeric_correlation(df, method="pearson")
+cramers = sedat.categorical_correlation(df)       # Cramér's V matrix
+eta = sedat.numeric_categorical_correlation(df)   # correlation ratio matrix
+fig = sedat.correlation_heatmap(pearson, title="Numeric correlations")
+
+plan = sedat.suggest_encodings(df, target=df["target"], cardinality_threshold=10)
+only_smoker = next(s for s in plan.suggestions if s.column == "smoker")
+one_hot_step = only_smoker.apply(df)              # accept one suggestion
+all_encoded = plan.apply_all(df)                  # accept all suggestions
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+python -m pytest
+```

se_dat-0.1.0/README.md

@@ -0,0 +1,70 @@
+# se-dat
+
+Simple Exploratory Data Analysis tool. Profiles your columns, measures
+associations across numeric/categorical data, and suggests encodings so you can
+move from raw data to a model-ready frame in one call.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+## Features
+
+1. **Column type profiling** — per column: inferred type (`numeric`, `boolean`,
+   `categorical`, `string`, `datetime`, `id`), missing %, unique count,
+   cardinality ratio, and a confidence flag when the inference is ambiguous
+   (numeric-looking strings, `0/1` vs `yes`/`no` booleans, etc.).
+2. **Correlation analysis** — Pearson/Spearman matrices + heatmap
+   (numeric-numeric), Cramér's V (categorical-categorical), correlation ratio
+   eta (numeric-categorical), plus multicollinearity flags above a threshold.
+3. **Encoding suggestions + auto-transform** — detects binary-like strings
+   (`yes`/`no`, `true`/`false`, `0/1`), suggests one-hot for low-cardinality
+   categoricals, and target/ordinal encoding for high-cardinality ones (with a
+   dimensionality warning). Apply suggestions individually or all at once.
+
+## Usage
+
+```python
+import pandas as pd
+import sedat
+
+df = pd.DataFrame({
+    "id": range(500),
+    "age": ...,
+    "income": ["45000", "67000", ...],       # numeric stored as string
+    "smoker": ["yes", "no", ...],            # binary-like
+    "region": ["north", "south", ...],
+    "target": ...,                            # numeric target
+})
+
+report = sedat.EDAReport.create(df, target=df["target"])
+print(report.profile.summary)                 # column type profiling
+print(report.correlations.flagged_pairs)      # multicollinearity warnings
+print(report.encoding_summary)                # suggested encodings
+
+model_ready = report.apply_all_encodings()    # apply every suggestion
+```
+
+### Granular APIs
+
+```python
+profile = sedat.profile_dataframe(df)
+pearson = sedat.numeric_correlation(df, method="pearson")
+cramers = sedat.categorical_correlation(df)       # Cramér's V matrix
+eta = sedat.numeric_categorical_correlation(df)   # correlation ratio matrix
+fig = sedat.correlation_heatmap(pearson, title="Numeric correlations")
+
+plan = sedat.suggest_encodings(df, target=df["target"], cardinality_threshold=10)
+only_smoker = next(s for s in plan.suggestions if s.column == "smoker")
+one_hot_step = only_smoker.apply(df)              # accept one suggestion
+all_encoded = plan.apply_all(df)                  # accept all suggestions
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+python -m pytest
+```

se_dat-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "se-dat"
+version = "0.1.0"
+description = "Simple Exploratory Data Analysis tool: column profiling, correlation analysis, and encoding suggestions."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "Haim Feldman", email = "haimfeld@gmail.com" }]
+keywords = ["eda", "data-analysis", "pandas", "correlation", "encoding"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "numpy>=1.23",
+    "pandas>=1.5",
+    "scipy>=1.9",
+    "matplotlib>=3.6",
+    "seaborn>=0.12",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7", "pytest-cov"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

se_dat-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

se_dat-0.1.0/src/se_dat.egg-info/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: se-dat
+Version: 0.1.0
+Summary: Simple Exploratory Data Analysis tool: column profiling, correlation analysis, and encoding suggestions.
+Author-email: Haim Feldman <haimfeld@gmail.com>
+License: MIT
+Keywords: eda,data-analysis,pandas,correlation,encoding
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=1.5
+Requires-Dist: scipy>=1.9
+Requires-Dist: matplotlib>=3.6
+Requires-Dist: seaborn>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+
+# se-dat
+
+Simple Exploratory Data Analysis tool. Profiles your columns, measures
+associations across numeric/categorical data, and suggests encodings so you can
+move from raw data to a model-ready frame in one call.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+## Features
+
+1. **Column type profiling** — per column: inferred type (`numeric`, `boolean`,
+   `categorical`, `string`, `datetime`, `id`), missing %, unique count,
+   cardinality ratio, and a confidence flag when the inference is ambiguous
+   (numeric-looking strings, `0/1` vs `yes`/`no` booleans, etc.).
+2. **Correlation analysis** — Pearson/Spearman matrices + heatmap
+   (numeric-numeric), Cramér's V (categorical-categorical), correlation ratio
+   eta (numeric-categorical), plus multicollinearity flags above a threshold.
+3. **Encoding suggestions + auto-transform** — detects binary-like strings
+   (`yes`/`no`, `true`/`false`, `0/1`), suggests one-hot for low-cardinality
+   categoricals, and target/ordinal encoding for high-cardinality ones (with a
+   dimensionality warning). Apply suggestions individually or all at once.
+
+## Usage
+
+```python
+import pandas as pd
+import sedat
+
+df = pd.DataFrame({
+    "id": range(500),
+    "age": ...,
+    "income": ["45000", "67000", ...],       # numeric stored as string
+    "smoker": ["yes", "no", ...],            # binary-like
+    "region": ["north", "south", ...],
+    "target": ...,                            # numeric target
+})
+
+report = sedat.EDAReport.create(df, target=df["target"])
+print(report.profile.summary)                 # column type profiling
+print(report.correlations.flagged_pairs)      # multicollinearity warnings
+print(report.encoding_summary)                # suggested encodings
+
+model_ready = report.apply_all_encodings()    # apply every suggestion
+```
+
+### Granular APIs
+
+```python
+profile = sedat.profile_dataframe(df)
+pearson = sedat.numeric_correlation(df, method="pearson")
+cramers = sedat.categorical_correlation(df)       # Cramér's V matrix
+eta = sedat.numeric_categorical_correlation(df)   # correlation ratio matrix
+fig = sedat.correlation_heatmap(pearson, title="Numeric correlations")
+
+plan = sedat.suggest_encodings(df, target=df["target"], cardinality_threshold=10)
+only_smoker = next(s for s in plan.suggestions if s.column == "smoker")
+one_hot_step = only_smoker.apply(df)              # accept one suggestion
+all_encoded = plan.apply_all(df)                  # accept all suggestions
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+python -m pytest
+```

se_dat-0.1.0/src/se_dat.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+src/se_dat.egg-info/PKG-INFO
+src/se_dat.egg-info/SOURCES.txt
+src/se_dat.egg-info/dependency_links.txt
+src/se_dat.egg-info/requires.txt
+src/se_dat.egg-info/top_level.txt
+src/sedat/__init__.py
+src/sedat/correlations.py
+src/sedat/encoding.py
+src/sedat/profile.py
+src/sedat/report.py
+src/sedat/types.py
+tests/test_se_dat.py

se_dat-0.1.0/src/se_dat.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

se_dat-0.1.0/src/se_dat.egg-info/requires.txt

@@ -0,0 +1,9 @@
+numpy>=1.23
+pandas>=1.5
+scipy>=1.9
+matplotlib>=3.6
+seaborn>=0.12
+
+[dev]
+pytest>=7
+pytest-cov

se_dat-0.1.0/src/se_dat.egg-info/top_level.txt

@@ -0,0 +1 @@
+sedat

se_dat-0.1.0/src/sedat/__init__.py

@@ -0,0 +1,51 @@
+"""se-dat: Simple Exploratory Data Analysis tool.
+
+Provides column type profiling, correlation analysis (numeric, categorical and
+mixed) and encoding suggestions with automatic transforms.
+"""
+
+from __future__ import annotations
+
+from .correlations import (
+    CorrelationReport,
+    categorical_correlation,
+    correlation_heatmap,
+    correlation_ratio,
+    correlation_report,
+    cramers_v,
+    numeric_categorical_correlation,
+    numeric_correlation,
+)
+from .encoding import (
+    EncodingPlan,
+    EncodingSuggestion,
+    apply_encodings,
+    suggest_encodings,
+)
+from .profile import ColumnProfile, DataFrameProfile, profile_column, profile_dataframe
+from .report import EDAReport
+from .types import infer_column_type, is_binary_like
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ColumnProfile",
+    "CorrelationReport",
+    "DataFrameProfile",
+    "EDAReport",
+    "EncodingPlan",
+    "EncodingSuggestion",
+    "apply_encodings",
+    "categorical_correlation",
+    "correlation_heatmap",
+    "correlation_ratio",
+    "correlation_report",
+    "cramers_v",
+    "infer_column_type",
+    "is_binary_like",
+    "numeric_categorical_correlation",
+    "numeric_correlation",
+    "profile_column",
+    "profile_dataframe",
+    "suggest_encodings",
+]

se_dat-0.1.0/src/sedat/correlations.py

@@ -0,0 +1,227 @@
+"""Correlation analysis.
+
+Numeric-numeric: Pearson/Spearman matrices plus a heatmap.
+Categorical-categorical: Cramer's V.
+Numeric-categorical: correlation ratio (eta), an ANOVA-style association measure.
+
+Highly correlated pairs above a threshold are flagged as potential
+multicollinearity.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterator
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2_contingency
+
+from .types import binary_mapping
+
+CORRELATION_TYPES = ("numeric", "boolean", "categorical")
+
+
+def _numeric_df(df: pd.DataFrame) -> pd.DataFrame:
+    return df.select_dtypes(include=[np.number])
+
+
+def _categorical_df(df: pd.DataFrame) -> pd.DataFrame:
+    picked = []
+    for col in df.columns:
+        dtype = df[col].dtype
+        if (
+            pd.api.types.is_bool_dtype(dtype)
+            or isinstance(dtype, pd.CategoricalDtype)
+            or dtype == object
+            or isinstance(dtype, pd.StringDtype)
+        ):
+            picked.append(col)
+    return df[picked]
+
+
+def numeric_correlation(df: pd.DataFrame, method: str = "pearson") -> pd.DataFrame:
+    """Pearson or Spearman correlation matrix over numeric columns."""
+    num = _numeric_df(df)
+    if num.shape[1] < 2:
+        return pd.DataFrame()
+    return num.corr(method=method)
+
+
+def cramers_v(a: pd.Series, b: pd.Series, correction: bool = True) -> float:
+    """Cramer's V association between two categorical series, in [0, 1]."""
+    if a.name is not None and b.name is not None and a.name == b.name:
+        return 1.0
+    cross = pd.crosstab(a.astype(str), b.astype(str))
+    if min(cross.shape) < 2:
+        return float("nan")
+    chi2, _, _, _ = chi2_contingency(cross, correction=correction)
+    n = int(cross.to_numpy().sum())
+    phi2 = chi2 / n
+    r, k = cross.shape
+    if correction:
+        phi2 = max(0.0, phi2 - ((k - 1) * (r - 1)) / (n - 1))
+        r = max(0, r - ((r - 1) ** 2) / (n - 1))
+        k = max(0, k - ((k - 1) ** 2) / (n - 1))
+    denom = min(k - 1, r - 1)
+    if denom <= 0:
+        return float("nan")
+    return float(np.sqrt(phi2 / denom))
+
+
+def categorical_correlation(df: pd.DataFrame) -> pd.DataFrame:
+    """Cramer's V matrix over categorical/boolean columns."""
+    cats = _categorical_df(df)
+    if cats.shape[1] < 2:
+        return pd.DataFrame()
+    mat = pd.DataFrame(index=cats.columns, columns=cats.columns, dtype=float)
+    for i, a in enumerate(cats.columns):
+        for j, b in enumerate(cats.columns):
+            if i <= j:
+                continue
+            v = cramers_v(cats[a], cats[b])
+            mat.loc[a, b] = v
+            mat.loc[b, a] = v
+    for i in range(mat.shape[0]):
+        mat.iat[i, i] = 1.0
+    return mat
+
+
+def correlation_ratio(numeric: pd.Series, categorical: pd.Series) -> float:
+    """Eta correlation ratio between one numeric and one categorical series."""
+    y = pd.to_numeric(numeric, errors="coerce")
+    g = categorical.astype(str)
+    d = pd.DataFrame({"y": y, "g": g}).dropna()
+    if len(d) < 2 or d["g"].nunique() < 2:
+        return float("nan")
+    grand_mean = float(d["y"].mean())
+    group = d.groupby("g")["y"].agg(["mean", "count"])
+    ss_between = float(((group["mean"] - grand_mean) ** 2 * group["count"]).sum())
+    ss_total = float(((d["y"] - grand_mean) ** 2).sum())
+    if ss_total <= 0:
+        return float("nan")
+    return float(np.sqrt(ss_between / ss_total))
+
+
+def numeric_categorical_correlation(df: pd.DataFrame) -> pd.DataFrame:
+    """Eta matrix with numeric columns as rows and categorical columns as cols."""
+    num = _numeric_df(df)
+    cats = _categorical_df(df)
+    if num.shape[1] < 1 or cats.shape[1] < 1:
+        return pd.DataFrame()
+    rows = []
+    for n in num.columns:
+        for c in cats.columns:
+            rows.append({"numeric": n, "categorical": c, "eta": correlation_ratio(num[n], cats[c])})
+    if not rows:
+        return pd.DataFrame()
+    return pd.DataFrame(rows).pivot(index="numeric", columns="categorical", values="eta")
+
+
+def _upper_pairs(mat: pd.DataFrame) -> Iterator[tuple[str, str, float]]:
+    cols = list(mat.columns)
+    for i in range(len(cols)):
+        for j in range(i + 1, len(cols)):
+            v = mat.iat[i, j]
+            if np.isnan(v):
+                continue
+            yield cols[i], cols[j], float(v)
+
+
+@dataclass
+class CorrelationReport:
+    numeric: pd.DataFrame
+    numeric_spearman: pd.DataFrame
+    categorical: pd.DataFrame
+    numeric_categorical: pd.DataFrame
+    method_numeric: str = "pearson"
+    threshold: float = 0.7
+
+    @property
+    def summary(self) -> pd.DataFrame:
+        rows = []
+        for mat, kind, method in (
+            (self.numeric, "numeric", self.method_numeric),
+            (self.numeric_spearman, "numeric", "spearman"),
+            (self.categorical, "categorical", "cramers_v"),
+            (self.numeric_categorical, "numeric-categorical", "eta"),
+        ):
+            if mat.empty:
+                continue
+            if kind == "numeric-categorical":
+                for num_col in mat.index:
+                    for cat_col in mat.columns:
+                        v = mat.loc[num_col, cat_col]
+                        if np.isnan(v):
+                            continue
+                        rows.append(
+                            {
+                                "column_a": num_col,
+                                "column_b": cat_col,
+                                "kind": kind,
+                                "method": method,
+                                "value": round(float(v), 4),
+                                "flagged": abs(float(v)) >= self.threshold,
+                            }
+                        )
+            else:
+                for a, b, v in _upper_pairs(mat):
+                    rows.append(
+                        {
+                            "column_a": a,
+                            "column_b": b,
+                            "kind": kind,
+                            "method": method,
+                            "value": round(v, 4),
+                            "flagged": abs(v) >= self.threshold,
+                        }
+                    )
+        return pd.DataFrame(rows)
+
+    @property
+    def flagged_pairs(self) -> pd.DataFrame:
+        s = self.summary
+        return s[s["flagged"]] if not s.empty else s
+
+
+def correlation_report(
+    df: pd.DataFrame,
+    method: str = "pearson",
+    threshold: float = 0.7,
+) -> CorrelationReport:
+    return CorrelationReport(
+        numeric=numeric_correlation(df, method=method),
+        numeric_spearman=numeric_correlation(df, method="spearman"),
+        categorical=categorical_correlation(df),
+        numeric_categorical=numeric_categorical_correlation(df),
+        method_numeric=method,
+        threshold=threshold,
+    )
+
+
+def _binary_numeric_map(df: pd.DataFrame) -> dict[str, dict[str, int]]:
+    return {
+        col: mapping
+        for col in df.columns
+        if (mapping := binary_mapping(df[col])) is not None
+    }
+
+
+def encode_binary_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Encode binary-like object columns to 0/1 numeric for correlation use."""
+    out = df.copy()
+    for col, mapping in _binary_numeric_map(df).items():
+        out[col] = out[col].astype(str).str.strip().str.lower().map(mapping)
+    return out
+
+
+def correlation_heatmap(matrix: pd.DataFrame, title: str = "Correlation matrix") -> object:
+    """Plot a heatmap for a correlation matrix; returns a matplotlib Figure."""
+    import matplotlib.pyplot as plt
+    import seaborn as sns
+
+    fig, ax = plt.subplots(figsize=(max(6, matrix.shape[1] * 0.7), max(5, matrix.shape[0] * 0.6)))
+    sns.heatmap(matrix, annot=True, fmt=".2f", cmap="coolwarm", vmin=-1, vmax=1, ax=ax)
+    ax.set_title(title)
+    fig.tight_layout()
+    return fig

se_dat-0.1.0/src/sedat/encoding.py

@@ -0,0 +1,237 @@
+"""Encoding suggestions and automatic transforms.
+
+Detects binary-like string columns ("yes"/"no", "true"/"false", 0/1) and offers
+an .encode()-style 0/1 map. Suggests one-hot encoding for low-cardinality
+categoricals and warns/suggests target or ordinal encoding for high-cardinality
+ones to avoid a dimensionality explosion.
+
+Suggestions can be applied individually (``suggestion.apply``) or all at once
+(``EncodingPlan.apply_all``).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+from .profile import profile_dataframe
+from .types import binary_mapping
+
+BINARY_STRATEGY = "binary_encode"
+ONE_HOT_STRATEGY = "one_hot"
+ORDINAL_STRATEGY = "ordinal_encode"
+TARGET_STRATEGY = "target_encode"
+NONE_STRATEGY = "none"
+
+DEFAULT_CARDINALITY_THRESHOLD = 10
+
+
+@dataclass
+class EncodingSuggestion:
+    column: str
+    current_type: str
+    suggested_type: str
+    strategy: str
+    cardinality: int
+    rationale: str
+    categories: list[str] = field(default_factory=list)
+    target_means: dict[str, float] = field(default_factory=dict)
+
+    def apply(self, df: pd.DataFrame, target: pd.Series | None = None) -> pd.DataFrame:
+        out = df.copy()
+        col = self.column
+        if col not in out.columns:
+            return out
+        if self.strategy == NONE_STRATEGY:
+            return out
+        if self.strategy == BINARY_STRATEGY:
+            mapping = binary_mapping(out[col])
+            if mapping is None:
+                ordered = sorted(out[col].dropna().unique())
+                mapping = {v: i for i, v in enumerate(ordered)}
+            out[col] = out[col].astype(str).str.strip().str.lower().map(mapping)
+        elif self.strategy == ONE_HOT_STRATEGY:
+            dummies = pd.get_dummies(out[col], prefix=col, dtype=int)
+            out = out.drop(columns=[col])
+            out = pd.concat([out, dummies], axis=1)
+        elif self.strategy == ORDINAL_STRATEGY:
+            ordered = sorted(out[col].dropna().astype(str).unique())
+            out[col] = out[col].astype(str).map({v: i for i, v in enumerate(ordered)})
+        elif self.strategy == TARGET_STRATEGY:
+            if target is not None:
+                means = target.groupby(out[col].astype(str)).transform("mean")
+                out[col] = pd.to_numeric(means, errors="coerce")
+            elif self.target_means:
+                out[col] = out[col].astype(str).map(self.target_means)
+        return out
+
+
+@dataclass
+class EncodingPlan:
+    suggestions: list[EncodingSuggestion]
+    cardinality_threshold: int = DEFAULT_CARDINALITY_THRESHOLD
+    target: pd.Series | None = None
+
+    @property
+    def summary(self) -> pd.DataFrame:
+        rows = [
+            {
+                "column": s.column,
+                "current_type": s.current_type,
+                "suggested_type": s.suggested_type,
+                "strategy": s.strategy,
+                "cardinality": s.cardinality,
+                "rationale": s.rationale,
+            }
+            for s in self.suggestions
+        ]
+        return pd.DataFrame(rows)
+
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        return self.apply_all(df)
+
+    def apply_all(self, df: pd.DataFrame, target: pd.Series | None = None) -> pd.DataFrame:
+        result = df.copy()
+        tgt = target if target is not None else self.target
+        for suggestion in self.suggestions:
+            result = suggestion.apply(result, target=tgt)
+        return result
+
+    def __repr__(self) -> str:
+        return repr(self.summary)
+
+
+def _warn(msg: str) -> None:
+    import warnings
+
+    warnings.warn(msg, UserWarning, stacklevel=3)
+
+
+def suggest_encodings(
+    df: pd.DataFrame,
+    target: pd.Series | None = None,
+    cardinality_threshold: int = DEFAULT_CARDINALITY_THRESHOLD,
+) -> EncodingPlan:
+    """Build an :class:`EncodingPlan` of encoding suggestions for ``df``.
+
+    Parameters
+    ----------
+    df:
+        Input DataFrame.
+    target:
+        Optional numeric target Series used for target-encoding suggestions on
+        high-cardinality categoricals. The Series index must match ``df``.
+    cardinality_threshold:
+        Categoricals with at most this many distinct values get a one-hot
+        suggestion; above it they get target/ordinal encoding.
+    """
+    profile = profile_dataframe(df)
+    suggestions: list[EncodingSuggestion] = []
+
+    for col_profile in profile.columns:
+        col = col_profile.name
+        series = df[col]
+        cardinality = col_profile.n_unique
+        current_type = col_profile.inferred_type
+
+        if current_type == "boolean":
+            if series.dtype == object or isinstance(series.dtype, pd.StringDtype):
+                suggestions.append(
+                    EncodingSuggestion(
+                        column=col,
+                        current_type=current_type,
+                        suggested_type="boolean (0/1)",
+                        strategy=BINARY_STRATEGY,
+                        cardinality=cardinality,
+                        rationale="binary-like strings map cleanly to 0/1",
+                        categories=sorted(series.dropna().unique().tolist()),
+                    )
+                )
+            continue
+
+        if current_type not in ("categorical",):
+            continue
+
+        if cardinality == 0:
+            continue
+
+        if cardinality == 2:
+            suggestions.append(
+                EncodingSuggestion(
+                    column=col,
+                    current_type=current_type,
+                    suggested_type="boolean (0/1)",
+                    strategy=BINARY_STRATEGY,
+                    cardinality=cardinality,
+                    rationale="two distinct values: binary encode",
+                    categories=sorted(series.dropna().unique().tolist()),
+                )
+            )
+        elif cardinality <= cardinality_threshold:
+            suggestions.append(
+                EncodingSuggestion(
+                    column=col,
+                    current_type=current_type,
+                    suggested_type="one-hot (dummy) columns",
+                    strategy=ONE_HOT_STRATEGY,
+                    cardinality=cardinality,
+                    rationale=f"low-cardinality categorical (k={cardinality})",
+                    categories=sorted(series.dropna().unique().tolist()),
+                )
+            )
+        else:
+            if target is not None:
+                suggestions.append(
+                    EncodingSuggestion(
+                        column=col,
+                        current_type=current_type,
+                        suggested_type="target-encoded numeric",
+                        strategy=TARGET_STRATEGY,
+                        cardinality=cardinality,
+                        rationale=(
+                            f"high-cardinality categorical (k={cardinality}): "
+                            "one-hot would explode dimensionality"
+                        ),
+                        categories=sorted(series.dropna().unique().tolist()),
+                        target_means={
+                            str(k): float(v)
+                            for k, v in target.groupby(series.astype(str)).mean().items()
+                        },
+                    )
+                )
+            else:
+                _warn(
+                    f"Column '{col}' has high cardinality ({cardinality}). "
+                    "Pass a `target` Series to enable target encoding; "
+                    "falling back to ordinal encoding."
+                )
+                suggestions.append(
+                    EncodingSuggestion(
+                        column=col,
+                        current_type=current_type,
+                        suggested_type="ordinal-encoded numeric",
+                        strategy=ORDINAL_STRATEGY,
+                        cardinality=cardinality,
+                        rationale=(
+                            f"high-cardinality categorical (k={cardinality}): "
+                            "one-hot would explode dimensionality"
+                        ),
+                        categories=sorted(series.dropna().astype(str).unique().tolist()),
+                    )
+                )
+
+    return EncodingPlan(suggestions=suggestions, cardinality_threshold=cardinality_threshold, target=target)
+
+
+def apply_encodings(
+    df: pd.DataFrame,
+    plan: EncodingPlan | None = None,
+    target: pd.Series | None = None,
+    cardinality_threshold: int = DEFAULT_CARDINALITY_THRESHOLD,
+) -> pd.DataFrame:
+    """Shortcut: build a plan (if not given) and apply every suggestion."""
+    if plan is None:
+        plan = suggest_encodings(df, target=target, cardinality_threshold=cardinality_threshold)
+    return plan.apply_all(df, target=target)

se_dat-0.1.0/src/sedat/profile.py

@@ -0,0 +1,75 @@
+"""Column type profiling.
+
+Produces a per-column profile with the inferred semantic type, basic stats
+(missing %, unique count, cardinality ratio) and a confidence flag when the
+inference is ambiguous.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import pandas as pd
+
+from .types import infer_column_type
+
+
+@dataclass
+class ColumnProfile:
+    name: str
+    dtype: str
+    inferred_type: str
+    n_non_null: int
+    n_unique: int
+    missing_pct: float
+    cardinality_ratio: float
+    confidence: str
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def ambiguous(self) -> bool:
+        return self.confidence in ("low", "medium")
+
+
+@dataclass
+class DataFrameProfile:
+    columns: list[ColumnProfile]
+
+    @property
+    def summary(self) -> pd.DataFrame:
+        rows = [
+            {
+                "column": c.name,
+                "dtype": c.dtype,
+                "inferred_type": c.inferred_type,
+                "missing_pct": c.missing_pct,
+                "n_unique": c.n_unique,
+                "cardinality_ratio": c.cardinality_ratio,
+                "confidence": c.confidence,
+                "notes": "; ".join(c.notes),
+            }
+            for c in self.columns
+        ]
+        return pd.DataFrame(rows).set_index("column")
+
+    def __repr__(self) -> str:
+        return repr(self.summary)
+
+
+def profile_column(series: pd.Series) -> ColumnProfile:
+    info = infer_column_type(series)
+    return ColumnProfile(
+        name=str(series.name),
+        dtype=str(series.dtype),
+        inferred_type=info["inferred_type"],
+        n_non_null=info["n_non_null"],
+        n_unique=info["n_unique"],
+        missing_pct=info["missing_pct"],
+        cardinality_ratio=info["cardinality_ratio"],
+        confidence=info["confidence"],
+        notes=info["notes"],
+    )
+
+
+def profile_dataframe(df: pd.DataFrame) -> DataFrameProfile:
+    return DataFrameProfile([profile_column(df[col]) for col in df.columns])

se_dat-0.1.0/src/sedat/report.py

@@ -0,0 +1,72 @@
+"""High-level facade combining profiling, correlations and encoding."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+from .correlations import CorrelationReport, correlation_report
+from .encoding import (
+    DEFAULT_CARDINALITY_THRESHOLD,
+    EncodingPlan,
+    suggest_encodings,
+)
+from .profile import DataFrameProfile, profile_dataframe
+
+
+@dataclass
+class EDAReport:
+    """One object summarizing profiling, correlations and encodings for a frame."""
+
+    df: pd.DataFrame
+    profile: DataFrameProfile
+    correlations: CorrelationReport
+    encodings: EncodingPlan
+    corr_threshold: float = 0.7
+    cardinality_threshold: int = DEFAULT_CARDINALITY_THRESHOLD
+
+    @classmethod
+    def create(
+        cls,
+        df: pd.DataFrame,
+        corr_threshold: float = 0.7,
+        cardinality_threshold: int = DEFAULT_CARDINALITY_THRESHOLD,
+        target: pd.Series | None = None,
+    ) -> "EDAReport":
+        return cls(
+            df=df,
+            profile=profile_dataframe(df),
+            correlations=correlation_report(df, threshold=corr_threshold),
+            encodings=suggest_encodings(
+                df, target=target, cardinality_threshold=cardinality_threshold
+            ),
+            corr_threshold=corr_threshold,
+            cardinality_threshold=cardinality_threshold,
+        )
+
+    @property
+    def summary(self) -> pd.DataFrame:
+        return self.profile.summary
+
+    @property
+    def correlation_summary(self) -> pd.DataFrame:
+        return self.correlations.summary
+
+    @property
+    def encoding_summary(self) -> pd.DataFrame:
+        return self.encodings.summary
+
+    def apply_all_encodings(self, target: pd.Series | None = None) -> pd.DataFrame:
+        return self.encodings.apply_all(self.df, target=target)
+
+    def __repr__(self) -> str:
+        lines = [repr(self.profile)]
+        corr = self.correlations.summary
+        if not corr.empty:
+            flagged = corr[corr["flagged"]]
+            if not flagged.empty:
+                lines.append("\nFlagged highly-correlated pairs:")
+                lines.append(repr(flagged))
+        lines.append("\n" + repr(self.encodings))
+        return "\n".join(lines)

se_dat-0.1.0/src/sedat/types.py

@@ -0,0 +1,152 @@
+"""Column type inference helpers.
+
+Infers a semantic type for each column: numeric, boolean, categorical,
+string/text, datetime or id (high-cardinality unique values), and reports a
+confidence level plus explanatory notes when the inference is ambiguous.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+BINARY_VALUE_SETS: dict[str, set[str]] = {
+    "yes/no": {"yes", "no", "y", "n"},
+    "true/false": {"true", "false", "t", "f"},
+    "0/1": {"0", "1"},
+}
+
+_BINARY_MAPS: dict[frozenset[str], dict[str, int]] = {
+    frozenset({"yes", "no", "y", "n"}): {"yes": 1, "no": 0, "y": 1, "n": 0},
+    frozenset({"true", "false", "t", "f"}): {"true": 1, "false": 0, "t": 1, "f": 0},
+    frozenset({"0", "1"}): {"0": 1, "1": 0},
+}
+
+
+def binary_mapping(series: pd.Series) -> dict[str, int] | None:
+    """Return a {value: 0/1} mapping if ``series`` is binary-like, else None."""
+    unique = set(series.dropna().astype(str).str.strip().str.lower().unique())
+    if not unique:
+        return None
+    for allowed, mapping in _BINARY_MAPS.items():
+        if unique.issubset(allowed):
+            return {v: mapping[v] for v in unique}
+    return None
+
+
+def is_binary_like(series: pd.Series) -> bool:
+    return binary_mapping(series) is not None
+
+
+def _parseable_as_datetime(series: pd.Series, threshold: float = 0.9) -> bool:
+    sample = series.dropna().astype(str).head(200)
+    if sample.empty:
+        return False
+    try:
+        with pd.option_context("mode.chained_assignment", None):
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                parsed = pd.to_datetime(sample, errors="coerce")
+    except (TypeError, ValueError, OverflowError):
+        return False
+    return float(parsed.notna().mean()) >= threshold
+
+
+def _numeric_like_string(series: pd.Series, threshold: float = 0.9) -> bool:
+    sample = series.dropna().head(500)
+    if sample.empty:
+        return False
+    coerced = pd.to_numeric(sample, errors="coerce")
+    return float(coerced.notna().mean()) >= threshold
+
+
+def _looks_like_identifier(series: pd.Series, threshold: float = 0.95) -> bool:
+    sample = series.dropna().astype(str).head(500)
+    if sample.empty:
+        return False
+    no_whitespace = ~sample.str.contains(r"\s", regex=True)
+    return float(no_whitespace.mean()) >= threshold
+
+
+def _basic_stats(series: pd.Series) -> dict[str, float | int]:
+    n_non_null = int(series.notna().sum())
+    n_rows = len(series)
+    n_unique = int(series.nunique(dropna=True))
+    missing_pct = 0.0 if n_rows == 0 else (n_rows - n_non_null) / n_rows * 100
+    cardinality_ratio = n_unique / n_non_null if n_non_null else 0.0
+    return {
+        "n_non_null": n_non_null,
+        "n_unique": n_unique,
+        "missing_pct": round(missing_pct, 2),
+        "cardinality_ratio": round(cardinality_ratio, 4),
+    }
+
+
+def infer_column_type(series: pd.Series) -> dict[str, Any]:
+    """Infer the semantic type of a single ``Series``.
+
+    Returns a dict with keys: inferred_type, confidence, notes, plus the basic
+    stats (n_non_null, n_unique, missing_pct, cardinality_ratio).
+    """
+    stats = _basic_stats(series)
+    notes: list[str] = []
+    inferred_type: str
+    confidence: str = "high"
+
+    if isinstance(series.dtype, pd.CategoricalDtype):
+        inferred_type = "categorical"
+    elif pd.api.types.is_datetime64_any_dtype(series.dtype):
+        inferred_type = "datetime"
+    elif pd.api.types.is_bool_dtype(series.dtype):
+        inferred_type = "boolean"
+    elif pd.api.types.is_numeric_dtype(series.dtype):
+        mapping = binary_mapping(series)
+        if mapping is not None and stats["n_unique"] <= 2:
+            inferred_type = "boolean"
+            confidence = "medium"
+            notes.append("numeric 0/1 column: ambiguous boolean vs numeric")
+        elif (
+            pd.api.types.is_integer_dtype(series.dtype)
+            and stats["cardinality_ratio"] >= 0.95
+        ):
+            inferred_type = "id"
+            notes.append("high-cardinality unique integer values")
+        else:
+            inferred_type = "numeric"
+    else:
+        if stats["n_non_null"] == 0:
+            inferred_type = "string"
+            confidence = "low"
+            notes.append("empty column")
+        else:
+            mapping = binary_mapping(series)
+            if mapping is not None:
+                inferred_type = "boolean"
+                confidence = "medium"
+                notes.append(f"stored as strings ({', '.join(sorted(mapping))})")
+            elif _parseable_as_datetime(series):
+                inferred_type = "datetime"
+                confidence = "medium"
+                notes.append("parsed from object dtype")
+            elif _numeric_like_string(series):
+                inferred_type = "numeric"
+                confidence = "low"
+                notes.append("numeric-looking values stored as string")
+            elif stats["cardinality_ratio"] >= 0.95:
+                if _looks_like_identifier(series):
+                    inferred_type = "id"
+                    notes.append("high-cardinality unique values")
+                else:
+                    inferred_type = "string"
+                    notes.append("high-cardinality free text")
+            elif stats["cardinality_ratio"] >= 0.5:
+                inferred_type = "string"
+                notes.append("many unique values")
+            else:
+                inferred_type = "categorical"
+
+    return {**stats, "inferred_type": inferred_type, "confidence": confidence, "notes": notes}

se_dat-0.1.0/tests/test_se_dat.py

@@ -0,0 +1,176 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import sedat
+
+
+def make_df(n=1000, seed=0):
+    rng = np.random.default_rng(seed)
+    df = pd.DataFrame(
+        {
+            "id": np.arange(n),
+            "age": rng.integers(18, 80, n),
+            "height": rng.normal(170, 10, n).round(2),
+            "income": [str(x) for x in rng.integers(30000, 120000, n)],
+            "gender": rng.choice(["M", "F"], n),
+            "smoker": rng.choice(["yes", "no"], n),
+            "active": rng.choice([True, False], n),
+            "flag01": rng.integers(0, 2, n),
+            "score_cat": rng.choice(["low", "med", "high"], n),
+            "free_text": [f"user comment number {i}" for i in range(n)],
+            "joined": pd.to_datetime("2020-01-01")
+            + pd.to_timedelta(rng.integers(0, 365 * 5, n), unit="D"),
+            "large_card": rng.choice([f"sku_{i}" for i in range(50)], n),
+            "target": rng.normal(0, 1, n),
+        }
+    )
+    df.loc[rng.choice(n, 25, replace=False), "age"] = np.nan
+    return df
+
+
+def type_map(profile):
+    return {c.name: c.inferred_type for c in profile.columns}
+
+
+def test_profile_infers_types():
+    df = make_df()
+    profile = sedat.profile_dataframe(df)
+    types = type_map(profile)
+    assert types["id"] == "id"
+    assert types["age"] == "numeric"
+    assert types["height"] == "numeric"
+    assert types["income"] == "numeric"
+    assert types["gender"] == "categorical"
+    assert types["smoker"] == "boolean"
+    assert types["active"] == "boolean"
+    assert types["flag01"] == "boolean"
+    assert types["free_text"] == "string"
+    assert types["joined"] == "datetime"
+    assert types["target"] == "numeric"
+
+
+def test_profile_confidence_flags():
+    df = make_df()
+    profile = sedat.profile_dataframe(df)
+    by_name = {c.name: c for c in profile.columns}
+    assert by_name["income"].confidence == "low"
+    assert by_name["income"].ambiguous
+    assert by_name["smoker"].confidence == "medium"
+    assert by_name["active"].confidence == "high"
+    assert "stored as string" in by_name["income"].notes[0]
+
+
+def test_profile_basic_stats():
+    df = make_df()
+    profile = sedat.profile_dataframe(df)
+    by_name = {c.name: c for c in profile.columns}
+    assert by_name["age"].missing_pct == pytest.approx(2.5, abs=0.05)
+    assert by_name["id"].cardinality_ratio == pytest.approx(1.0)
+    assert by_name["gender"].n_unique == 2
+
+
+def test_numeric_correlation():
+    df = make_df()
+    pearson = sedat.numeric_correlation(df)
+    assert {"age", "height"} <= set(pearson.columns)
+    assert abs(pearson.loc["height", "age"]) < 0.3
+    spearman = sedat.numeric_correlation(df, method="spearman")
+    assert "age" in spearman.index
+
+
+def test_cramers_v_range():
+    df = make_df()
+    mat = sedat.categorical_correlation(df)
+    assert {"gender", "smoker", "score_cat"} <= set(mat.columns)
+    assert mat.loc["gender", "gender"] == 1.0
+    for col in mat.columns:
+        v = mat.loc["gender", col]
+        assert np.isnan(v) or 0.0 <= v <= 1.0
+
+
+def test_eta_range():
+    df = make_df()
+    mat = sedat.numeric_categorical_correlation(df)
+    assert "gender" in mat.columns
+    assert "age" in mat.index
+    v = mat.loc["age", "gender"]
+    assert np.isnan(v) or 0.0 <= v <= 1.0
+
+
+def test_correlation_report_flags():
+    df = make_df()
+    df2 = df.copy()
+    df2["height2"] = df["height"] * 1.5 + 3
+    report = sedat.correlation_report(df2, threshold=0.95)
+    flagged = report.flagged_pairs
+    assert not flagged.empty
+    pair = flagged[(flagged["column_a"] == "height") & (flagged["column_b"] == "height2")]
+    assert not pair.empty
+    assert pair["value"].iloc[0] == pytest.approx(1.0, abs=0.01)
+
+
+def test_encode_binary_suggestion():
+    df = make_df()
+    plan = sedat.suggest_encodings(df)
+    by_col = {s.column: s for s in plan.suggestions}
+    assert by_col["smoker"].strategy == "binary_encode"
+    assert "active" not in by_col
+    assert by_col["gender"].strategy == "binary_encode"
+    assert by_col["score_cat"].strategy == "one_hot"
+    assert by_col["large_card"].strategy == "ordinal_encode"
+
+
+def test_apply_encodings_all():
+    df = make_df()
+    plan = sedat.suggest_encodings(df)
+    out = plan.apply_all(df)
+    object_cols = [c for c in out.columns if out[c].dtype == object]
+    assert object_cols == []
+    assert out["smoker"].isin([0, 1]).all()
+    assert out["score_cat_low"].dtype == int
+    assert "large_card" in out.columns and out["large_card"].dtype in (int, float)
+
+
+def test_target_encoding_high_cardinality():
+    df = make_df()
+    plan = sedat.suggest_encodings(df, target=df["target"])
+    by_col = {s.column: s for s in plan.suggestions}
+    assert by_col["large_card"].strategy == "target_encode"
+    out = plan.apply_all(df)
+    assert out["large_card"].dtype == float
+
+
+def test_apply_single_suggestion():
+    df = make_df()
+    plan = sedat.suggest_encodings(df)
+    smoker = next(s for s in plan.suggestions if s.column == "smoker")
+    out = smoker.apply(df)
+    assert out["smoker"].isin([0, 1]).all()
+
+
+def test_report_end_to_end():
+    df = make_df()
+    report = sedat.EDAReport.create(df, target=df["target"])
+    assert isinstance(report.profile, sedat.DataFrameProfile)
+    assert report.encoding_summary is not None
+    out = report.apply_all_encodings()
+    assert object not in out.dtypes.values
+    assert "score_cat" not in out.columns
+    assert "score_cat_low" in out.columns
+
+
+def test_binary_mapping_helpers():
+    s = pd.Series(["yes", "no", "yes"])
+    assert sedat.is_binary_like(s)
+    s2 = pd.Series([0, 1, 1])
+    assert sedat.is_binary_like(s2)
+    s3 = pd.Series(["alpha", "beta", "gamma"])
+    assert not sedat.is_binary_like(s3)
+
+
+def test_infer_column_type_direct():
+    s = pd.Series(["1.5", "2.5", "3.0"])
+    info = sedat.infer_column_type(s)
+    assert info["inferred_type"] == "numeric"
+    assert info["confidence"] == "low"