dataforge-ml 2.0.0__tar.gz → 2.0.2__tar.gz

{dataforge_ml-2.0.0/src/dataforge_ml.egg-info → dataforge_ml-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataforge-ml
-Version: 2.0.0
+Version: 2.0.2
 Summary: A automated feature engineering and designing pipeline library
 License: MIT
 Classifier: License :: OSI Approved :: MIT License

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataforge-ml"
-version = "2.0.0"
+version = "2.0.2"
 description = "A automated feature engineering and designing pipeline library"
 readme = "README.md"
 requires-python = ">3.10"

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/src/dataforge_ml/profiling/_categorical.py

@@ -11,9 +11,9 @@ Per-column metrics (opt-in via ProfileConfig.categorical_columns):
   7. Free-text / natural-language flag
         (avg word count >5 OR avg char length >50 OR avg token count >10)
   8. Imbalance metrics
-        – class ratio  (max_freq / min_freq)
-        – Shannon entropy
-        – Gini impurity
+        – dominant class ratio (max_freq / second_max_freq)
+        – normalized Shannon entropy
+        – normalized Gini impurity
 
 Integration
 -----------
@@ -221,24 +221,36 @@ class CategoricalProfiler(ColumnBatchProfiler[CategoricalProfileResult]):
         )
 
         # --- Imbalance metrics ---
-        # Class Ratio -> raw distribution
-        # Entropy -> randomness / information content
-        # Gini -> impurity / misclassification risk
+        # Dominant Class Ratio -> max_freq / second_max_freq
+        # Normalized Entropy -> scaled to [0, 1]
+        # Normalized Gini -> scaled to [0, 1]
         counts = vc["count"].cast(pl.Float64)
         total = float(counts.sum())
-        if total > 0:
+        card = len(counts)
+        if total > 0 and card >= 2:
             probs = counts / total
-            max_freq = float(probs.max())  # type: ignore[arg-type]
-            min_freq = float(probs.min())  # type: ignore[arg-type]
+            
+            max_freq = float(probs[0])
+            second_max_freq = float(probs[1])
 
-            class_ratio = max_freq / min_freq if min_freq > 0 else float("inf")
+            dominant_class_ratio = max_freq / second_max_freq if second_max_freq > 0 else float("inf")
+            
             entropy = float(-(probs * probs.log(base=2)).fill_nan(0.0).sum())
+            normalized_entropy = entropy / math.log2(card)
+            
             gini = float(1.0 - (probs**2).sum())
+            normalized_gini = gini / (1.0 - 1.0 / card)
 
             profile.imbalance = ImbalanceMetrics(
-                class_ratio=class_ratio,
-                shannon_entropy=entropy,
-                gini_impurity=gini,
+                dominant_class_ratio=dominant_class_ratio,
+                normalized_shannon_entropy=normalized_entropy,
+                normalized_gini=normalized_gini,
+            )
+        else:
+            profile.imbalance = ImbalanceMetrics(
+                dominant_class_ratio=None,
+                normalized_shannon_entropy=None,
+                normalized_gini=None,
             )
 
         return vc

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/src/dataforge_ml/profiling/_categorical_config.py

@@ -194,20 +194,20 @@ class ImbalanceMetrics:
 
     Attributes
     ----------
-    class_ratio : float
-        Ratio of the least-frequent to most-frequent class, in [0, 1].
-        A value of 1.0 indicates perfect balance.
-    shannon_entropy : float
-        Shannon entropy of the class distribution (nats).  Higher values
-        indicate more uniform distributions.
-    gini_impurity : float
-        Gini impurity of the class distribution, in [0, 1].  Zero means
-        a single class dominates entirely.
+    dominant_class_ratio : float, optional
+        Ratio of the most-frequent to second-most-frequent class frequency.
+        None when cardinality < 2.
+    normalized_shannon_entropy : float, optional
+        Shannon entropy of the class distribution scaled to [0, 1] by dividing
+        by log2(cardinality). None when cardinality < 2.
+    normalized_gini : float, optional
+        Gini impurity of the class distribution scaled to [0, 1] by dividing
+        by (1 - 1/cardinality). None when cardinality < 2.
     """
 
-    class_ratio: float = 0.0
-    shannon_entropy: float = 0.0
-    gini_impurity: float = 0.0
+    dominant_class_ratio: float | None = None
+    normalized_shannon_entropy: float | None = None
+    normalized_gini: float | None = None
 
     def to_dict(self) -> dict:
         """
@@ -216,12 +216,12 @@ class ImbalanceMetrics:
         Returns
         -------
         dict
-            Keys: ``class_ratio``, ``shannon_entropy``, ``gini_impurity``.
+            Keys: ``dominant_class_ratio``, ``normalized_shannon_entropy``, ``normalized_gini``.
         """
         return {
-            "class_ratio": self.class_ratio,
-            "shannon_entropy": self.shannon_entropy,
-            "gini_impurity": self.gini_impurity,
+            "dominant_class_ratio": self.dominant_class_ratio,
+            "normalized_shannon_entropy": self.normalized_shannon_entropy,
+            "normalized_gini": self.normalized_gini,
         }
 
 

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/src/dataforge_ml/profiling/_config.py

@@ -15,7 +15,9 @@ from typing import Optional, Union
 from ..config import SemanticType, Modality
 from ._missingness_config import (
     ColumnMissingnessProfile,
+    MissingnessFlag,
     MissingnessProfileConfig,
+    MissingSeverity,
     RowMissingnessDistribution,
 )
 from ._correlation_config import (
@@ -27,6 +29,8 @@ from ._categorical_config import (
     CategoricalProfileConfig,
 )
 from ._numeric_config import (
+    NonlinearityTag,
+    NumericFlag,
     NumericStats,
     NumericProfileConfig,
     NonlinearityProfileConfig,
@@ -70,6 +74,121 @@ class TypeFlag(StrEnum):
 AnyStats = Union[NumericStats, CategoricalStats, DatetimeStats, BooleanStats, TextStats]
 
 
+def _format_dict_lines(d: dict, indent: int = 0) -> list[str]:
+    out = []
+    prefix = "  " * indent
+    for k, v in d.items():
+        if isinstance(v, dict):
+            if not v:
+                out.append(f"{prefix}- **{k}**: (empty)")
+            else:
+                out.append(f"{prefix}- **{k}**:")
+                out.extend(_format_dict_lines(v, indent + 1))
+        elif isinstance(v, list):
+            if not v:
+                out.append(f"{prefix}- **{k}**: (empty)")
+            else:
+                out.append(f"{prefix}- **{k}**:")
+                for item in v:
+                    out.append(f"{prefix}  - {item}")
+        else:
+            out.append(f"{prefix}- **{k}**: {v}")
+    return out
+
+
+def _top_n_abs_correlations(
+    matrix: dict[str, dict[str, float]], col_name: str, n: int = 5
+) -> list[tuple[str, float]]:
+    row = matrix.get(col_name, {})
+    pairs = [(other, value) for other, value in row.items() if other != col_name]
+    pairs.sort(key=lambda item: abs(item[1]), reverse=True)
+    return pairs[:n]
+
+
+def _is_clean_column(col: "ColumnProfile") -> bool:
+    missingness = col.missingness
+    if missingness is not None and missingness.flags:
+        return False
+    severity = missingness.severity if missingness is not None else None
+    if severity not in (None, MissingSeverity.Minor):
+        return False
+    if isinstance(col.stats, NumericStats):
+        if col.stats.flags:
+            return False
+        if col.stats.nonlinearity_tag not in (None, NonlinearityTag.Linear):
+            return False
+    return True
+
+
+def _compact_column_detail(
+    col: "ColumnProfile",
+    feature_correlation: Optional[CorrelationProfileResult],
+) -> dict:
+    """
+    Build the per-column dict used in the Flagged Columns detail section.
+
+    Applies the compact-view field rules (ADR-0040) on top of
+    ``ColumnProfile.to_dict()``: drops ``total_rows`` from the missingness
+    subsection and ``histogram`` from the stats subsection, and caps
+    ``top_values`` (present on both ``NumericStats`` and ``CategoricalStats``)
+    to 3 entries. All other fields — including redundant scalar pairs,
+    percentiles, bimodal stats, and ``correlated_with`` — pass through
+    unchanged. When ``feature_correlation`` carries a row for this column,
+    a ``correlations`` entry is added with the top-5 highest absolute
+    Pearson and top-5 highest absolute Spearman correlations (descending
+    by absolute value) in place of the full N×N matrices.
+
+    Parameters
+    ----------
+    col : ColumnProfile
+        The column profile to render.
+    feature_correlation : CorrelationProfileResult or None
+        Dataset-level feature-feature correlation result, if computed.
+
+    Returns
+    -------
+    dict
+        Trimmed dictionary suitable for ``_format_dict_lines``.
+    """
+    data = col.to_dict()
+    missingness = data.get("missingness")
+    if missingness is not None:
+        missingness.pop("total_rows", None)
+    stats = data.get("stats")
+    if stats is not None:
+        stats.pop("histogram", None)
+        if "top_values" in stats:
+            stats["top_values"] = stats["top_values"][:3]
+
+    if feature_correlation is not None:
+        top_pearson = _top_n_abs_correlations(feature_correlation.pearson_matrix, col.name)
+        top_spearman = _top_n_abs_correlations(feature_correlation.spearman_matrix, col.name)
+        if top_pearson or top_spearman:
+            data["correlations"] = {
+                "top_pearson": [f"{name}: {value}" for name, value in top_pearson],
+                "top_spearman": [f"{name}: {value}" for name, value in top_spearman],
+            }
+    return data
+
+
+def _flagged_column_tier(col: "ColumnProfile") -> int:
+    missingness = col.missingness
+    flags = missingness.flags if missingness is not None else []
+    severity = missingness.severity if missingness is not None else None
+
+    if MissingnessFlag.DropCandidate in flags or MissingnessFlag.FullyNull in flags:
+        return 0
+    if severity == MissingSeverity.Severe:
+        return 1
+    if severity == MissingSeverity.High:
+        return 2
+    if severity == MissingSeverity.Moderate:
+        return 3
+    if flags:
+        return 4
+    return 5
+
+
 @dataclass
 class ColumnProfile:
     """
@@ -236,6 +355,194 @@ class StructuralProfileResult:
         """
         return json.dumps(self.to_dict(), indent=indent, default=str)
 
+    def to_markdown(self) -> str:
+        """
+        Produce a compact, human-oriented Markdown view of the profiling result.
+
+        The document contains a Dataset Overview section (scalar
+        dataset-level fields only — ``memory_breakdown`` and
+        ``missingness_matrix`` are omitted), a Column Summary table with one
+        row per column, and a Flagged Columns section with a full detail
+        subsection for every column that exceeds the clean threshold (see
+        ``_is_clean_column``), ordered severity-first then alphabetically
+        (see ``_flagged_column_tier``). Within each flagged column's detail
+        section, ``histogram`` bins and the missingness ``total_rows`` field
+        are dropped, ``top_values`` is capped to 3 entries, and the full
+        Pearson/Spearman correlation matrices are replaced by the top-5
+        highest absolute correlations for that column (see
+        ``_compact_column_detail``); all other fields — including redundant
+        scalar pairs, percentiles, bimodal stats, and ``correlated_with`` —
+        are kept in full. A Target Analysis section follows with the top-5
+        absolute Pearson and Spearman correlations per feature column for
+        each declared target, and a Sentinels section renders
+        ``numeric_sentinels`` / ``string_sentinels`` unchanged. Use
+        ``to_full_markdown()`` for the complete lossless serialization.
+
+        Returns
+        -------
+        str
+            Markdown string containing the Dataset Overview, Column Summary,
+            Flagged Columns, Target Analysis, and Sentinels sections.
+        """
+        lines = ["# Structural Profile Report (Compact)\n"]
+
+        ds = self.dataset
+        lines.append("## Dataset Overview\n")
+        lines.append(f"- **modality**: {ds.modality}")
+        lines.append(f"- **row_count**: {ds.row_count}")
+        lines.append(f"- **column_count**: {ds.column_count}")
+        lines.append(f"- **memory_bytes**: {ds.memory_bytes}")
+        lines.append(f"- **duplicate_count**: {ds.duplicate_count}")
+        lines.append(f"- **duplicate_ratio**: {ds.duplicate_ratio}")
+        lines.append(f"- **overall_sparsity**: {ds.overall_sparsity}")
+        lines.append(f"- **was_chunked**: {ds.was_chunked}")
+        lines.append("- **row_distribution**:")
+        for key, value in ds.row_distribution.to_dict().items():
+            lines.append(f"  - **{key}**: {value}")
+        lines.append("")
+
+        lines.append("## Column Summary\n")
+        lines.append(
+            "| Column | Semantic Type | Missing % | Severity | "
+            "Missingness Flags | Numeric Flags |"
+        )
+        lines.append("|---|---|---|---|---|---|")
+        for col_name, col in self.columns.items():
+            sem_type = col.semantic_type if col.semantic_type else "None"
+            missingness = col.missingness
+            if missingness is not None:
+                missing_str = f"{missingness.effective_null_ratio * 100:.2f}%"
+                severity = missingness.severity if missingness.severity else "None"
+                missingness_flags = (
+                    ", ".join(str(f) for f in missingness.flags)
+                    if missingness.flags
+                    else "None"
+                )
+            else:
+                missing_str = "0.00%"
+                severity = "None"
+                missingness_flags = "None"
+            numeric_flags = "None"
+            if isinstance(col.stats, NumericStats) and col.stats.flags:
+                numeric_flags = ", ".join(str(f) for f in col.stats.flags)
+            lines.append(
+                f"| `{col_name}` | {sem_type} | {missing_str} | {severity} | "
+                f"{missingness_flags} | {numeric_flags} |"
+            )
+        lines.append("")
+
+        lines.append("## Flagged Columns\n")
+        flagged_columns = [
+            col for col in self.columns.values() if not _is_clean_column(col)
+        ]
+        flagged_columns.sort(key=lambda col: (_flagged_column_tier(col), col.name))
+        for col in flagged_columns:
+            lines.append(f"### `{col.name}`\n")
+            lines.extend(
+                _format_dict_lines(
+                    _compact_column_detail(col, self.dataset.feature_correlation)
+                )
+            )
+            lines.append("")
+
+        if self.dataset.target_correlations:
+            lines.append("## Target Analysis\n")
+            for target_name, corr in self.dataset.target_correlations.items():
+                lines.append(f"### Target: `{target_name}`\n")
+                feature_cols = [
+                    c for c in corr.analysed_numeric_columns if c != target_name
+                ]
+                for feat in feature_cols:
+                    top_pearson = _top_n_abs_correlations(corr.pearson_matrix, feat)
+                    top_spearman = _top_n_abs_correlations(corr.spearman_matrix, feat)
+                    lines.append(f"#### `{feat}`\n")
+                    lines.extend(
+                        _format_dict_lines(
+                            {
+                                "top_pearson": [
+                                    f"{name}: {value}" for name, value in top_pearson
+                                ],
+                                "top_spearman": [
+                                    f"{name}: {value}" for name, value in top_spearman
+                                ],
+                            }
+                        )
+                    )
+                    lines.append("")
+
+        lines.append("## Sentinels\n")
+        lines.extend(
+            _format_dict_lines(
+                {
+                    "numeric_sentinels": dict(self.numeric_sentinels),
+                    "string_sentinels": {
+                        k: list(v) for k, v in self.string_sentinels.items()
+                    },
+                }
+            )
+        )
+        lines.append("")
+
+        return "\n".join(lines).strip() + "\n"
+
+    def to_full_markdown(self) -> str:
+        """
+        Produce a complete, lossless Markdown serialization for debugging and archival use.
+
+        Every field present in ``to_dict()`` — including histogram bins, full
+        correlation matrices, memory breakdown, and all per-column fields —
+        is rendered as Markdown. For an 82-column dataset this produces
+        roughly 1 MB of text; for human inspection of large datasets prefer
+        ``to_markdown()`` once the compact view lands (ADR-0040).
+
+        Returns
+        -------
+        str
+            Markdown string containing a summary table followed by per-column detail
+            sections and dataset-level statistics.
+        """
+        data = self.to_dict()
+        lines = ["# Structural Profile Report\n"]
+
+        # 1. Summary navigation table
+        lines.append("## Summary\n")
+        lines.append("| Column | Semantic Type | Missing % | Severity | Key Flags |")
+        lines.append("|---|---|---|---|---|")
+        
+        for col_name, col_data in data.get("columns", {}).items():
+            sem_type = col_data.get("semantic_type") or "None"
+            missingness = col_data.get("missingness") or {}
+            missing_pct = missingness.get("effective_null_ratio", 0.0) * 100
+            missing_str = f"{missing_pct:.2f}%"
+            severity = missingness.get("severity") or "None"
+            flags = ", ".join(col_data.get("type_flags", [])) or "None"
+            lines.append(f"| {col_name} | {sem_type} | {missing_str} | {severity} | {flags} |")
+        
+        lines.append("\n## Column Details\n")
+
+        for col_name, col_data in data.get("columns", {}).items():
+            lines.append(f"### `{col_name}`\n")
+            lines.extend(_format_dict_lines(col_data))
+            lines.append("")
+
+        lines.append("## Dataset\n")
+        lines.extend(_format_dict_lines(data.get("dataset", {})))
+        lines.append("")
+
+        lines.append("## Targets\n")
+        lines.extend(_format_dict_lines(data.get("targets", {})))
+        lines.append("")
+
+        lines.append("## Numeric Sentinels\n")
+        lines.extend(_format_dict_lines(data.get("numeric_sentinels", {})))
+        lines.append("")
+
+        lines.append("## String Sentinels\n")
+        lines.extend(_format_dict_lines(data.get("string_sentinels", {})))
+        lines.append("")
+
+        return "\n".join(lines).strip() + "\n"
+
 
 # ---------------------------------------------------------------------------
 # ProfileConfig — clean break from per-profiler column lists

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/src/dataforge_ml/profiling/_numeric_profiler.py

@@ -74,6 +74,22 @@ class NumericProfiler(ColumnBatchProfiler[NumericProfileResult]):
         data: pl.DataFrame,
         columns: list[str],
     ) -> NumericProfileResult:
+        """
+        Profile the specified numeric columns in a DataFrame.
+
+        Parameters
+        ----------
+        data : pl.DataFrame
+            The input Polars DataFrame containing the columns to profile.
+        columns : list[str]
+            A list of column names to profile. Non-numeric columns in this list
+            are skipped.
+
+        Returns
+        -------
+        NumericProfileResult
+            A result object containing distribution statistics for the profiled columns.
+        """
         return self._run(data, columns)
 
     # ------------------------------------------------------------------
@@ -128,7 +144,7 @@ class NumericProfiler(ColumnBatchProfiler[NumericProfileResult]):
             profile.mean = mean
             profile.median = median
             if median == 0.0:
-                profile.mean_median_ratio = float("inf") if mean != 0.0 else 1.0
+                profile.mean_median_ratio = None if mean != 0.0 else 1.0
             else:
                 profile.mean_median_ratio = mean / median
 

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2}/src/dataforge_ml/profiling/_target_profiler.py

@@ -137,11 +137,12 @@ class TargetProfiler(DatasetLevelProfiler[TargetProfileResult]):
         result.categorical_profile = cat_profile
 
         # Flag Imbalances
-        ratio = cat_profile.imbalance.class_ratio
-        if ratio > 20.0:
-            result.flags.append(TargetFlag.SevereImbalance)
-        elif ratio > 5.0:
-            result.flags.append(TargetFlag.HighImbalance)
+        ratio = cat_profile.imbalance.dominant_class_ratio
+        if ratio is not None:
+            if ratio > 20.0:
+                result.flags.append(TargetFlag.SevereImbalance)
+            elif ratio > 5.0:
+                result.flags.append(TargetFlag.HighImbalance)
 
     def _profile_regression(
         self, series: pl.Series, n_rows: int, result: TargetProfileResult

{dataforge_ml-2.0.0 → dataforge_ml-2.0.2/src/dataforge_ml.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataforge-ml
-Version: 2.0.0
+Version: 2.0.2
 Summary: A automated feature engineering and designing pipeline library
 License: MIT
 Classifier: License :: OSI Approved :: MIT License