dataforge-ml 1.0.0__tar.gz → 1.0.1__tar.gz

dataforge_ml-1.0.1/PKG-INFO

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: dataforge-ml
+Version: 1.0.1
+Summary: A automated feature engineering and designing pipeline library
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: numpy>=2.0.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: chardet>=5.0.0
+Requires-Dist: iterative-stratification>=0.1.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# DataForgeML
+
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/DEVunderdog/DataForgeML)
+
+Automated data profiling and splitting pipeline for ML datasets.
+
+DataForgeML inspects your dataset, detects each column's semantic type (numeric, categorical, boolean, text, datetime, or identifier), computes per-column statistics and missingness, and produces a structured result ready for downstream feature engineering — no manual schema wrangling required.
+
+## Installation
+
+```bash
+pip install dataforge-ml
+```
+
+## Quick Start
+
+```python
+from dataforge_ml import DataLoader, PipelineConfig, StructuralProfiler
+
+df = DataLoader().load("titanic.csv")
+
+config = PipelineConfig()
+result = StructuralProfiler(config).profile(df)
+
+print(result.columns["Age"].semantic_type)  # SemanticType.Numeric
+print(result.dataset.row_count)             # total rows
+```
+
+`DataLoader` auto-detects encoding and delimiter. Supported formats: CSV, TSV, Parquet, JSON, NDJSON, JSONL, XLSX, XLS, Arrow, Feather.
+
+## Column Type Overrides
+
+Override the auto-detected type for any column before profiling:
+
+```python
+config = PipelineConfig()
+config.set_column_type("PassengerId", "identifier")           # skip stats entirely
+config.set_columns_type(["Survived", "Pclass"], "categorical")
+
+result = StructuralProfiler(config).profile(df)
+```
+
+To drop a column from all processing entirely, use `exclude_columns`:
+
+```python
+config = PipelineConfig(exclude_columns=["PassengerId", "Name"])
+```
+
+## Splitting
+
+```python
+from dataforge_ml import DataLoader, DataSplitter
+
+df = DataLoader().load("titanic.csv")
+splitter = DataSplitter(df, target="Survived", random_seed=42)
+
+# Random train/test split (stratified by default when target is set)
+split = splitter.random_split(test_size=0.2)
+print(split.train.shape, split.test.shape)
+
+# Chronological split (no temporal leakage)
+split = splitter.time_split(time_column="date", test_size=0.2)
+
+# K-fold cross-validation
+for fold in splitter.kfold(k=5):
+    print(f"Fold {fold.fold_index}: train={fold.train_size}, val={fold.val_size}")
+```
+
+## License
+
+MIT

dataforge_ml-1.0.1/README.md

@@ -0,0 +1,71 @@
+# DataForgeML
+
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/DEVunderdog/DataForgeML)
+
+Automated data profiling and splitting pipeline for ML datasets.
+
+DataForgeML inspects your dataset, detects each column's semantic type (numeric, categorical, boolean, text, datetime, or identifier), computes per-column statistics and missingness, and produces a structured result ready for downstream feature engineering — no manual schema wrangling required.
+
+## Installation
+
+```bash
+pip install dataforge-ml
+```
+
+## Quick Start
+
+```python
+from dataforge_ml import DataLoader, PipelineConfig, StructuralProfiler
+
+df = DataLoader().load("titanic.csv")
+
+config = PipelineConfig()
+result = StructuralProfiler(config).profile(df)
+
+print(result.columns["Age"].semantic_type)  # SemanticType.Numeric
+print(result.dataset.row_count)             # total rows
+```
+
+`DataLoader` auto-detects encoding and delimiter. Supported formats: CSV, TSV, Parquet, JSON, NDJSON, JSONL, XLSX, XLS, Arrow, Feather.
+
+## Column Type Overrides
+
+Override the auto-detected type for any column before profiling:
+
+```python
+config = PipelineConfig()
+config.set_column_type("PassengerId", "identifier")           # skip stats entirely
+config.set_columns_type(["Survived", "Pclass"], "categorical")
+
+result = StructuralProfiler(config).profile(df)
+```
+
+To drop a column from all processing entirely, use `exclude_columns`:
+
+```python
+config = PipelineConfig(exclude_columns=["PassengerId", "Name"])
+```
+
+## Splitting
+
+```python
+from dataforge_ml import DataLoader, DataSplitter
+
+df = DataLoader().load("titanic.csv")
+splitter = DataSplitter(df, target="Survived", random_seed=42)
+
+# Random train/test split (stratified by default when target is set)
+split = splitter.random_split(test_size=0.2)
+print(split.train.shape, split.test.shape)
+
+# Chronological split (no temporal leakage)
+split = splitter.time_split(time_column="date", test_size=0.2)
+
+# K-fold cross-validation
+for fold in splitter.kfold(k=5):
+    print(f"Fold {fold.fold_index}: train={fold.train_size}, val={fold.val_size}")
+```
+
+## License
+
+MIT

{dataforge_ml-1.0.0 → dataforge_ml-1.0.1}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataforge-ml"
-version = "1.0.0"
+version = "1.0.1"
 description = "A automated feature engineering and designing pipeline library"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">3.10"
 license = {text = "MIT"}
 classifiers = [
     "License :: OSI Approved :: MIT License",
@@ -23,6 +23,7 @@ dependencies = [
     "numpy>=2.0.0",
     "pandas>=2.0.0",
     "chardet>=5.0.0",
+    "iterative-stratification>=0.1.9",
 ]
 
 [project.optional-dependencies]

{dataforge_ml-1.0.0 → dataforge_ml-1.0.1}/src/dataforge_ml/__init__.py

@@ -1,5 +1,5 @@
 from .config import PipelineConfig, PipelinePhase, SemanticType, Modality
-from .profiling.structural import StructuralProfiler
+from .profiling.orchestrator import StructuralProfiler
 from .profiling._config import (
     ProfileConfig,
     StructuralProfileResult,

{dataforge_ml-1.0.0 → dataforge_ml-1.0.1}/src/dataforge_ml/config.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from dataforge_ml.profiling._config import ProfileConfig
+    from dataforge_ml.imputation._config import ImputationConfig
 
 
 class SemanticType(StrEnum):
@@ -36,6 +37,11 @@ def _default_profile_config() -> ProfileConfig:
     return ProfileConfig()
 
 
+def _default_imputation_config() -> ImputationConfig:
+    from dataforge_ml.imputation._config import ImputationConfig
+    return ImputationConfig()
+
+
 @dataclass
 class PipelineConfig:
     """
@@ -58,6 +64,7 @@ class PipelineConfig:
     phase_exclusions: dict[PipelinePhase, list[str]] = field(default_factory=dict)
     column_overrides: dict[str, SemanticType] = field(default_factory=dict)
     profiling: ProfileConfig = field(default_factory=_default_profile_config)
+    imputation: ImputationConfig = field(default_factory=_default_imputation_config)
 
     def resolve_active_columns(
         self, phase: PipelinePhase, available_columns: list[str]
@@ -107,11 +114,13 @@ class PipelineConfig:
                 for col, sem_type in self.column_overrides.items()
             },
             "profiling": self.profiling.to_dict(),
+            "imputation": self.imputation.to_dict(),
         }
 
     @classmethod
     def from_dict(cls, data: dict) -> PipelineConfig:
         from dataforge_ml.profiling._config import ProfileConfig
+        from dataforge_ml.imputation._config import ImputationConfig
         return cls(
             exclude_columns=list(data.get("exclude_columns", [])),
             phase_exclusions={
@@ -123,6 +132,7 @@ class PipelineConfig:
                 for col, sem_str in data.get("column_overrides", {}).items()
             },
             profiling=ProfileConfig.from_dict(data.get("profiling", {})),
+            imputation=ImputationConfig.from_dict(data.get("imputation", {})),
         )
 
     def to_json(self, indent: int = 2) -> str:

dataforge_ml-1.0.1/src/dataforge_ml/imputation/__init__.py

@@ -0,0 +1,21 @@
+from ._config import (
+    ColumnImputationRecord,
+    ImputationConfig,
+    ImputationResult,
+    ImputationStrategy,
+    NumericImputationConfig,
+)
+from ._fitted_imputer import FittedImputer, UnfittedColumnError
+from .orchestrator import ImputationOrchestrator, SplitImbalanceWarning
+
+__all__ = [
+    "ImputationStrategy",
+    "NumericImputationConfig",
+    "ImputationConfig",
+    "ColumnImputationRecord",
+    "ImputationResult",
+    "FittedImputer",
+    "UnfittedColumnError",
+    "ImputationOrchestrator",
+    "SplitImbalanceWarning",
+]

dataforge_ml-1.0.1/src/dataforge_ml/imputation/_config.py

@@ -0,0 +1,165 @@
+"""
+Configuration and result dataclasses for the imputation phase — Phase 2.
+
+ImputationConfig controls strategy thresholds and MNAR declarations.
+Result dataclasses carry per-column audit records and the imputed DataFrame.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, Optional
+
+import polars as pl
+
+from ..config import SemanticType
+
+
+class ImputationStrategy(StrEnum):
+    Mean = "mean"
+    Median = "median"
+    Mode = "mode"
+    KNN = "knn"
+    Regression = "regression"
+    MICE = "mice"
+    Constant = "constant"
+    Dropped = "dropped"
+    Passthrough = "passthrough"
+
+
+@dataclass
+class NumericImputationConfig:
+    """
+    Operational thresholds for the numeric imputation sub-processor.
+
+    Parameters
+    ----------
+    knn_max_rows : int
+        Maximum number of rows before KNN is skipped in favour of Regression.
+    knn_max_features : int
+        Maximum number of features before KNN is skipped in favour of Regression.
+    regression_min_rows : int
+        Minimum number of rows required to fit a stable Regression model.
+    mnar_constant_fill : float
+        Constant value used to fill MNAR-declared numeric columns.
+    """
+
+    knn_max_rows: int = 50_000
+    knn_max_features: int = 50
+    regression_min_rows: int = 500
+    mnar_constant_fill: float = -1
+
+    def to_dict(self) -> dict:
+        return {
+            "knn_max_rows": self.knn_max_rows,
+            "knn_max_features": self.knn_max_features,
+            "regression_min_rows": self.regression_min_rows,
+            "mnar_constant_fill": self.mnar_constant_fill,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> NumericImputationConfig:
+        return cls(
+            knn_max_rows=int(data.get("knn_max_rows", 50_000)),
+            knn_max_features=int(data.get("knn_max_features", 50)),
+            regression_min_rows=int(data.get("regression_min_rows", 500)),
+            mnar_constant_fill=float(data.get("mnar_constant_fill", -1)),
+        )
+
+
+@dataclass
+class ImputationConfig:
+    """
+    Cross-type Phase 2 configuration.
+
+    Parameters
+    ----------
+    numeric : NumericImputationConfig
+        Thresholds and fill values for numeric imputation.
+    mnar_columns : list[str]
+        Columns declared by the user as Missing Not At Random.
+        These receive Constant fill + a missingness indicator regardless
+        of the signals detected in Phase 1.
+    add_indicator_columns : list[str]
+        Columns for which a binary missingness indicator should be added
+        even when they are not MNAR.
+    """
+
+    numeric: NumericImputationConfig = field(default_factory=NumericImputationConfig)
+    mnar_columns: list[str] = field(default_factory=list)
+    add_indicator_columns: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "numeric": self.numeric.to_dict(),
+            "mnar_columns": list(self.mnar_columns),
+            "add_indicator_columns": list(self.add_indicator_columns),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> ImputationConfig:
+        return cls(
+            numeric=NumericImputationConfig.from_dict(data.get("numeric", {})),
+            mnar_columns=list(data.get("mnar_columns", [])),
+            add_indicator_columns=list(data.get("add_indicator_columns", [])),
+        )
+
+
+@dataclass
+class ColumnImputationRecord:
+    """
+    Per-column audit entry produced after fit().
+
+    Parameters
+    ----------
+    column : str
+        Column name.
+    semantic_type : SemanticType
+        Detected semantic type of the column.
+    strategy : ImputationStrategy
+        Strategy applied to this column.
+    fill_value : Any, optional
+        Scalar fill value learned from training data (None for model-based strategies).
+    indicator_added : bool
+        Whether a binary missingness indicator column was appended.
+    signals : list[str]
+        Human-readable reasons that drove the strategy decision.
+    """
+
+    column: str
+    semantic_type: SemanticType
+    strategy: ImputationStrategy
+    fill_value: Optional[Any] = None
+    indicator_added: bool = False
+    signals: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "column": self.column,
+            "semantic_type": str(self.semantic_type),
+            "strategy": str(self.strategy),
+            "fill_value": self.fill_value,
+            "indicator_added": self.indicator_added,
+            "signals": list(self.signals),
+        }
+
+
+@dataclass
+class ImputationResult:
+    """
+    Output of FittedImputer.transform().
+
+    Parameters
+    ----------
+    dataframe : pl.DataFrame
+        DataFrame with imputed values (and any indicator columns appended).
+    records : dict[str, ColumnImputationRecord]
+        Per-column audit log keyed by column name.
+    dropped_columns : list[str]
+        Columns removed because they exceeded the drop threshold (>50% missing).
+    """
+
+    dataframe: pl.DataFrame
+    records: dict[str, ColumnImputationRecord] = field(default_factory=dict)
+    dropped_columns: list[str] = field(default_factory=list)