dataforge-ml 0.1.0__tar.gz

dataforge_ml-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DEVunderdog
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dataforge_ml-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: dataforge-ml
+Version: 0.1.0
+Summary: A automated feature engineering and designing pipeline library
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: numpy>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# FeatureForge
+
+Automated feature engineering and data profiling pipeline library for tabular datasets.
+
+## Installation
+
+```bash
+pip install dataforge-ml
+```
+
+## License
+
+MIT

dataforge_ml-0.1.0/README.md

@@ -0,0 +1,13 @@
+# FeatureForge
+
+Automated feature engineering and data profiling pipeline library for tabular datasets.
+
+## Installation
+
+```bash
+pip install dataforge-ml
+```
+
+## License
+
+MIT

dataforge_ml-0.1.0/dataforge_ml.egg-info/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: dataforge-ml
+Version: 0.1.0
+Summary: A automated feature engineering and designing pipeline library
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: scipy>=1.10.0
+Requires-Dist: numpy>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# FeatureForge
+
+Automated feature engineering and data profiling pipeline library for tabular datasets.
+
+## Installation
+
+```bash
+pip install dataforge-ml
+```
+
+## License
+
+MIT

dataforge_ml-0.1.0/dataforge_ml.egg-info/SOURCES.txt

@@ -0,0 +1,57 @@
+LICENSE
+README.md
+pyproject.toml
+dataforge_ml.egg-info/PKG-INFO
+dataforge_ml.egg-info/SOURCES.txt
+dataforge_ml.egg-info/dependency_links.txt
+dataforge_ml.egg-info/requires.txt
+dataforge_ml.egg-info/top_level.txt
+models/__init__.py
+models/_data_structure.py
+models/_data_types.py
+profiling/__init__.py
+profiling/_base.py
+profiling/_boolean_config.py
+profiling/_boolean_profiler.py
+profiling/_categorical.py
+profiling/_categorical_config.py
+profiling/_correlation_config.py
+profiling/_correlation_profiler.py
+profiling/_datetime_config.py
+profiling/_datetime_profiler.py
+profiling/_missingness_config.py
+profiling/_missingness_profiler.py
+profiling/_numeric_config.py
+profiling/_numeric_profiler.py
+profiling/_tabular.py
+profiling/_target_config.py
+profiling/_target_profiler.py
+profiling/_text_config.py
+profiling/_text_profiler.py
+profiling/_type_detector.py
+profiling/config.py
+profiling/structural.py
+splitting/__init__.py
+splitting/_config.py
+splitting/_splitter.py
+tests/__init__.py
+tests/conftest.py
+tests/integration/__init__.py
+tests/integration/conftest.py
+tests/integration/test_structural_end_to_end.py
+tests/unit/__init__.py
+tests/unit/profiling/__init__.py
+tests/unit/profiling/conftest.py
+tests/unit/profiling/test_boolean_profiler.py
+tests/unit/profiling/test_categorical_profiler.py
+tests/unit/profiling/test_correlation_profiler.py
+tests/unit/profiling/test_datetime_profiler.py
+tests/unit/profiling/test_missingness_profiler.py
+tests/unit/profiling/test_numeric_profiler.py
+tests/unit/profiling/test_target_profiler.py
+tests/unit/profiling/test_text_profiler.py
+tests/unit/profiling/test_type_detector.py
+tests/unit/splitting/__init__.py
+tests/unit/splitting/test_data_splitter.py
+utils/__init__.py
+utils/data_loader.py

dataforge_ml-0.1.0/dataforge_ml.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dataforge_ml-0.1.0/dataforge_ml.egg-info/requires.txt

@@ -0,0 +1,7 @@
+polars>=1.0.0
+scikit-learn>=1.0.0
+scipy>=1.10.0
+numpy>=2.0.0
+
+[dev]
+pytest>=8.0

dataforge_ml-0.1.0/dataforge_ml.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+dist
+models
+profiling
+splitting
+tests
+utils

dataforge_ml-0.1.0/models/_data_structure.py

@@ -0,0 +1,7 @@
+from enum import StrEnum
+
+
+class DataStructure(StrEnum):
+    Tabular = "TABULAR"
+    TimeSeries = "TIME_SERIES"
+    GeoSpatial = "GEO_SPATIAL"

dataforge_ml-0.1.0/models/_data_types.py

@@ -0,0 +1,12 @@
+import polars as pl
+
+_INT_DTYPES = {
+    pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+    pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+}
+
+_CAT_DTYPES = {pl.Utf8, pl.String, pl.Categorical, pl.Boolean}
+
+_NUMERIC_DTYPES = _INT_DTYPES | {pl.Float32, pl.Float64}
+
+_DATETIME_DTYPES = {pl.Date, pl.Datetime}

dataforge_ml-0.1.0/profiling/__init__.py

@@ -0,0 +1,35 @@
+from .structural import StructuralProfiler
+from .config import (
+    ProfileConfig,
+    SemanticType,
+    Modality,
+    TypeFlag,
+    NumericKind,
+    NumericStats,
+    CategoricalStats,
+    DatetimeStats,
+    BooleanStats,
+    TextStats,
+    ColumnProfile,
+    DatasetStats,
+    StructuralProfileResult,
+)
+from ._base import ModalityProfiler
+
+__all__ = [
+    "StructuralProfiler",
+    "ProfileConfig",
+    "SemanticType",
+    "Modality",
+    "TypeFlag",
+    "NumericKind",
+    "NumericStats",
+    "CategoricalStats",
+    "DatetimeStats",
+    "BooleanStats",
+    "TextStats",
+    "ColumnProfile",
+    "DatasetStats",
+    "StructuralProfileResult",
+    "ModalityProfiler",
+]

dataforge_ml-0.1.0/profiling/_base.py

@@ -0,0 +1,101 @@
+"""
+Abstract base classes for all structural profilers.
+
+Hierarchy
+---------
+Profiling[R]                    — root: stores config, provides _resolve_columns
+├── ColumnBatchProfiler[R]      — registry tier: __init__(config=None) only;
+│   │                             profile(df, columns) processes a typed column batch
+│   ├── NumericProfiler
+│   ├── CategoricalProfiler
+│   ├── DatetimeProfiler
+│   ├── BooleanProfiler
+│   └── TextProfiler
+├── DatasetLevelProfiler[R]     — direct-call tier: may have extra __init__ params;
+│   │                             not compatible with the SemanticType registry
+│   ├── MissingnessProfiler
+│   ├── TargetProfiler
+│   └── CorrelationProfiler
+└── ModalityProfiler            — dataset-shape tier: profile(df) → DatasetStats
+    └── TabularProfiler
+"""
+
+from __future__ import annotations
+
+import polars as pl
+from abc import abstractmethod, ABC
+from typing import Generic, TypeVar
+
+from .config import DatasetStats, ProfileConfig
+
+R = TypeVar("R")
+
+
+class Profiling(ABC, Generic[R]):
+    """
+    Root base for all profilers.
+
+    Stores config and provides _resolve_columns. Not instantiated directly —
+    use one of the three concrete tier bases below.
+    """
+
+    def __init__(self, config: ProfileConfig | None = None):
+        self.config = config or ProfileConfig()
+
+    @abstractmethod
+    def profile(self, data: pl.DataFrame, **kwargs) -> R: ...
+
+    def _resolve_columns(
+        self,
+        available: list[str],
+        requested: list[str] | None,
+    ) -> list[str]:
+        if requested is None:
+            return list(available)
+        available_set = set(available)
+        return [c for c in requested if c in available_set]
+
+
+class ColumnBatchProfiler(Profiling[R]):
+    """
+    Registry-compatible column profiler.
+
+    Contract
+    --------
+    - __init__ must accept ONLY config (no extra required params). This allows
+      StructuralProfiler to instantiate any registered profiler uniformly via
+          profiler_cls(config=self.config)
+    - profile(df, columns) receives the full DataFrame and the list of same-type
+      column names to process. Returns a result with:
+          .columns: dict[str, <Stats>]        — per-column stats
+          .analysed_columns: list[str]        — columns actually profiled
+    """
+
+    @abstractmethod
+    def profile(self, data: pl.DataFrame, columns: list[str]) -> R: ...  # type: ignore[override]
+
+
+class DatasetLevelProfiler(Profiling[R]):
+    """
+    Directly-called profiler.
+
+    May have extra __init__ params (e.g. target_column, numeric_columns).
+    Never registered in the SemanticType registry — always instantiated
+    explicitly with its specific arguments.
+    """
+
+    @abstractmethod
+    def profile(self, data: pl.DataFrame, **kwargs) -> R: ...
+
+
+class ModalityProfiler(Profiling[DatasetStats]):
+    """
+    Dataset-shape profiler.
+
+    One concrete implementation per Modality. Returns DatasetStats covering
+    shape, memory, duplicates, sparsity, and chunking metadata.
+    profile(df) takes only the DataFrame — no column list needed.
+    """
+
+    @abstractmethod
+    def profile(self, data: pl.DataFrame, **kwargs) -> DatasetStats: ...  # type: ignore[override]

dataforge_ml-0.1.0/profiling/_boolean_config.py

@@ -0,0 +1,37 @@
+"""
+Result dataclass for boolean column profiling.
+
+Populated by BooleanProfiler.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class BooleanStats:
+    true_count: int = 0
+    false_count: int = 0
+    true_ratio: float = 0.0
+    false_ratio: float = 0.0
+    mode: Optional[bool] = None
+
+
+@dataclass
+class BooleanProfileResult:
+    """
+    Boolean profile for all eligible columns.
+
+    Attributes
+    ----------
+    columns : dict[str, BooleanStats]
+        Per-column boolean profiles, keyed by column name.
+    analysed_columns : list[str]
+        Columns that were actually profiled (after schema intersection
+        and eligibility check).
+    """
+
+    columns: dict[str, BooleanStats] = field(default_factory=dict)
+    analysed_columns: list[str] = field(default_factory=list)

dataforge_ml-0.1.0/profiling/_boolean_profiler.py

@@ -0,0 +1,191 @@
+"""
+BooleanProfiler  –  Phase 1 extension: Boolean Column Profiling.
+
+Handles columns classified as SemanticType.Boolean, which includes:
+  - Native Polars Boolean dtype
+  - Integer {0, 1} columns with a Boolean override in ProfileConfig
+  - Boolean-string columns ("true"/"false", "yes"/"no", "1"/"0") with override
+
+Per-column metrics:
+  1. true_count   – count of non-null truthy values
+  2. false_count  – count of non-null falsy values
+  3. true_ratio   – true_count / non_null_count  (nulls excluded)
+  4. false_ratio  – false_count / non_null_count (nulls excluded)
+  5. mode         – most frequent non-null value (True / False), or None if tied
+
+Null values are NOT counted in ratios — missingness is already captured by
+the upstream MissingnessProfiler pass and lives in ColumnProfile.missingness.
+"""
+
+from __future__ import annotations
+
+import polars as pl
+
+from ._base import ColumnBatchProfiler
+from .config import (
+    ProfileConfig,
+    BooleanStats,
+    SemanticType,
+)
+from ._boolean_config import BooleanProfileResult
+from ..models._data_types import _INT_DTYPES
+
+# ---------------------------------------------------------------------------
+# String values that represent True / False
+# ---------------------------------------------------------------------------
+
+_TRUE_STRINGS: frozenset[str] = frozenset({"true", "yes", "1", "t", "y"})
+_FALSE_STRINGS: frozenset[str] = frozenset({"false", "no", "0", "f", "n"})
+
+
+class BooleanProfiler(ColumnBatchProfiler[BooleanProfileResult]):
+    """
+    Boolean column profiler for Polars DataFrames.
+
+    A column is eligible when:
+      - Its Polars dtype is pl.Boolean, OR
+      - Its dtype is an integer with values exclusively in {0, 1}, OR
+      - It has a SemanticType.Boolean override in ProfileConfig.column_overrides
+
+    Non-eligible columns in the provided list are silently skipped.
+
+    Parameters
+    ----------
+    config : ProfileConfig | None
+        Shared profiling configuration.
+    """
+
+    def __init__(self, config: ProfileConfig | None = None) -> None:
+        super().__init__(config)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def profile(
+        self,
+        data: pl.DataFrame,
+        columns: list[str],
+    ) -> BooleanProfileResult:
+        return self._run(data, columns)
+
+    # ------------------------------------------------------------------
+    # Eligibility
+    # ------------------------------------------------------------------
+
+    def _eligible(self, series: pl.Series) -> bool:
+        override = self.config.column_overrides.get(series.name)
+
+        # Explicit override — trust it
+        if override == SemanticType.Boolean:
+            return True
+
+        # Another override takes precedence over auto-detection
+        if override is not None:
+            return False
+
+        # Native boolean dtype
+        if series.dtype == pl.Boolean:
+            return True
+
+        # Integer {0, 1} column — check after dropping nulls
+        if series.dtype in _INT_DTYPES:
+            clean = series.drop_nulls()
+            if clean.len() == 0:
+                return False
+            unique_vals = set(clean.unique().to_list())
+            return unique_vals <= {0, 1}
+
+        return False
+
+    # ------------------------------------------------------------------
+    # Orchestration
+    # ------------------------------------------------------------------
+
+    def _run(
+        self,
+        df: pl.DataFrame,
+        columns: list[str],
+    ) -> BooleanProfileResult:
+        result = BooleanProfileResult()
+
+        available = [
+            c
+            for c in self._resolve_columns(df.columns, columns)
+            if self._eligible(df[c])
+        ]
+        result.analysed_columns = available
+
+        for col_name in available:
+            result.columns[col_name] = self._profile_column(df[col_name], df.height)
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Per-column driver
+    # ------------------------------------------------------------------
+
+    def _profile_column(
+        self,
+        series: pl.Series,
+        n_rows: int,
+    ) -> BooleanStats:
+        profile = BooleanStats()
+
+        # Coerce to a clean boolean series (drop nulls)
+        bool_series = self._to_bool_series(series)
+        non_null_count = bool_series.len()
+
+        if non_null_count == 0:
+            return profile
+
+        true_count = int(bool_series.sum())
+        false_count = non_null_count - true_count
+
+        profile.true_count = true_count
+        profile.false_count = false_count
+        profile.true_ratio = true_count / non_null_count
+        profile.false_ratio = false_count / non_null_count
+
+        # Mode: True if more trues, False if more falses, None if perfectly tied
+        if true_count > false_count:
+            profile.mode = True
+        elif false_count > true_count:
+            profile.mode = False
+        else:
+            profile.mode = None  # tied — no single mode
+
+        return profile
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _to_bool_series(series: pl.Series) -> pl.Series:
+        """
+        Return a null-free Boolean Series regardless of the input dtype.
+
+        Handles:
+          - pl.Boolean      → drop nulls directly
+          - integer {0, 1}  → cast to Boolean, drop nulls
+          - string          → map known true/false strings, drop nulls
+        """
+        if series.dtype == pl.Boolean:
+            return series.drop_nulls()
+
+        if series.dtype in _INT_DTYPES:
+            return series.cast(pl.Boolean).drop_nulls()
+
+        if series.dtype == pl.Utf8:
+            lower = series.str.to_lowercase().str.strip_chars()
+            true_mask = lower.is_in(list(_TRUE_STRINGS))
+            false_mask = lower.is_in(list(_FALSE_STRINGS))
+            known_mask = true_mask | false_mask
+            return true_mask.filter(known_mask)
+
+        # Fallback: attempt a cast and drop nulls (covers e.g. Categorical)
+        try:
+            return series.cast(pl.Boolean).drop_nulls()
+        except Exception:
+            return pl.Series([], dtype=pl.Boolean)