mlpipe-cli 0.1.0__py3-none-any.whl

mlpipe/data/ingestion.py

@@ -0,0 +1,138 @@
+"""
+Data ingestion module for MLPipe.
+
+Provides a structured Dataset abstraction and safe CSV loading with validation,
+hashing, and descriptive error messages.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+import pandas as pd
+
+from mlpipe.core.exceptions import DatasetError
+from mlpipe.utils.hashing import compute_file_hash
+
+
+@dataclass
+class Dataset:
+    """Structured representation of an ingested dataset."""
+
+    filepath: Path
+    df: pd.DataFrame
+    num_rows: int
+    num_cols: int
+    memory_bytes: int
+    sha256_hash: str
+    column_names: List[str]
+
+    @property
+    def filename(self) -> str:
+        return self.filepath.name
+
+    @property
+    def data(self) -> pd.DataFrame:
+        """Alias for df."""
+        return self.df
+
+    @property
+    def memory_mb(self) -> float:
+        return round(self.memory_bytes / (1024 * 1024), 2)
+
+    def summary(self) -> Dict[str, Any]:
+        """Return a clean dictionary summary of the dataset."""
+        return {
+            "filename": self.filename,
+            "filepath": str(self.filepath),
+            "rows": self.num_rows,
+            "columns": self.num_cols,
+            "memory_mb": self.memory_mb,
+            "sha256": self.sha256_hash,
+            "column_names": self.column_names,
+        }
+
+
+def load_dataset(path: Union[str, Path]) -> Dataset:
+    """
+    Safely load a CSV dataset from disk and return a structured Dataset object.
+
+    Args:
+        path: Path to the CSV file.
+
+    Returns:
+        Dataset object containing metadata and dataframe.
+
+    Raises:
+        DatasetError: If the file does not exist, has an invalid format, or is empty.
+    """
+    file_path = Path(path).resolve()
+
+    if not file_path.exists():
+        raise DatasetError(
+            f"Dataset file not found at '{file_path}'.",
+            "Verify that the file path is correct and accessible."
+        )
+
+    if not file_path.is_file():
+        raise DatasetError(
+            f"Path '{file_path}' is a directory, not a file.",
+            "Provide the path to a valid CSV file."
+        )
+
+    if file_path.suffix.lower() not in [".csv"]:
+        raise DatasetError(
+            f"Unsupported file format '{file_path.suffix}'. MLPipe currently supports '.csv'.",
+            "Please convert your dataset to CSV format."
+        )
+
+    # Check file size (e.g. empty file)
+    if file_path.stat().st_size == 0:
+        raise DatasetError(
+            f"Dataset file '{file_path.name}' is completely empty (0 bytes).",
+            "Ensure the file contains valid CSV data with a header and rows."
+        )
+
+    try:
+        df = pd.read_csv(file_path, low_memory=False)
+    except pd.errors.EmptyDataError:
+        raise DatasetError(
+            f"Dataset file '{file_path.name}' contains no headers or data.",
+            "Check that the CSV has column names and at least one data row."
+        )
+    except pd.errors.ParserError as e:
+        raise DatasetError(
+            f"Failed to parse CSV file '{file_path.name}': {e}",
+            "Check for delimiter issues, unescaped quotes, or inconsistent row lengths."
+        )
+    except Exception as e:
+        raise DatasetError(
+            f"Unexpected error while reading '{file_path.name}': {e}",
+            "Verify file permissions and that the file is not locked by another application."
+        )
+
+    if df.empty or len(df) == 0:
+        raise DatasetError(
+            f"Dataset '{file_path.name}' contains headers but zero data rows.",
+            "Provide a dataset with at least a few sample rows."
+        )
+
+    if len(df.columns) == 0:
+        raise DatasetError(
+            f"Dataset '{file_path.name}' contains no columns.",
+            "Check the delimiter of your CSV file."
+        )
+
+    # Compute SHA-256
+    file_hash = compute_file_hash(file_path)
+    memory_usage = int(df.memory_usage(deep=True).sum())
+
+    return Dataset(
+        filepath=file_path,
+        df=df,
+        num_rows=len(df),
+        num_cols=len(df.columns),
+        memory_bytes=memory_usage,
+        sha256_hash=file_hash,
+        column_names=list(df.columns),
+    )

mlpipe/data/profiling.py

@@ -0,0 +1,227 @@
+"""
+Data profiling module for MLPipe.
+
+Calculates comprehensive statistics on datasets and columns without hardcoded values.
+Detects column types, distributions, missingness, and structural issues.
+"""
+
+from dataclasses import asdict, dataclass, field
+import json
+import math
+import re
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from mlpipe.data.ingestion import Dataset
+
+
+def _safe_val(val: Any) -> Any:
+    """Convert numpy/pandas types to JSON-serializable standard Python types."""
+    if val is None or pd.isna(val):
+        return None
+    if isinstance(val, (np.integer, int)):
+        return int(val)
+    if isinstance(val, (np.floating, float)):
+        if math.isnan(val) or math.isinf(val):
+            return None
+        return round(float(val), 4)
+    if isinstance(val, (np.bool_, bool)):
+        return bool(val)
+    return str(val)
+
+
+def detect_column_type(series: pd.Series) -> str:
+    """Infer column type: 'numeric', 'categorical', 'boolean', or 'datetime'."""
+    if pd.api.types.is_bool_dtype(series):
+        return "boolean"
+
+    if pd.api.types.is_datetime64_any_dtype(series):
+        return "datetime"
+
+    if pd.api.types.is_numeric_dtype(series):
+        # Even if numeric, check if it's strictly binary 0/1 with boolean meaning
+        unique_vals = set(series.dropna().unique())
+        if unique_vals.issubset({0, 1}) and len(unique_vals) <= 2:
+            # Let boolean be inferred if column name or values suggest it
+            name_lower = str(series.name).lower()
+            if any(p in name_lower for p in ["is_", "has_", "flag", "active", "churn"]):
+                return "categorical"  # target or binary categorical
+        return "numeric"
+
+    # For object/string columns, try datetime detection
+    if series.dtype == object or pd.api.types.is_string_dtype(series):
+        non_null = series.dropna()
+        if len(non_null) > 0:
+            sample = non_null.head(min(50, len(non_null)))
+            try:
+                pd.to_datetime(sample, format="mixed")
+                # Also check full or larger sample if small sample succeeded
+                return "datetime"
+            except Exception:
+                pass
+
+            # Check boolean-like strings
+            lower_sample = set(str(v).strip().lower() for v in sample.unique())
+            if lower_sample.issubset({"true", "false", "yes", "no", "t", "f", "1", "0", "y", "n"}):
+                return "boolean"
+
+    return "categorical"
+
+
+@dataclass
+class ColumnProfile:
+    """Profile of an individual dataset column."""
+
+    name: str
+    detected_type: str
+    missing_count: int
+    missing_pct: float
+    unique_count: int
+    examples: List[Any]
+    # Numeric stats
+    min_val: Optional[float] = None
+    max_val: Optional[float] = None
+    mean_val: Optional[float] = None
+    median_val: Optional[float] = None
+    std_val: Optional[float] = None
+    # Categorical stats
+    top_values: List[Dict[str, Any]] = field(default_factory=list)
+    # Datetime stats
+    min_date: Optional[str] = None
+    max_date: Optional[str] = None
+    # Warnings/Flags
+    flags: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class DatasetProfile:
+    """Complete dataset profile including dataset-level and column-level information."""
+
+    num_rows: int
+    num_cols: int
+    memory_bytes: int
+    memory_mb: float
+    duplicate_rows: int
+    total_missing_values: int
+    columns: List[ColumnProfile]
+    warnings: List[str]
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def to_json(self, indent: int = 2) -> str:
+        return json.dumps(self.to_dict(), indent=indent)
+
+    def get_column(self, name: str) -> Optional[ColumnProfile]:
+        for col in self.columns:
+            if col.name == name:
+                return col
+        return None
+
+
+def profile_dataset(dataset: Union[Dataset, pd.DataFrame]) -> DatasetProfile:
+    """
+    Profile a dataset and generate accurate, computed descriptive statistics.
+
+    Args:
+        dataset: Ingested Dataset or pandas DataFrame.
+
+    Returns:
+        DatasetProfile object.
+    """
+    df = dataset.df if isinstance(dataset, Dataset) else dataset
+
+    num_rows = len(df)
+    num_cols = len(df.columns)
+    memory_bytes = int(df.memory_usage(deep=True).sum())
+    memory_mb = round(memory_bytes / (1024 * 1024), 2)
+    duplicate_rows = int(df.duplicated().sum())
+    total_missing = int(df.isna().sum().sum())
+
+    column_profiles: List[ColumnProfile] = []
+    dataset_warnings: List[str] = []
+
+    if duplicate_rows > 0:
+        dataset_warnings.append(f"Found {duplicate_rows} duplicate rows ({round(duplicate_rows/num_rows*100, 1)}%).")
+
+    id_regex = re.compile(r"(^id$|_id$|^id_|^index$|^guid$|^uuid$)", re.IGNORECASE)
+
+    for col_name in df.columns:
+        series = df[col_name]
+        col_type = detect_column_type(series)
+        missing_count = int(series.isna().sum())
+        missing_pct = round((missing_count / num_rows) * 100, 2) if num_rows > 0 else 0.0
+        unique_count = int(series.nunique(dropna=True))
+
+        examples = [_safe_val(x) for x in series.dropna().unique()[:5].tolist()]
+        flags = []
+
+        # Check flags
+        if missing_pct > 50.0:
+            flags.append(f"High missingness: {missing_pct}% missing")
+        if unique_count <= 1:
+            flags.append("Constant column (only 1 unique value)")
+        elif unique_count == num_rows and col_type in ("numeric", "categorical"):
+            flags.append("Suspicious unique/ID column (every value is distinct)")
+        elif id_regex.search(str(col_name)) and unique_count > num_rows * 0.8:
+            flags.append("Possible identifier/index column")
+
+        if col_type == "categorical" and unique_count > 100 and unique_count > num_rows * 0.5:
+            flags.append(f"Extremely high cardinality: {unique_count} distinct categories")
+
+        min_val = max_val = mean_val = median_val = std_val = None
+        top_values: List[Dict[str, Any]] = []
+        min_date = max_date = None
+
+        if col_type == "numeric":
+            desc = series.describe()
+            min_val = _safe_val(desc.get("min"))
+            max_val = _safe_val(desc.get("max"))
+            mean_val = _safe_val(desc.get("mean"))
+            median_val = _safe_val(series.median())
+            std_val = _safe_val(desc.get("std"))
+        elif col_type == "categorical":
+            vc = series.value_counts(dropna=True).head(5)
+            top_values = [{"value": str(k), "count": int(v)} for k, v in vc.items()]
+        elif col_type == "datetime":
+            try:
+                dt_series = pd.to_datetime(series, errors="coerce")
+                min_date = _safe_val(dt_series.min())
+                max_date = _safe_val(dt_series.max())
+            except Exception:
+                pass
+
+        col_prof = ColumnProfile(
+            name=str(col_name),
+            detected_type=col_type,
+            missing_count=missing_count,
+            missing_pct=missing_pct,
+            unique_count=unique_count,
+            examples=examples,
+            min_val=min_val,
+            max_val=max_val,
+            mean_val=mean_val,
+            median_val=median_val,
+            std_val=std_val,
+            top_values=top_values,
+            min_date=min_date,
+            max_date=max_date,
+            flags=flags,
+        )
+        column_profiles.append(col_prof)
+
+    return DatasetProfile(
+        num_rows=num_rows,
+        num_cols=num_cols,
+        memory_bytes=memory_bytes,
+        memory_mb=memory_mb,
+        duplicate_rows=duplicate_rows,
+        total_missing_values=total_missing,
+        columns=column_profiles,
+        warnings=dataset_warnings,
+    )

mlpipe/data/splitting.py

@@ -0,0 +1,130 @@
+"""
+Train/test splitting for MLPipe.
+
+Implements leakage-free splitting with stratification for classification when appropriate,
+and random splitting for regression.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Tuple, Union
+
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from mlpipe.utils.logging import get_logger
+
+logger = get_logger("splitting")
+
+
+@dataclass
+class SplitData:
+    """Container for train/test data splits."""
+
+    X_train: pd.DataFrame
+    X_test: pd.DataFrame
+    y_train: pd.Series
+    y_test: pd.Series
+    train_size: int
+    test_size: int
+    is_stratified: bool
+    target_name: str = "target"
+
+    @property
+    def train_df(self) -> pd.DataFrame:
+        """Combined DataFrame of training features and target."""
+        df = self.X_train.copy()
+        col_name = self.target_name or (self.y_train.name if self.y_train.name else "target")
+        df[col_name] = self.y_train.values
+        return df
+
+    @property
+    def test_df(self) -> pd.DataFrame:
+        """Combined DataFrame of test features and target."""
+        df = self.X_test.copy()
+        col_name = self.target_name or (self.y_test.name if self.y_test.name else "target")
+        df[col_name] = self.y_test.values
+        return df
+
+    def export(self, output_dir: Union[str, Path] = ".") -> Tuple[Path, Path]:
+        """Save train.csv and test.csv to an output directory."""
+        out_dir = Path(output_dir)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        train_path = out_dir / "train.csv"
+        test_path = out_dir / "test.csv"
+        self.train_df.to_csv(train_path, index=False)
+        self.test_df.to_csv(test_path, index=False)
+        return train_path, test_path
+
+
+def split_data(
+    df: pd.DataFrame,
+    target_column: str,
+    task_type: str,
+    test_size: float = 0.20,
+    random_seed: int = 42,
+) -> SplitData:
+    """
+    Split a DataFrame into train and test sets without data leakage.
+
+    Args:
+        df: Raw input DataFrame.
+        target_column: Name of the target column.
+        task_type: 'classification' or 'regression'.
+        test_size: Fraction of samples to allocate to test set.
+        random_seed: Random state for reproducibility.
+
+    Returns:
+        SplitData container with X_train, X_test, y_train, y_test.
+    """
+    # 1. Drop rows where target is NaN/null
+    clean_df = df.dropna(subset=[target_column]).copy()
+    if len(clean_df) < len(df):
+        logger.info("Dropped %d rows with missing target values prior to splitting.", len(df) - len(clean_df))
+
+    X = clean_df.drop(columns=[target_column])
+    y = clean_df[target_column]
+
+    stratify = None
+    is_stratified = False
+
+    if task_type == "classification":
+        # Check if every class has at least 2 samples for stratification
+        # and test set has at least as many samples as classes
+        class_counts = y.value_counts()
+        test_samples = int(len(clean_df) * test_size)
+        if (class_counts >= 2).all() and len(class_counts) > 1 and test_samples >= len(class_counts):
+            stratify = y
+            is_stratified = True
+        else:
+            logger.warning(
+                "Cannot stratify: either classes have <2 samples or test set size (%d) < classes (%d). Using random split.",
+                test_samples,
+                len(class_counts),
+            )
+
+
+    X_train, X_test, y_train, y_test = train_test_split(
+        X,
+        y,
+        test_size=test_size,
+        random_state=random_seed,
+        stratify=stratify,
+    )
+
+    # Reset indices to ensure clean indexing during transformations
+    X_train = X_train.reset_index(drop=True)
+    X_test = X_test.reset_index(drop=True)
+    y_train = y_train.reset_index(drop=True)
+    y_test = y_test.reset_index(drop=True)
+
+    return SplitData(
+        X_train=X_train,
+        X_test=X_test,
+        y_train=y_train,
+        y_test=y_test,
+        train_size=len(X_train),
+        test_size=len(X_test),
+        is_stratified=is_stratified,
+        target_name=target_column,
+    )