pypreflight 0.1.0__py3-none-any.whl

preflight/__init__.py

@@ -0,0 +1,210 @@
+"""
+exposes prepare(), profile(), clean(), engineer(), compare() as public API
+"""
+
+import pandas as pd
+from preflight import assembler
+from preflight.types import PrepResult
+from preflight.report import Report
+from preflight.profiler import run_profiler
+from preflight.cleaner import run_cleaner
+from preflight.engineer import run_engineer
+def prepare(
+    df: pd.DataFrame,
+    target: str,
+    task: str = "classification",
+    model_hint: str = "tree",
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> PrepResult:
+    if len(df) == 0:
+        raise ValueError("DataFrame cannot be empty")
+        
+    if target not in df.columns:
+        raise ValueError(f"target '{target}' not found in DataFrame columns: {list(df.columns)}")
+        
+    if len(df.columns) < 2:
+        raise ValueError("DataFrame must contain at least one feature column besides the target")
+        
+    if df.columns.duplicated().any():
+        raise ValueError("DataFrame contains duplicate column names")
+        
+    if task not in ["classification", "regression"]:
+        raise ValueError(f"task must be 'classification' or 'regression', got '{task}'")
+        
+    if model_hint not in ["tree", "linear"]:
+        raise ValueError(f"model_hint must be 'tree' or 'linear', got '{model_hint}'")
+
+    return assembler.run_assembler(
+        df=df,
+        target=target,
+        task=task,
+        model_hint=model_hint,
+        drop_threshold=drop_threshold,
+        outlier_method=outlier_method,
+        cardinality_threshold=cardinality_threshold,
+    )
+
+def profile(
+    df: pd.DataFrame,
+    target: str,
+    task: str = "classification",
+    cardinality_threshold: int = 20,
+) -> PrepResult:
+    if len(df) == 0:
+        raise ValueError("DataFrame cannot be empty")
+    if target not in df.columns:
+        raise ValueError(f"target '{target}' not found in DataFrame columns: {list(df.columns)}")
+    if len(df.columns) < 2:
+        raise ValueError("DataFrame must contain at least one feature column besides the target")
+    if df.columns.duplicated().any():
+        raise ValueError("DataFrame contains duplicate column names")
+    if task not in ["classification", "regression"]:
+        raise ValueError(f"task must be 'classification' or 'regression', got '{task}'")
+        
+    profiles, profiler_entries = run_profiler(
+        df=df, target=target, task=task, cardinality_threshold=cardinality_threshold
+    )
+    return PrepResult(
+        df=df,
+        pipeline=None,
+        report=Report(profiler_entries)
+    )
+
+def clean(
+    df: pd.DataFrame,
+    target: str,
+    task: str = "classification",
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> PrepResult:
+    if len(df) == 0:
+        raise ValueError("DataFrame cannot be empty")
+    if target not in df.columns:
+        raise ValueError(f"target '{target}' not found in DataFrame columns: {list(df.columns)}")
+    if len(df.columns) < 2:
+        raise ValueError("DataFrame must contain at least one feature column besides the target")
+    if df.columns.duplicated().any():
+        raise ValueError("DataFrame contains duplicate column names")
+    if task not in ["classification", "regression"]:
+        raise ValueError(f"task must be 'classification' or 'regression', got '{task}'")
+        
+    profiles, profiler_entries = run_profiler(
+        df=df, target=target, task=task, cardinality_threshold=cardinality_threshold
+    )
+    df_clean, surviving_profiles, cleaner_entries, specs = run_cleaner(
+        df=df, profiles=profiles, target=target, drop_threshold=drop_threshold, outlier_method=outlier_method
+    )
+    return PrepResult(
+        df=df_clean,
+        pipeline=None,
+        report=Report(profiler_entries + cleaner_entries)
+    )
+
+def engineer(
+    df: pd.DataFrame,
+    target: str,
+    task: str = "classification",
+    model_hint: str = "tree",
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> PrepResult:
+    if len(df) == 0:
+        raise ValueError("DataFrame cannot be empty")
+    if target not in df.columns:
+        raise ValueError(f"target '{target}' not found in DataFrame columns: {list(df.columns)}")
+    if len(df.columns) < 2:
+        raise ValueError("DataFrame must contain at least one feature column besides the target")
+    if df.columns.duplicated().any():
+        raise ValueError("DataFrame contains duplicate column names")
+    if task not in ["classification", "regression"]:
+        raise ValueError(f"task must be 'classification' or 'regression', got '{task}'")
+    if model_hint not in ["tree", "linear"]:
+        raise ValueError(f"model_hint must be 'tree' or 'linear', got '{model_hint}'")
+        
+    profiles, profiler_entries = run_profiler(
+        df=df, target=target, task=task, cardinality_threshold=cardinality_threshold
+    )
+    df_clean, surviving_profiles, cleaner_entries, specs_c = run_cleaner(
+        df=df, profiles=profiles, target=target, drop_threshold=drop_threshold, outlier_method=outlier_method
+    )
+    df_eng, engineer_entries, specs_e = run_engineer(
+        df=df_clean, profiles=surviving_profiles, target=target, model_hint=model_hint, cardinality_threshold=cardinality_threshold
+    )
+    return PrepResult(
+        df=df_eng,
+        pipeline=None,
+        report=Report(profiler_entries + cleaner_entries + engineer_entries)
+    )
+
+def _extract_actions_per_column(report: Report | None) -> dict[str, set[str]]:
+    if report is None:
+        return {}
+    
+    actions = {}
+    for entry in report.entries:
+        if entry.column not in actions:
+            actions[entry.column] = set()
+        actions[entry.column].add(entry.action)
+    return actions
+
+def _compute_decision_diff(actions_a: dict[str, set[str]], actions_b: dict[str, set[str]]) -> list[str]:
+    diff = []
+    shared_cols = set(actions_a.keys()).intersection(actions_b.keys())
+    for col in sorted(shared_cols):
+        if actions_a[col] != actions_b[col]:
+            diff.append(col)
+    return diff
+
+def compare(a: PrepResult, b: PrepResult) -> dict:
+    cols_a = set(a.df.columns)
+    cols_b = set(b.df.columns)
+    
+    columns_only_in_a = sorted(cols_a - cols_b)
+    columns_only_in_b = sorted(cols_b - cols_a)
+    columns_in_both = sorted(cols_a.intersection(cols_b))
+    
+    report_entry_counts_a = a.report.summary_counts() if a.report is not None else None
+    report_entry_counts_b = b.report.summary_counts() if b.report is not None else None
+    
+    actions_a = _extract_actions_per_column(a.report)
+    actions_b = _extract_actions_per_column(b.report)
+    
+    if a.report is None or b.report is None:
+        decision_diff = []
+    else:
+        decision_diff = _compute_decision_diff(actions_a, actions_b)
+        
+    diff = {
+        "shape_a": a.df.shape,
+        "shape_b": b.df.shape,
+        "columns_only_in_a": columns_only_in_a,
+        "columns_only_in_b": columns_only_in_b,
+        "columns_in_both": columns_in_both,
+        "report_entry_counts_a": report_entry_counts_a,
+        "report_entry_counts_b": report_entry_counts_b,
+        "decision_diff": decision_diff,
+    }
+    
+    # Print summary
+    print("=== PreFlight Compare ===")
+    print(f"Shape A: {diff['shape_a']}")
+    print(f"Shape B: {diff['shape_b']}")
+    
+    if columns_only_in_a:
+        print(f"Columns only in A ({len(columns_only_in_a)}): {', '.join(columns_only_in_a[:5])}{'...' if len(columns_only_in_a) > 5 else ''}")
+    
+    if columns_only_in_b:
+        print(f"Columns only in B ({len(columns_only_in_b)}): {', '.join(columns_only_in_b[:5])}{'...' if len(columns_only_in_b) > 5 else ''}")
+        
+    if decision_diff:
+        print(f"Differing decisions in shared columns:")
+        for col in decision_diff:
+            print(f"  {col}: A={actions_a.get(col, set())} | B={actions_b.get(col, set())}")
+            
+    print("=========================")
+    
+    return diff

preflight/assembler.py

@@ -0,0 +1,384 @@
+import pandas as pd
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.pipeline import Pipeline
+from sklearn.exceptions import NotFittedError
+
+from preflight.types import ColumnProfile, SemanticType, PrepResult
+from preflight.profiler import run_profiler
+from preflight.report import Report
+from preflight.cleaner import (
+    run_cleaner,
+    add_missing_indicator,
+    normalize_category_values,
+    coerce_string_dates_to_datetime,
+    group_rare_categories,
+)
+from preflight.engineer import run_engineer, expand_datetime
+
+# =====================================================================
+# Architectural Decision: Two-Phase Pipeline Assembly
+# ---------------------------------------------------------------------
+# EngineerTransformer requires the POST-CLEANING column profiles. 
+# However, the surviving profiles are only known AFTER CleanerTransformer 
+# computes its dynamic drops during `fit()`.
+# 
+# To resolve this coupling without breaking standard sklearn Pipeline 
+# semantics, the `build_pipeline` function returns an UNFIT pipeline 
+# (which would fail or process dropped columns incorrectly if fitted 
+# blindly). Instead, `build_pipeline_two_phase` acts as the actual 
+# usable constructor: it manually fits the Cleaner stage, reads the 
+# resulting `fitted_profiles_`, uses them to construct and fit the 
+# Engineer stage, and then returns a fully assembled, pre-fitted 
+# Pipeline ready for `transform()`.
+# =====================================================================
+
+class CleanerTransformer(BaseEstimator, TransformerMixin):
+    def __init__(
+        self,
+        profiles: list[ColumnProfile],
+        target: str,
+        drop_threshold: float = 0.6,
+        outlier_method: str = "iqr",
+    ) -> None:
+        self.profiles = profiles
+        self.target = target
+        self.drop_threshold = drop_threshold
+        self.outlier_method = outlier_method
+
+    def fit(self, X: pd.DataFrame, y=None) -> "CleanerTransformer":
+        # run_cleaner computes the required stats
+        df_clean, self.fitted_profiles_, self.report_entries_, self.specs_ = run_cleaner(
+            X, self.profiles, self.target, self.drop_threshold, self.outlier_method
+        )
+        
+        surviving_cols = {p.name for p in self.fitted_profiles_}
+        self.columns_to_drop_ = [p.name for p in self.profiles if p.name not in surviving_cols]
+        
+        # Augment specs_ with exact bounds and indicators to avoid recomputing on test data
+        for p in self.fitted_profiles_:
+            col = p.name
+            if col not in self.specs_:
+                self.specs_[col] = {}
+                
+            # Missing indicator logic from cleaner.py
+            if p.missing_rate > 0 and p.semantic_type in (SemanticType.CATEGORICAL_LOW, SemanticType.NUMERIC_FEATURE):
+                self.specs_[col]["add_missing_indicator"] = True
+                
+            # Winsorization bounds logic
+            if p.semantic_type == SemanticType.NUMERIC_FEATURE and "outlier_method" in self.specs_[col]:
+                series = X[col]
+                method = self.specs_[col]["outlier_method"]
+                if method == "iqr":
+                    q1 = series.quantile(0.25)
+                    q3 = series.quantile(0.75)
+                    iqr = q3 - q1
+                    self.specs_[col]["lower_bound"] = q1 - 1.5 * iqr
+                    self.specs_[col]["upper_bound"] = q3 + 1.5 * iqr
+                elif method == "zscore":
+                    mean = series.mean()
+                    std = series.std()
+                    self.specs_[col]["lower_bound"] = mean - 3 * std
+                    self.specs_[col]["upper_bound"] = mean + 3 * std
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not hasattr(self, "specs_"):
+            raise NotFittedError("CleanerTransformer is not fitted yet. Call 'fit' before 'transform'.")
+            
+        X_out = X.copy()
+        
+        # 1. Drop columns (by name, not by recomputing missingness)
+        cols_to_drop = [c for c in self.columns_to_drop_ if c in X_out.columns]
+        X_out = X_out.drop(columns=cols_to_drop)
+        
+        # 2. Duplicate row removal is explicitly skipped here.
+        # Removing rows during transform (inference) would silently drop rows the caller expects back.
+        
+        for p in self.fitted_profiles_:
+            col = p.name
+            if col not in X_out.columns or col == self.target:
+                continue
+                
+            spec = self.specs_.get(col, {})
+            stype = p.semantic_type
+            
+            # Category normalization applies to both CATEGORICAL_LOW and CATEGORICAL_HIGH
+            if stype in (SemanticType.CATEGORICAL_LOW, SemanticType.CATEGORICAL_HIGH):
+                X_out[col] = normalize_category_values(X_out[col])
+                
+            # Date coercion
+            if stype == SemanticType.DATETIME_STRING:
+                X_out[col] = coerce_string_dates_to_datetime(X_out[col])
+                
+            # Group rare categories
+            if stype == SemanticType.CATEGORICAL_HIGH and "rare_categories" in spec:
+                X_out[col] = group_rare_categories(X_out[col], spec["rare_categories"])
+                
+            # Add missing indicator based on frozen fit-time decision
+            if spec.get("add_missing_indicator"):
+                missing_ind = add_missing_indicator(X_out[col])
+                X_out[missing_ind.name] = missing_ind
+                
+            # Impute values
+            if "impute_strategy" in spec:
+                X_out[col] = X_out[col].fillna(spec["impute_value"])
+                
+            # Winsorize using bounds computed at fit-time
+            if stype == SemanticType.NUMERIC_FEATURE and "lower_bound" in spec and "upper_bound" in spec:
+                X_out[col] = X_out[col].clip(
+                    lower=spec["lower_bound"], 
+                    upper=spec["upper_bound"]
+                )
+                
+        return X_out
+
+    def get_feature_names_out(self, input_features=None) -> np.ndarray:
+        if not hasattr(self, "fitted_profiles_"):
+            raise NotFittedError("CleanerTransformer is not fitted yet.")
+        cols = []
+        missing_cols = []
+        for p in self.fitted_profiles_:
+            cols.append(p.name)
+            if self.specs_.get(p.name, {}).get("add_missing_indicator"):
+                missing_cols.append(f"{p.name}_missing")
+        return np.array(cols + missing_cols)
+
+class EngineerTransformer(BaseEstimator, TransformerMixin):
+    def __init__(
+        self,
+        profiles: list[ColumnProfile],
+        target: str,
+        model_hint: str,
+        cardinality_threshold: int = 20,
+    ) -> None:
+        self.profiles = profiles
+        self.target = target
+        self.model_hint = model_hint
+        self.cardinality_threshold = cardinality_threshold
+
+    def fit(self, X: pd.DataFrame, y=None) -> "EngineerTransformer":
+        if y is None:
+            has_high_card = any(p.semantic_type == SemanticType.CATEGORICAL_HIGH for p in self.profiles)
+            if has_high_card:
+                raise ValueError("Target y must be provided for target encoding high-cardinality columns.")
+                
+        df_for_engineer = X.copy()
+        if y is not None:
+            df_for_engineer[self.target] = y
+
+        df_eng, self.report_entries_, self.specs_ = run_engineer(
+            df_for_engineer, self.profiles, self.target, self.model_hint, self.cardinality_threshold
+        )
+        
+        cols = list(df_eng.columns)
+        if self.target in cols and (y is not None and self.target not in X.columns):
+            cols.remove(self.target)
+        self.output_columns_ = cols
+        
+        if y is not None:
+            if isinstance(y, pd.Series):
+                global_mean = y.mean()
+            else:
+                global_mean = pd.Series(y).mean()
+            for p in self.profiles:
+                if p.semantic_type == SemanticType.CATEGORICAL_HIGH:
+                    if p.name in self.specs_ and isinstance(self.specs_[p.name], dict):
+                        self.specs_[p.name]["global_mean"] = global_mean
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not hasattr(self, "specs_"):
+            raise NotFittedError("EngineerTransformer is not fitted yet.")
+            
+        X_out = X.copy()
+        
+        for p in self.profiles:
+            col = p.name
+            stype = p.semantic_type
+            
+            if col not in X_out.columns or col == self.target:
+                continue
+                
+            spec = self.specs_.get(col, {})
+            trans_type = spec.get("transform")
+            
+            if trans_type == "passthrough":
+                continue
+                
+            elif trans_type == "expand_datetime":
+                expanded = expand_datetime(X_out[col])
+                X_out = X_out.drop(columns=[col]).join(expanded)
+                
+            elif trans_type == "ordinal_encode":
+                mapping = spec["mapping"]
+                X_out[col] = X_out[col].map(mapping).fillna(-1).astype(int)
+                
+            elif trans_type == "one_hot_encode":
+                dummies = pd.get_dummies(X_out[col], prefix=col, prefix_sep='_')
+                X_out = X_out.drop(columns=[col]).join(dummies)
+                
+            elif trans_type == "target_encode_cross_fit":
+                mapping = spec["mapping"]
+                global_mean = spec.get("global_mean", 0.0)
+                X_out[col] = X_out[col].map(mapping).fillna(global_mean)
+                
+            elif trans_type == "linear_numeric":
+                if spec.get("log1p"):
+                    X_out[col] = np.log1p(X_out[col])
+                
+                params = spec["scale_params"]
+                mean_val = params["mean"]
+                std_val = params["std"]
+                X_out[col] = (X_out[col] - mean_val) / std_val
+                
+        for expected_col in self.output_columns_:
+            if expected_col not in X_out.columns:
+                X_out[expected_col] = False
+                
+        X_out = X_out.reindex(columns=self.output_columns_)
+        
+        return X_out
+
+    def get_feature_names_out(self, input_features=None) -> np.ndarray:
+        if not hasattr(self, "output_columns_"):
+            raise NotFittedError("EngineerTransformer is not fitted yet.")
+        return np.array(self.output_columns_)
+
+def build_pipeline(
+    profiles: list[ColumnProfile],
+    target: str,
+    model_hint: str,
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> Pipeline:
+    cleaner = CleanerTransformer(
+        profiles=profiles,
+        target=target,
+        drop_threshold=drop_threshold,
+        outlier_method=outlier_method,
+    )
+    engineer = EngineerTransformer(
+        profiles=profiles,  # Will be overridden in two-phase fit
+        target=target,
+        model_hint=model_hint,
+        cardinality_threshold=cardinality_threshold,
+    )
+    
+    pipeline = Pipeline([
+        ("cleaner", cleaner),
+        ("engineer", engineer),
+    ])
+    pipeline.set_output(transform="pandas")
+    return pipeline
+
+def build_pipeline_two_phase(
+    profiles: list[ColumnProfile],
+    target: str,
+    model_hint: str,
+    X: pd.DataFrame,
+    y: pd.Series,
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> Pipeline:
+    cleaner = CleanerTransformer(
+        profiles=profiles,
+        target=target,
+        drop_threshold=drop_threshold,
+        outlier_method=outlier_method,
+    )
+    
+    cleaner.fit(X, y)
+    X_cleaned = cleaner.transform(X)
+    
+    engineer = EngineerTransformer(
+        profiles=cleaner.fitted_profiles_,
+        target=target,
+        model_hint=model_hint,
+        cardinality_threshold=cardinality_threshold,
+    )
+    engineer.fit(X_cleaned, y)
+    
+    pipeline = Pipeline([
+        ("cleaner", cleaner),
+        ("engineer", engineer),
+    ])
+    pipeline.set_output(transform="pandas")
+    return pipeline
+
+def run_assembler(
+    df: pd.DataFrame,
+    target: str,
+    task: str,
+    model_hint: str,
+    drop_threshold: float = 0.6,
+    outlier_method: str = "iqr",
+    cardinality_threshold: int = 20,
+) -> PrepResult:
+    # 1. Run profiler
+    profiles, profiler_report = run_profiler(
+        df, target, task, cardinality_threshold
+    )
+    
+    # 2. Split df into X and y
+    X = df.drop(columns=[target])
+    y = df[target]
+    
+    # 3. Call build_pipeline_two_phase
+    pipeline = build_pipeline_two_phase(
+        profiles=profiles,
+        target=target,
+        model_hint=model_hint,
+        X=X,
+        y=y,
+        drop_threshold=drop_threshold,
+        outlier_method=outlier_method,
+        cardinality_threshold=cardinality_threshold,
+    )
+    
+    # 4. Transform X and reattach target
+    X_final = pipeline.transform(X)
+    final_df = X_final.copy()
+    final_df[target] = y
+    
+    # 5. Collect all ReportEntries
+    cleaner_step = pipeline.named_steps["cleaner"]
+    engineer_step = pipeline.named_steps["engineer"]
+    all_entries = profiler_report + cleaner_step.report_entries_ + engineer_step.report_entries_
+    
+    # 6. Construct Report
+    report = Report(
+        entries=all_entries,
+        df=df,
+        profiles=profiles,
+        target=target
+    )
+    
+    # 7. Return PrepResult
+    return PrepResult(
+        df=final_df,
+        pipeline=pipeline,
+        report=report
+    )
+
+def transform_new_data(
+    pipeline: Pipeline, 
+    new_df: pd.DataFrame, 
+    target: str | None = None
+) -> pd.DataFrame:
+    df_in = new_df.copy()
+    target_series = None
+    
+    if target is not None and target in df_in.columns:
+        target_series = df_in.pop(target)
+        
+    df_out = pipeline.transform(df_in)
+    
+    if target_series is not None:
+        df_out[target] = target_series
+        
+    return df_out