datapilot-polars 0.3.0__py3-none-any.whl

datapilot/__init__.py

@@ -0,0 +1,119 @@
+__version__ = "0.3.0"
+
+# ── Analysis ──────────────────────────────────────────────────────────────────
+
+def summary(df):
+    """Calculates lightning fast structural overview metrics of the dataset."""
+    from .analysis.summary import summary as _summary
+    return _summary(df)
+
+def missing(df):
+    """Generates a report detailing missing count and percentage per column."""
+    from .analysis.missing import missing as _missing
+    return _missing(df)
+
+def duplicates(df):
+    """Calculates duplicate row counts and percentages using multi-threaded hashing."""
+    from .analysis.duplicates import duplicates as _duplicates
+    return _duplicates(df)
+
+def correlation(df, threshold=0.6):
+    """Calculates a high-speed Pearson correlation matrix and flags strong pairs."""
+    from .analysis.correlations import correlation as _correlation
+    return _correlation(df, threshold)
+
+def analyze(df, use_ai=False, ai_provider="ollama", ai_model=None, api_key=None):
+    """Runs automated EDA with optional AI insights from local or cloud providers.
+
+    ai_provider options: 'ollama' (local/default), 'openai', 'gemini', 'claude', 'groq'
+    """
+    from .analysis.eda import analyze as _analyze
+    return _analyze(df, use_ai=use_ai, ai_provider=ai_provider,
+                    ai_model=ai_model, api_key=api_key)
+
+def suggest(df):
+    """Analyses dataset structure and returns actionable preprocessing suggestions."""
+    from .analysis.suggest import suggest as _suggest
+    return _suggest(df)
+
+def compare(df_train, df_test, threshold=0.1):
+    """Detects distribution drift between a training and test/production DataFrame."""
+    from .analysis.compare import compare as _compare
+    return _compare(df_train, df_test, threshold)
+
+def outliers(df, method="both", z_threshold=3.0, iqr_multiplier=1.5):
+    """Detects outliers in all numeric columns using IQR and/or Z-score methods."""
+    from .analysis.outliers import outliers as _outliers
+    return _outliers(df, method=method, z_threshold=z_threshold, iqr_multiplier=iqr_multiplier)
+
+def auto_clean(df, drop_null_threshold=0.6, impute_strategy="auto",
+               drop_id_columns=True, drop_constant_columns=True):
+    """Automatically cleans a dataset — drops bad columns, imputes nulls."""
+    from .analysis.auto_clean import auto_clean as _auto_clean
+    return _auto_clean(
+        df,
+        drop_null_threshold=drop_null_threshold,
+        impute_strategy=impute_strategy,
+        drop_id_columns=drop_id_columns,
+        drop_constant_columns=drop_constant_columns,
+    )
+
+def benchmark(df):
+    """Benchmarks DataPilot (Polars) operations vs equivalent Pandas operations."""
+    from .analysis.benchmark import benchmark as _benchmark
+    return _benchmark(df)
+
+# ── Visualization ─────────────────────────────────────────────────────────────
+
+def hist(df, column, bins=10):
+    """Generates a clean distribution histogram for a selected numerical feature."""
+    from .visualization.histogram import hist as _hist
+    return _hist(df, column, bins)
+
+def box(df, column):
+    """Generates a clean box plot to showcase distribution quartiles and outliers."""
+    from .visualization.boxplot import box as _box
+    return _box(df, column)
+
+def heatmap(df):
+    """Generates a visual correlation matrix heatmap for all numerical features."""
+    from .visualization.heatmap import heatmap as _heatmap
+    return _heatmap(df)
+
+# ── Machine Learning ──────────────────────────────────────────────────────────
+
+def classification_report(y_true, y_pred, average="auto"):
+    """Calculates key classification performance metrics (binary or multi-class)."""
+    from .ml.classification import classification_report as _cr
+    return _cr(y_true, y_pred, average=average)
+
+def regression_report(y_true, y_pred):
+    """Calculates comprehensive regression evaluation metrics (MAE, RMSE, R², MAPE)."""
+    from .ml.regression import regression_report as _rr
+    return _rr(y_true, y_pred)
+
+def diagnose(train_score, test_score, metric_name="Accuracy"):
+    """Diagnoses model health, highlighting overfitting, underfitting, or optimal states."""
+    from .ml.diagnostics import diagnose as _diagnose
+    return _diagnose(train_score, test_score, metric_name)
+
+# ── Dashboard ─────────────────────────────────────────────────────────────────
+
+def dashboard(df, output_path="datapilot_report.html"):
+    """Generates a standalone, beautifully styled HTML dashboard report from the dataset."""
+    from .dashboard.dashboard import dashboard as _dashboard
+    return _dashboard(df, output_path)
+
+
+__all__ = [
+    "__version__",
+    # Analysis
+    "summary", "missing", "duplicates", "correlation", "analyze",
+    "suggest", "compare", "outliers", "auto_clean", "benchmark",
+    # Visualization
+    "hist", "box", "heatmap",
+    # ML
+    "classification_report", "regression_report", "diagnose",
+    # Dashboard
+    "dashboard",
+]

datapilot/ai/__init__.py

@@ -0,0 +1,3 @@
+from .insights import generate_insights
+
+__all__ = ["generate_insights"]

datapilot/ai/base.py

@@ -0,0 +1,57 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List
+
+
+class BaseAIProvider(ABC):
+    """Abstract base class for all DataPilot AI provider backends.
+
+    Every provider must implement a single `generate` method that accepts
+    the dataset metadata and returns a plain-text insight string.
+    Raw data rows are NEVER passed to any provider — only lightweight
+    statistical summaries are transmitted (metadata-only pattern).
+    """
+
+    def __init__(self, model: str, api_key: str = None, **kwargs):
+        self.model = model
+        self.api_key = api_key
+
+    @abstractmethod
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        """Generate AI insights from dataset metadata.
+
+        Args:
+            meta: Structural stats dict (rows, columns, dtypes, null counts).
+            missing_list: List of dicts with column null stats.
+            strong_relations: List of strong correlation pair strings.
+
+        Returns:
+            A plain-text string with actionable recommendations.
+        """
+
+    def _build_system_prompt(self) -> str:
+        return (
+            "You are DataPilot, an elite data science assistant. "
+            "Analyse the structural statistics provided and write exactly 3 brief, "
+            "impactful bullet points giving expert data preprocessing, cleaning, or "
+            "modelling advice. Be direct and conversational. "
+            "Do not include introductory text or markdown headers."
+        )
+
+    def _build_user_prompt(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        return (
+            f"Dataset Profile:\n"
+            f"- Dimensions: {meta['rows']} rows × {meta['columns']} columns\n"
+            f"- Duplicate Rows: {meta.get('duplicates_count', 'N/A')}\n"
+            f"- Missing Data: {missing_list}\n"
+            f"- Strong Correlations: {strong_relations}\n"
+        )

datapilot/ai/factory.py

@@ -0,0 +1,70 @@
+from .base import BaseAIProvider
+from typing import Optional
+
+
+# Registry mapping provider name → class (lazy import avoids hard dependencies)
+_PROVIDER_REGISTRY = {
+    "ollama": "datapilot.ai.providers.ollama_provider.OllamaProvider",
+    "openai": "datapilot.ai.providers.openai_provider.OpenAIProvider",
+    "gemini": "datapilot.ai.providers.gemini_provider.GeminiProvider",
+    "claude": "datapilot.ai.providers.claude_provider.ClaudeProvider",
+    "groq":   "datapilot.ai.providers.groq_provider.GroqProvider",
+}
+
+# Human-readable default models per provider
+DEFAULT_MODELS = {
+    "ollama": "llama3",
+    "openai": "gpt-4o-mini",
+    "gemini": "gemini-1.5-flash",
+    "claude": "claude-3-haiku-20240307",
+    "groq":   "llama3-70b-8192",
+}
+
+
+def get_provider(
+    ai_provider: str,
+    ai_model: Optional[str] = None,
+    api_key: Optional[str] = None,
+) -> BaseAIProvider:
+    """Resolves and instantiates the correct AI provider from a string name.
+
+    Args:
+        ai_provider: Name of the AI backend. One of:
+                     'ollama' (local, default), 'openai', 'gemini', 'claude', 'groq'.
+        ai_model:    Model name within the provider. If None, uses the provider's
+                     sensible default (e.g. 'gpt-4o-mini' for OpenAI).
+        api_key:     API key required for cloud providers. Not needed for Ollama.
+
+    Returns:
+        An instantiated BaseAIProvider subclass ready to call .generate().
+
+    Raises:
+        ValueError: If the provider name is not recognised.
+    """
+    provider_name = ai_provider.lower().strip()
+
+    if provider_name not in _PROVIDER_REGISTRY:
+        supported = ", ".join(f"'{k}'" for k in _PROVIDER_REGISTRY)
+        raise ValueError(
+            f"Unknown AI provider: '{ai_provider}'. "
+            f"Supported providers: {supported}."
+        )
+
+    # Validate API key requirement for cloud providers
+    if provider_name != "ollama" and not api_key:
+        raise ValueError(
+            f"'{ai_provider}' is a cloud provider and requires an API key. "
+            f"Pass it via: dp.analyze(df, use_ai=True, ai_provider='{ai_provider}', "
+            f"api_key='your-key-here')"
+        )
+
+    # Resolve default model
+    model = ai_model or DEFAULT_MODELS[provider_name]
+
+    # Lazy import of the provider class
+    module_path, class_name = _PROVIDER_REGISTRY[provider_name].rsplit(".", 1)
+    import importlib
+    module = importlib.import_module(module_path)
+    provider_class = getattr(module, class_name)
+
+    return provider_class(model=model, api_key=api_key)

datapilot/ai/insights.py

@@ -0,0 +1,46 @@
+from .factory import get_provider
+from typing import Dict, Any, List, Optional
+
+
+def generate_insights(
+    meta: Dict[str, Any],
+    missing_list: List[Dict[str, Any]],
+    strong_relations: List[str],
+    model_name: str = "llama3",
+    ai_provider: str = "ollama",
+    api_key: Optional[str] = None,
+) -> str:
+    """Sends lightweight dataset metadata to the configured AI provider and
+    returns conversational, context-aware engineering recommendations.
+
+    DataPilot uses a Metadata-Only AI Pattern — raw data rows are NEVER
+    transmitted to any provider. Only concise statistical summaries are sent.
+
+    Args:
+        meta:             Structural stats dict from dp.summary().
+        missing_list:     List of column null-stat dicts from dp.missing().
+        strong_relations: List of correlation pair strings from dp.correlation().
+        model_name:       Model to use within the chosen provider.
+                          Defaults vary per provider if not specified.
+        ai_provider:      AI backend to use. Options:
+                            'ollama'  — local, free, private (default)
+                            'openai'  — GPT-4o, GPT-4, GPT-3.5
+                            'gemini'  — Google Gemini 1.5 Pro/Flash
+                            'claude'  — Anthropic Claude 3.5 Sonnet/Haiku
+                            'groq'    — Ultra-fast free-tier (Llama3, Mixtral)
+        api_key:          API key for cloud providers. Not needed for Ollama.
+
+    Returns:
+        Plain-text string with 3 actionable data science recommendations.
+    """
+    try:
+        provider = get_provider(
+            ai_provider=ai_provider,
+            ai_model=model_name,
+            api_key=api_key,
+        )
+        return provider.generate(meta, missing_list, strong_relations)
+    except ValueError as e:
+        return f"⚠️  Configuration error: {e}"
+    except Exception as e:
+        return f"⚠️  Could not generate AI Insights. Error: {e}"

datapilot/ai/providers/claude_provider.py

@@ -0,0 +1,45 @@
+from ..base import BaseAIProvider
+from typing import Dict, Any, List
+
+
+class ClaudeProvider(BaseAIProvider):
+    """Anthropic Claude cloud AI provider (claude-3-5-sonnet, claude-3-haiku, etc.)
+
+    Requires the `anthropic` package: `pip install anthropic`
+    Only dataset metadata (stats, null counts) is sent — never raw data rows.
+
+    Usage:
+        dp.analyze(df, use_ai=True, ai_provider="claude",
+                   ai_model="claude-3-5-sonnet-20241022", api_key="sk-ant-...")
+    """
+
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        try:
+            import anthropic
+        except ImportError:
+            return (
+                "⚠️  Anthropic package not installed. "
+                "Run: pip install anthropic"
+            )
+
+        try:
+            client = anthropic.Anthropic(api_key=self.api_key)
+            response = client.messages.create(
+                model=self.model,
+                max_tokens=300,
+                system=self._build_system_prompt(),
+                messages=[
+                    {
+                        "role": "user",
+                        "content": self._build_user_prompt(meta, missing_list, strong_relations),
+                    }
+                ],
+            )
+            return response.content[0].text.strip()
+        except Exception as e:
+            return f"⚠️  Anthropic Claude API error: {e}"

datapilot/ai/providers/gemini_provider.py

@@ -0,0 +1,46 @@
+from ..base import BaseAIProvider
+from typing import Dict, Any, List
+
+
+class GeminiProvider(BaseAIProvider):
+    """Google Gemini cloud AI provider (gemini-1.5-pro, gemini-1.5-flash, etc.)
+
+    Requires the `google-generativeai` package: `pip install google-generativeai`
+    Only dataset metadata (stats, null counts) is sent — never raw data rows.
+
+    Usage:
+        dp.analyze(df, use_ai=True, ai_provider="gemini",
+                   ai_model="gemini-1.5-flash", api_key="AIza...")
+    """
+
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        try:
+            import google.generativeai as genai
+        except ImportError:
+            return (
+                "⚠️  Google Generative AI package not installed. "
+                "Run: pip install google-generativeai"
+            )
+
+        try:
+            genai.configure(api_key=self.api_key)
+            model = genai.GenerativeModel(
+                model_name=self.model,
+                system_instruction=self._build_system_prompt(),
+            )
+            user_prompt = self._build_user_prompt(meta, missing_list, strong_relations)
+            response = model.generate_content(
+                user_prompt,
+                generation_config=genai.GenerationConfig(
+                    max_output_tokens=300,
+                    temperature=0.4,
+                ),
+            )
+            return response.text.strip()
+        except Exception as e:
+            return f"⚠️  Google Gemini API error: {e}"

datapilot/ai/providers/groq_provider.py

@@ -0,0 +1,52 @@
+from ..base import BaseAIProvider
+from typing import Dict, Any, List
+
+
+class GroqProvider(BaseAIProvider):
+    """Groq cloud AI provider — ultra-fast inference (llama3-70b, mixtral, gemma2, etc.)
+
+    Groq offers a generous free tier and is significantly faster than OpenAI
+    for real-time use cases. Uses an OpenAI-compatible API.
+
+    Requires the `groq` package: `pip install groq`
+    Only dataset metadata (stats, null counts) is sent — never raw data rows.
+
+    Usage:
+        dp.analyze(df, use_ai=True, ai_provider="groq",
+                   ai_model="llama3-70b-8192", api_key="gsk_...")
+
+    Popular free models:
+        - llama3-70b-8192    (recommended — best quality)
+        - llama3-8b-8192     (fastest)
+        - mixtral-8x7b-32768 (long context)
+        - gemma2-9b-it       (Google's Gemma via Groq)
+    """
+
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        try:
+            from groq import Groq
+        except ImportError:
+            return (
+                "⚠️  Groq package not installed. "
+                "Run: pip install groq"
+            )
+
+        try:
+            client = Groq(api_key=self.api_key)
+            response = client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": self._build_system_prompt()},
+                    {"role": "user",   "content": self._build_user_prompt(meta, missing_list, strong_relations)},
+                ],
+                max_tokens=300,
+                temperature=0.4,
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"⚠️  Groq API error: {e}"

datapilot/ai/providers/ollama_provider.py

@@ -0,0 +1,35 @@
+from ..base import BaseAIProvider
+from typing import Dict, Any, List
+
+
+class OllamaProvider(BaseAIProvider):
+    """Local Ollama AI provider — fully private, no internet required.
+
+    Communicates with a locally running Ollama daemon. Raw data rows
+    are never transmitted; only metadata summaries are sent.
+
+    Usage:
+        dp.analyze(df, use_ai=True, ai_provider="ollama", ai_model="llama3")
+    """
+
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        try:
+            import ollama
+            response = ollama.chat(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": self._build_system_prompt()},
+                    {"role": "user",   "content": self._build_user_prompt(meta, missing_list, strong_relations)},
+                ],
+            )
+            return response["message"]["content"].strip()
+        except Exception as e:
+            return (
+                f"⚠️  Could not reach Ollama. Ensure it is running (`ollama serve`). "
+                f"Error: {e}"
+            )

datapilot/ai/providers/openai_provider.py

@@ -0,0 +1,43 @@
+from ..base import BaseAIProvider
+from typing import Dict, Any, List
+
+
+class OpenAIProvider(BaseAIProvider):
+    """OpenAI cloud AI provider (GPT-4o, GPT-4, GPT-3.5-turbo, etc.)
+
+    Requires the `openai` package: `pip install openai`
+    Only dataset metadata (stats, null counts) is sent — never raw data rows.
+
+    Usage:
+        dp.analyze(df, use_ai=True, ai_provider="openai",
+                   ai_model="gpt-4o", api_key="sk-...")
+    """
+
+    def generate(
+        self,
+        meta: Dict[str, Any],
+        missing_list: List[Dict[str, Any]],
+        strong_relations: List[str],
+    ) -> str:
+        try:
+            from openai import OpenAI
+        except ImportError:
+            return (
+                "⚠️  OpenAI package not installed. "
+                "Run: pip install openai"
+            )
+
+        try:
+            client = OpenAI(api_key=self.api_key)
+            response = client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": self._build_system_prompt()},
+                    {"role": "user",   "content": self._build_user_prompt(meta, missing_list, strong_relations)},
+                ],
+                max_tokens=300,
+                temperature=0.4,
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"⚠️  OpenAI API error: {e}"

datapilot/analysis/auto_clean.py

@@ -0,0 +1,131 @@
+from ..utils.validation import ensure_polars
+import polars as pl
+import pandas as pd
+from typing import Union, Tuple, Dict, Any, List
+
+
+def auto_clean(
+    df: Union[pd.DataFrame, pl.DataFrame],
+    drop_null_threshold: float = 0.6,
+    impute_strategy: str = "auto",
+    drop_id_columns: bool = True,
+    drop_constant_columns: bool = True,
+) -> Tuple[Union[pd.DataFrame, pl.DataFrame], List[Dict[str, Any]]]:
+    """Automatically cleans a dataset by applying common preprocessing steps.
+
+    Performs the following operations (in order):
+    1. Drops constant columns (zero variance)
+    2. Drops ID-like columns (100% unique values)
+    3. Drops columns exceeding the null threshold
+    4. Imputes remaining numeric nulls with median
+    5. Imputes remaining categorical/string nulls with mode
+
+    Args:
+        df: Input DataFrame (Pandas or Polars).
+        drop_null_threshold: Columns with null % above this are dropped (default: 0.60 = 60%).
+        impute_strategy: 'auto' (median for numeric, mode for categorical), 'median', or 'mode'.
+        drop_id_columns: Whether to drop columns where all values are unique (default: True).
+        drop_constant_columns: Whether to drop columns with a single unique value (default: True).
+
+    Returns:
+        Tuple of (cleaned_df, change_log) where change_log is a list of dicts
+        describing every action taken. The returned DataFrame matches the input type.
+    """
+    local_df, original_engine = ensure_polars(df)
+    total_rows = local_df.height
+    change_log: List[Dict[str, Any]] = []
+    cols_to_drop: List[str] = []
+
+    print("=" * 58)
+    print("         DATAPILOT AUTO-CLEAN ENGINE              ")
+    print("=" * 58)
+    print(f"  Input:  {total_rows:,} rows × {local_df.width} columns")
+
+    null_counts = local_df.null_count()
+
+    for col in local_df.columns:
+        series  = local_df[col]
+        n_unique = series.n_unique()
+        null_ct  = null_counts[col][0]
+        null_pct = null_ct / total_rows if total_rows > 0 else 0
+
+        # ── 1. Constant columns ──────────────────────────────────────────────
+        if drop_constant_columns and n_unique == 1:
+            cols_to_drop.append(col)
+            change_log.append({
+                "action": "DROPPED",
+                "column": col,
+                "reason": "Constant column — all values are identical (zero variance)",
+            })
+            print(f"\n  🗑️  DROP   '{col}'  →  constant column (1 unique value)")
+            continue
+
+        # ── 2. ID-like columns ───────────────────────────────────────────────
+        if drop_id_columns and n_unique == total_rows and total_rows > 10:
+            cols_to_drop.append(col)
+            change_log.append({
+                "action": "DROPPED",
+                "column": col,
+                "reason": "ID/key column — 100% unique values carry no predictive signal",
+            })
+            print(f"\n  🗑️  DROP   '{col}'  →  ID-like column (all {n_unique} values unique)")
+            continue
+
+        # ── 3. High-null columns ──────────────────────────────────────────────
+        if null_pct >= drop_null_threshold:
+            cols_to_drop.append(col)
+            change_log.append({
+                "action": "DROPPED",
+                "column": col,
+                "reason": f"High null rate ({null_pct*100:.1f}%) exceeds threshold ({drop_null_threshold*100:.0f}%)",
+            })
+            print(f"\n  🗑️  DROP   '{col}'  →  {null_pct*100:.1f}% nulls (above {drop_null_threshold*100:.0f}% threshold)")
+
+    # Apply drops
+    clean_df = local_df.drop(cols_to_drop) if cols_to_drop else local_df.clone()
+
+    # ── 4 & 5. Impute remaining nulls ─────────────────────────────────────────
+    impute_exprs = []
+    for col in clean_df.columns:
+        dtype    = clean_df[col].dtype
+        null_ct  = clean_df[col].null_count()
+        if null_ct == 0:
+            continue
+
+        if dtype.is_numeric():
+            if impute_strategy in ("auto", "median"):
+                median_val = clean_df[col].median()
+                if median_val is not None:
+                    impute_exprs.append(pl.col(col).fill_null(median_val))
+                    change_log.append({
+                        "action": "IMPUTED",
+                        "column": col,
+                        "reason": f"Filled {null_ct} null(s) with median ({round(median_val, 4)})",
+                    })
+                    print(f"\n  🔧  IMPUTE '{col}'  →  filled {null_ct} null(s) with median={round(median_val, 4)}")
+        else:
+            if impute_strategy in ("auto", "mode"):
+                mode_series = clean_df[col].drop_nulls().value_counts().sort("count", descending=True)
+                if mode_series.height > 0:
+                    mode_val = str(mode_series["value"][0])
+                    impute_exprs.append(pl.col(col).fill_null(mode_val))
+                    change_log.append({
+                        "action": "IMPUTED",
+                        "column": col,
+                        "reason": f"Filled {null_ct} null(s) with mode ('{mode_val}')",
+                    })
+                    print(f"\n  🔧  IMPUTE '{col}'  →  filled {null_ct} null(s) with mode='{mode_val}'")
+
+    if impute_exprs:
+        clean_df = clean_df.with_columns(impute_exprs)
+
+    print("\n" + "-" * 58)
+    print(f"  Output: {clean_df.height:,} rows × {clean_df.width} columns")
+    dropped = local_df.width - clean_df.width
+    imputed = sum(1 for c in change_log if c["action"] == "IMPUTED")
+    print(f"  ✅ {dropped} column(s) dropped  |  {imputed} column(s) imputed")
+    print("=" * 58)
+
+    # Return in original format
+    result_df = clean_df.to_pandas() if original_engine == "pandas" else clean_df
+    return result_df, change_log