datadoc-cli 0.1.0__py3-none-any.whl

datadoc/plugins/base.py

@@ -0,0 +1,70 @@
+from abc import ABC, abstractmethod
+import polars as pl
+
+class BasePlugin(ABC):
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "No description provided."
+
+    @property
+    def priority(self) -> int:
+        """Lower number = runs first. Default is 50."""
+        return 50
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["numeric", "categorical"]
+
+    @property
+    def dependencies(self) -> list:
+        """List of plugin names that must run before this one."""
+        return []
+
+    @abstractmethod
+    def analyze(self, df: pl.DataFrame) -> dict:
+        """Analyze the dataframe and return metadata/flags if the plugin should run."""
+        pass
+
+    @abstractmethod
+    def recommend(self, analysis_result: dict) -> list[str]:
+        """Return a list of recommendations based on the analysis."""
+        pass
+
+    @abstractmethod
+    def generate_code(self, analysis_result: dict) -> str:
+        """Return the Python code string to replicate this plugin's transformation."""
+        pass
+
+    @abstractmethod
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        """Apply the engineering transformation and return the new dataframe."""
+        pass
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        """Validate that the transformation produced a valid result."""
+        if df.is_empty():
+            return False
+        return True
+
+    def rollback(self, original_df: pl.DataFrame) -> pl.DataFrame:
+        """Return the original dataframe, undoing any transformation."""
+        return original_df.clone()
+
+    def explain(self) -> str:
+        """Return a human-readable explanation of what this plugin does."""
+        return f"{self.name} (v{self.version}): {self.description}"
+
+    def estimate_runtime(self, df: pl.DataFrame) -> float:
+        """Estimate runtime in seconds based on dataset size."""
+        rows = df.height
+        # Simple linear estimate: ~1ms per 1000 rows
+        return round(rows / 1_000_000, 4)

datadoc/plugins/datetime_feat.py

@@ -0,0 +1,102 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class DatetimePlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "DatetimePlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Detects datetime-like columns and extracts year, month, day, dayofweek features."
+
+    @property
+    def priority(self) -> int:
+        return 30  # After missing values and outliers, before encoding
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["datetime"]
+
+    @property
+    def dependencies(self) -> list:
+        return ["MissingValuePlugin"]
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        datetime_cols = []
+        for col in df.columns:
+            if df[col].dtype in [pl.Date, pl.Datetime]:
+                datetime_cols.append(col)
+            elif df[col].dtype == pl.String:
+                try:
+                    parsed = df.select(pl.col(col).str.to_datetime(strict=False))
+                    # if more than 50% are successfully parsed
+                    if parsed.height > 0:
+                        not_null_ratio = 1 - (parsed[col].null_count() / parsed.height)
+                        if not_null_ratio > 0.5:
+                            datetime_cols.append(col)
+                except Exception:
+                    pass
+
+        return {
+            "has_datetime": len(datetime_cols) > 0,
+            "datetime_columns": datetime_cols,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        if analysis_result.get("has_datetime"):
+            cols = analysis_result["datetime_columns"]
+            recs.append(
+                f"Datetime columns detected: {', '.join(cols)}. "
+                f"Recommendation: Extract year, month, day, day_of_week features and drop original."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        if not analysis_result.get("has_datetime"):
+            return ""
+        cols_str = str(analysis_result.get("datetime_columns", []))
+        return f"""# Datetime Feature Extraction
+datetime_cols = {cols_str}
+for col in datetime_cols:
+    if df[col].dtype == pl.String:
+        df = df.with_columns(pl.col(col).str.to_datetime(strict=False).alias(col))
+    
+    df = df.with_columns([
+        pl.col(col).dt.year().alias(col + '_year'),
+        pl.col(col).dt.month().alias(col + '_month'),
+        pl.col(col).dt.day().alias(col + '_day'),
+        pl.col(col).dt.weekday().alias(col + '_dayofweek')
+    ]).drop(col)"""
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        dt_cols = self.analyze(df_clean).get("datetime_columns", [])
+
+        for col in dt_cols:
+            if df_clean[col].dtype == pl.String:
+                df_clean = df_clean.with_columns(pl.col(col).str.to_datetime(strict=False).alias(col))
+                
+            df_clean = df_clean.with_columns([
+                pl.col(col).dt.year().alias(col + '_year'),
+                pl.col(col).dt.month().alias(col + '_month'),
+                pl.col(col).dt.day().alias(col + '_day'),
+                pl.col(col).dt.weekday().alias(col + '_dayofweek')
+            ]).drop(col)
+
+        return df_clean
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        return True
+
+    def explain(self) -> str:
+        return (
+            "DatetimePlugin detects columns containing dates/times and extracts "
+            "year, month, day, and day_of_week as new numeric features. "
+            "The original datetime column is dropped."
+        )

datadoc/plugins/encoders.py

@@ -0,0 +1,73 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class CategoricalEncoderPlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "CategoricalEncoderPlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Detects categorical columns and applies One-Hot Encoding (drop_first=True)."
+
+    @property
+    def priority(self) -> int:
+        return 40  # After missing values and outliers
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["categorical"]
+
+    @property
+    def dependencies(self) -> list:
+        return ["MissingValuePlugin"]
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        cat_cols = [col for col in df.columns if df[col].dtype == pl.String]
+        valid_cats = [col for col in cat_cols if df[col].n_unique() < 10 and df[col].n_unique() > 1]
+        cardinality = {col: df[col].n_unique() for col in valid_cats}
+
+        return {
+            "has_categorical": len(valid_cats) > 0,
+            "categorical_columns": valid_cats,
+            "cardinality": cardinality,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        if analysis_result.get("has_categorical"):
+            info = analysis_result.get("cardinality", {})
+            detail = ", ".join([f"{c} ({v} unique)" for c, v in info.items()])
+            recs.append(
+                f"Categorical columns found: {detail}. "
+                f"Recommendation: Apply One-Hot Encoding."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        if not analysis_result.get("has_categorical"):
+            return ""
+        cols_str = str(analysis_result.get("categorical_columns", []))
+        return f"""# Categorical Encoding (One-Hot)
+cat_cols = {cols_str}
+df = df.to_dummies(columns=cat_cols, drop_first=True)"""
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        cat_cols = self.analyze(df_clean).get("categorical_columns", [])
+        if cat_cols:
+            df_clean = df_clean.to_dummies(columns=cat_cols, drop_first=True)
+        return df_clean
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        return True
+
+    def explain(self) -> str:
+        return (
+            "CategoricalEncoderPlugin detects text/string columns with fewer than 10 unique values "
+            "and applies One-Hot Encoding with drop_first=True to avoid multicollinearity."
+        )

datadoc/plugins/missing_values.py

@@ -0,0 +1,74 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class MissingValuePlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "MissingValuePlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Detects and imputes missing values using Median (numeric) and Mode (categorical)."
+
+    @property
+    def priority(self) -> int:
+        return 10  # Should run first
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["numeric", "categorical"]
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        cols_with_missing = {c: df[c].null_count() for c in df.columns if df[c].null_count() > 0}
+        total_missing = sum(cols_with_missing.values())
+        return {
+            "has_missing_values": bool(total_missing > 0),
+            "total_missing": int(total_missing),
+            "columns_affected": cols_with_missing,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        if analysis_result.get("has_missing_values"):
+            total = analysis_result["total_missing"]
+            cols = analysis_result.get("columns_affected", {})
+            col_detail = ", ".join([f"{c} ({v})" for c, v in cols.items()])
+            recs.append(
+                f"Found {total} missing values in: {col_detail}. "
+                f"Recommendation: Impute numeric with Median and categorical with Mode."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        if not analysis_result.get("has_missing_values"):
+            return ""
+        return """# Missing Value Imputation
+for col in df.columns:
+    if df[col].null_count() > 0:
+        if df[col].dtype.is_numeric():
+            df = df.with_columns(pl.col(col).fill_null(pl.col(col).median()))
+        else:
+            df = df.with_columns(pl.col(col).fill_null(pl.col(col).drop_nulls().mode().first()))"""
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        for col in df_clean.columns:
+            if df_clean[col].null_count() > 0:
+                if df_clean[col].dtype.is_numeric():
+                    df_clean = df_clean.with_columns(pl.col(col).fill_null(pl.col(col).median()))
+                else:
+                    df_clean = df_clean.with_columns(pl.col(col).fill_null(pl.col(col).drop_nulls().mode().first()))
+        return df_clean
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        return bool(sum(df[c].null_count() for c in df.columns) == 0)
+
+    def explain(self) -> str:
+        return (
+            "MissingValuePlugin fills missing numeric values with the column median "
+            "and missing categorical values with the column mode (most frequent value)."
+        )

datadoc/plugins/outliers.py

@@ -0,0 +1,93 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class OutlierPlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "OutlierPlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Detects outliers using IQR and clips extreme values at 5th/95th percentiles."
+
+    @property
+    def priority(self) -> int:
+        return 20  # After missing values
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["numeric"]
+
+    @property
+    def dependencies(self) -> list:
+        return ["MissingValuePlugin"]
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        outlier_info = {}
+        num_cols = [c for c in df.columns if df[c].dtype.is_numeric()]
+
+        for col in num_cols:
+            Q1 = df[col].quantile(0.25)
+            Q3 = df[col].quantile(0.75)
+            if Q1 is None or Q3 is None:
+                continue
+            IQR = Q3 - Q1
+            lower = Q1 - 1.5 * IQR
+            upper = Q3 + 1.5 * IQR
+            
+            # Count outliers
+            count = df.select(((pl.col(col) < lower) | (pl.col(col) > upper)).sum())[col][0]
+            if count and count > 0:
+                outlier_info[col] = int(count)
+
+        return {
+            "has_outliers": len(outlier_info) > 0,
+            "outlier_columns": list(outlier_info.keys()),
+            "outlier_counts": outlier_info,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        if analysis_result.get("has_outliers"):
+            info = analysis_result.get("outlier_counts", {})
+            detail = ", ".join([f"{c} ({v} outliers)" for c, v in info.items()])
+            recs.append(
+                f"Outliers detected in: {detail}. "
+                f"Recommendation: Clip values at 5th and 95th percentiles."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        if not analysis_result.get("has_outliers"):
+            return ""
+        cols_str = str(analysis_result.get("outlier_columns", []))
+        return f"""# Outlier Clipping
+outlier_cols = {cols_str}
+for col in outlier_cols:
+    lower = df[col].quantile(0.05)
+    upper = df[col].quantile(0.95)
+    if lower is not None and upper is not None:
+        df = df.with_columns(pl.col(col).clip(lower, upper))"""
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        outlier_cols = self.analyze(df_clean).get("outlier_columns", [])
+        for col in outlier_cols:
+            lower = df_clean[col].quantile(0.05)
+            upper = df_clean[col].quantile(0.95)
+            if lower is not None and upper is not None:
+                df_clean = df_clean.with_columns(pl.col(col).clip(lower, upper))
+        return df_clean
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        return True
+
+    def explain(self) -> str:
+        return (
+            "OutlierPlugin detects statistical outliers using the IQR (Interquartile Range) "
+            "method and clips extreme values to the 5th and 95th percentile boundaries."
+        )

datadoc/plugins/scaling.py

@@ -0,0 +1,103 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class ScalingPlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "ScalingPlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Detects numeric columns with vastly different scales and applies StandardScaler normalization."
+
+    @property
+    def priority(self) -> int:
+        return 45  # Near the end, after encoding
+
+    @property
+    def supported_datatypes(self) -> list:
+        return ["numeric"]
+
+    @property
+    def dependencies(self) -> list:
+        return ["MissingValuePlugin", "OutlierPlugin"]
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        num_cols = [c for c in df.columns if df[c].dtype.is_numeric()]
+        if len(num_cols) < 2:
+            return {"has_scale_issues": False, "columns_to_scale": []}
+
+        stds = {}
+        for col in num_cols:
+            std = df[col].std()
+            if std is not None:
+                stds[col] = std
+                
+        if not stds:
+            return {"has_scale_issues": False, "columns_to_scale": []}
+
+        min_std = min(stds.values())
+        max_std = max(stds.values())
+
+        if min_std == 0:
+            cols_to_scale = [c for c, s in stds.items() if s > 0]
+            ratio = float('inf')
+        else:
+            ratio = max_std / min_std
+            cols_to_scale = num_cols if ratio > 10 else []
+
+        return {
+            "has_scale_issues": len(cols_to_scale) > 0,
+            "columns_to_scale": cols_to_scale,
+            "scale_ratio": round(ratio, 2) if ratio != float('inf') else ratio,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        if analysis_result.get("has_scale_issues"):
+            ratio = analysis_result.get("scale_ratio", 0)
+            cols = analysis_result["columns_to_scale"]
+            recs.append(
+                f"Numeric columns have a scale ratio of {ratio}x across {len(cols)} columns. "
+                f"Recommendation: Apply Standard Scaling (zero mean, unit variance)."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        if not analysis_result.get("has_scale_issues"):
+            return ""
+        cols_str = str(analysis_result.get("columns_to_scale", []))
+        return f"""# Standard Scaling
+scale_cols = {cols_str}
+exprs = [((pl.col(c) - pl.col(c).mean()) / pl.col(c).std()).alias(c) for c in scale_cols]
+if exprs:
+    df = df.with_columns(exprs)"""
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        cols_to_scale = self.analyze(df_clean).get("columns_to_scale", [])
+        
+        exprs = []
+        for col in cols_to_scale:
+            exprs.append(
+                ((pl.col(col) - pl.col(col).mean()) / pl.col(col).std()).alias(col)
+            )
+            
+        if exprs:
+            df_clean = df_clean.with_columns(exprs)
+            
+        return df_clean
+
+    def validate(self, df: pl.DataFrame) -> bool:
+        return True
+
+    def explain(self) -> str:
+        return (
+            "ScalingPlugin detects when numeric columns have vastly different scales "
+            "(std ratio > 10x) and applies Standard Scaling: (value - mean) / std, "
+            "resulting in zero-mean, unit-variance features."
+        )