datadoc-cli 0.2.0__tar.gz → 0.3.0__tar.gz

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 narain-karti
+Copyright (c) 2026 narain-karti
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datadoc-cli
-Version: 0.2.0
+Version: 0.3.0
 Summary: The Open Source Operating System for Dataset Engineering.
 Project-URL: Homepage, https://github.com/narain-karti/DATADOC
 Project-URL: Repository, https://github.com/narain-karti/DATADOC
@@ -33,7 +33,7 @@ Requires-Dist: ruff>=0.4.0; extra == 'dev'
 Description-Content-Type: text/markdown
 
 <p align="center">
-  <pre>
+  <pre align="center">
  ____    _  _____  _    ____   ___   ____
 |  _ \  / \|_   _|/ \  |  _ \ / _ \ / ___|
 | | | |/ _ \ | | / _ \ | | | | | | | |
@@ -44,6 +44,10 @@ Description-Content-Type: text/markdown
 
 <h3 align="center">The Open Source Operating System for Dataset Engineering.</h3>
 
+<p align="center">
+  <a href="https://narain-karti.github.io/DATADOC/"><b>&#x1F4D6; View Official Documentation Website</b></a>
+</p>
+
 <p align="center">
   <a href="https://pypi.org/project/datadoc-cli/"><img alt="PyPI version" src="https://img.shields.io/pypi/v/datadoc-cli.svg"></a>
   <a href="https://pypi.org/project/datadoc-cli/"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/datadoc-cli.svg"></a>
@@ -51,14 +55,14 @@ Description-Content-Type: text/markdown
 </p>
 
 <p align="center">
-  <a href="#installation">Install</a> |
-  <a href="#why-datadoc">Why DATADOC?</a> |
-  <a href="#quick-start">Quick Start</a> |
-  <a href="#cli-commands">CLI Commands</a> |
-  <a href="#architecture--plugins">Architecture</a>
+  <a href="#installation"><b>Install</b></a> &bull;
+  <a href="#why-datadoc"><b>Why DATADOC?</b></a> &bull;
+  <a href="#quick-start"><b>Quick Start</b></a> &bull;
+  <a href="#cli-commands"><b>CLI Commands</b></a> &bull;
+  <a href="#architecture--plugins"><b>Architecture</b></a>
 </p>
 
----
+<hr>
 
 ## 🚀 What is DATADOC?
 

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <pre>
+  <pre align="center">
  ____    _  _____  _    ____   ___   ____
 |  _ \  / \|_   _|/ \  |  _ \ / _ \ / ___|
 | | | |/ _ \ | | / _ \ | | | | | | | |
@@ -10,6 +10,10 @@
 
 <h3 align="center">The Open Source Operating System for Dataset Engineering.</h3>
 
+<p align="center">
+  <a href="https://narain-karti.github.io/DATADOC/"><b>&#x1F4D6; View Official Documentation Website</b></a>
+</p>
+
 <p align="center">
   <a href="https://pypi.org/project/datadoc-cli/"><img alt="PyPI version" src="https://img.shields.io/pypi/v/datadoc-cli.svg"></a>
   <a href="https://pypi.org/project/datadoc-cli/"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/datadoc-cli.svg"></a>
@@ -17,14 +21,14 @@
 </p>
 
 <p align="center">
-  <a href="#installation">Install</a> |
-  <a href="#why-datadoc">Why DATADOC?</a> |
-  <a href="#quick-start">Quick Start</a> |
-  <a href="#cli-commands">CLI Commands</a> |
-  <a href="#architecture--plugins">Architecture</a>
+  <a href="#installation"><b>Install</b></a> &bull;
+  <a href="#why-datadoc"><b>Why DATADOC?</b></a> &bull;
+  <a href="#quick-start"><b>Quick Start</b></a> &bull;
+  <a href="#cli-commands"><b>CLI Commands</b></a> &bull;
+  <a href="#architecture--plugins"><b>Architecture</b></a>
 </p>
 
----
+<hr>
 
 ## 🚀 What is DATADOC?
 

datadoc_cli-0.3.0/comprehensive_test.csv

@@ -0,0 +1,11 @@
+ID,Name,Age,Income,Score,City,Membership_Level,Signup_Date
+1,Alice,25,50000,0.85,New York,Silver,2023-01-15 10:00:00
+2,Bob,30,60000,0.90,London,Gold,2023-02-20 14:30:00
+3,Charlie,,55000,0.75,,Bronze,2023-03-05 09:15:00
+4,David,45,10000000,0.92,Paris,Gold,2023-04-10 16:45:00
+5,Eve,28,45000,0.88,New York,Silver,2023-05-12 11:20:00
+6,Frank,35,70000,,London,Silver,2023-06-18 08:00:00
+7,Grace,40,-500000,0.95,New York,Bronze,2023-07-22 13:10:00
+8,Heidi,32,65000,0.82,Paris,Gold,2023-08-30 17:55:00
+9,Ivan,29,52000,0.78,London,,2023-09-14 10:40:00
+10,Judy,38,75000,0.89,New York,Bronze,2023-10-01 15:25:00

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/datadoc/cli/app.py

@@ -11,10 +11,10 @@ from dotenv import load_dotenv
 # Load environment variables (including .env) at startup
 load_dotenv()
 
-# Determine the default model from the environment, fallback to Gemini
-DEFAULT_MODEL = os.getenv("DATADOC_MODEL", "gemini/gemini-2.0-flash")
+# Determine the default model from the environment, fallback to Groq
+DEFAULT_MODEL = os.getenv("DATADOC_MODEL", "groq/llama-3.3-70b-versatile")
 
-from datadoc.core.engine import DATADOC
+from datadoc.core.engine import DATADOC  # noqa: E402
 
 app = typer.Typer(
     help="DATADOC: The Open Source Operating System for Dataset Engineering.",
@@ -55,7 +55,7 @@ def load_dataset(file_path: str) -> DATADOC:
     print_step("[>>]", f"Loading [cyan]{file_path}[/cyan]...")
     try:
         doc = DATADOC(file_path)
-    except Exception as e:
+    except (FileNotFoundError, ValueError, RuntimeError) as e:
         console.print(f"\n  [bold red][X] Failed to read dataset:[/] {e}")
         raise typer.Exit(code=1)
 
@@ -65,23 +65,15 @@ def load_dataset(file_path: str) -> DATADOC:
 
 
 def _get_api_key(model: str) -> str:
-    api_key = None
-    if model.startswith("gemini"):
-        api_key = os.getenv("GEMINI_API_KEY")
-        key_name = "GEMINI_API_KEY"
-    elif model.startswith("gpt") or model.startswith("openai"):
-        api_key = os.getenv("OPENAI_API_KEY")
-        key_name = "OPENAI_API_KEY"
-    elif model.startswith("claude") or model.startswith("anthropic"):
-        api_key = os.getenv("ANTHROPIC_API_KEY")
-        key_name = "ANTHROPIC_API_KEY"
-    elif model.startswith("groq"):
-        api_key = os.getenv("GROQ_API_KEY")
-        key_name = "GROQ_API_KEY"
-    else:
-        api_key = os.getenv("OPENAI_API_KEY") # Default fallback
-        key_name = "API_KEY"
-        
+    key_map = {
+        "gemini": "GEMINI_API_KEY",
+        "gpt": "OPENAI_API_KEY", "openai": "OPENAI_API_KEY",
+        "claude": "ANTHROPIC_API_KEY", "anthropic": "ANTHROPIC_API_KEY",
+        "groq": "GROQ_API_KEY"
+    }
+    key_name = next((v for k, v in key_map.items() if model.startswith(k)), "API_KEY")
+    api_key = os.getenv("OPENAI_API_KEY") if key_name == "API_KEY" else os.getenv(key_name)
+
     if not api_key:
         console.print(f"\n  [bold yellow][!][/bold yellow] {key_name} not found in environment or .env file.")
         api_key = typer.prompt(f"Please enter your {key_name}", hide_input=True)
@@ -149,7 +141,7 @@ def analyze(
             has_issue = p_stats.get("has_scale_issues", False)
             ratio = p_stats.get("scale_ratio", 0)
             if has_issue:
-                table.add_row("Scale Mismatch", f"{ratio}x", f"[yellow][!!] Needs scaling[/yellow]")
+                table.add_row("Scale Mismatch", f"{ratio}x", "[yellow][!!] Needs scaling[/yellow]")
             else:
                 table.add_row("Scale Mismatch", "--", "[green][OK] Balanced[/green]")
 
@@ -389,7 +381,6 @@ def visualize(file_path: str):
     Generates a massive, interactive terminal dashboard comparing the before and after states.
     """
     import plotext as plt
-    import numpy as np
     import sys
     import polars as pl
     
@@ -520,8 +511,8 @@ def report(file_path: str):
         "",
         "## Dataset Overview",
         "",
-        f"| Metric | Value |",
-        f"|--------|-------|",
+        "| Metric | Value |",
+        "|--------|-------|",
         f"| File | `{file_path}` |",
         f"| Rows | {report_data['rows']:,} |",
         f"| Columns | {report_data['cols']} |",
@@ -585,12 +576,7 @@ def plugin_list():
     """
     print_banner()
 
-    # Create a temporary DATADOC instance with a minimal df just to list plugins
-    # We don't need a real file for this
-    import io
-    dummy_csv = b"a,b\n1,2\n"
-    import polars as pl
-    dummy_df = pl.read_csv(dummy_csv)
+
 
     from datadoc.plugins.missing_values import MissingValuePlugin
     from datadoc.plugins.outliers import OutlierPlugin
@@ -784,7 +770,7 @@ Be helpful, analytical, and concise. Format your responses with markdown."""
                                 tool_result = f"Dataset successfully saved to {args['filename']}."
                             else:
                                 tool_result = f"Error: Unknown function {func_name}"
-                        except Exception as e:
+                        except (ValueError, KeyError, RuntimeError) as e:
                             tool_result = f"Error executing {func_name}: {e}"
                             
                         messages.append({
@@ -812,7 +798,7 @@ Be helpful, analytical, and concise. Format your responses with markdown."""
         except KeyboardInterrupt:
             console.print("\n[bold green]Ending chat session. Goodbye![/bold green]\n")
             break
-        except Exception as e:
+        except (KeyboardInterrupt, RuntimeError) as e:
             console.print(f"\n[bold red]Error:[/] {e}")
 
 if __name__ == "__main__":

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/datadoc/core/engine.py

@@ -1,5 +1,5 @@
 import polars as pl
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List
 import json
 from collections import Counter
 import litellm
@@ -26,6 +26,50 @@ class DATADOC:
             ScalingPlugin(),
         ], key=lambda p: p.priority)
 
+    @staticmethod
+    def _detect_column_roles(df: pl.DataFrame) -> Dict[str, str]:
+        """Classify each column as 'id', 'name', 'constant', or 'feature'."""
+        roles = {}
+        for col in df.columns:
+            series = df[col]
+            # Constant columns (single unique value)
+            if series.drop_nulls().n_unique() <= 1:
+                roles[col] = "constant"
+            # Likely ID: integer, all unique, monotonically increasing
+            elif series.dtype.is_integer() and series.drop_nulls().n_unique() == series.drop_nulls().len() and series.drop_nulls().len() > 0:
+                sorted_vals = series.drop_nulls().sort()
+                diffs = sorted_vals.diff().drop_nulls()
+                if diffs.len() > 0 and (diffs == diffs[0]).all():
+                    roles[col] = "id"
+                else:
+                    roles[col] = "feature"
+            # String columns need extra checks
+            elif series.dtype == pl.String and series.drop_nulls().n_unique() == series.drop_nulls().len() and series.drop_nulls().len() > 0:
+                # Check if it's a datetime string before labelling as name
+                try:
+                    parsed = df.select(pl.col(col).str.to_datetime(strict=False))
+                    not_null_ratio = 1 - (parsed[col].null_count() / max(parsed.height, 1))
+                    if not_null_ratio > 0.5:
+                        roles[col] = "feature"  # Datetime string — let DatetimePlugin handle it
+                    else:
+                        roles[col] = "name"
+                except (ValueError, TypeError, pl.exceptions.ComputeError, pl.exceptions.InvalidOperationError):
+                    roles[col] = "name"
+            else:
+                roles[col] = "feature"
+        return roles
+
+    @staticmethod
+    def _drop_constant_columns(df: pl.DataFrame) -> tuple[pl.DataFrame, list[str]]:
+        """Drop columns with zero variance (single unique non-null value)."""
+        to_drop = []
+        for col in df.columns:
+            if df[col].drop_nulls().n_unique() <= 1:
+                to_drop.append(col)
+        if to_drop:
+            df = df.drop(to_drop)
+        return df, to_drop
+
     def analyze(self) -> Dict[str, Any]:
         """Runs the analyze step across all plugins."""
         dtype_counts = Counter([str(dt) for dt in self.df.dtypes])
@@ -89,6 +133,17 @@ class DATADOC:
 
         self._applied_plugins = []
         self._skipped_plugins = []
+        self._dropped_columns = []
+
+        # Phase 0: Drop ID and name columns before any plugin runs
+        roles = self._detect_column_roles(df_transformed)
+        cols_to_drop = [c for c, r in roles.items() if r in ("id", "name")]
+        if cols_to_drop:
+            df_transformed = df_transformed.drop(cols_to_drop)
+            self._dropped_columns.extend(cols_to_drop)
+            if progress_callback:
+                progress_callback("ColumnFilter", "applied",
+                    [f"Dropped non-feature columns: {', '.join(cols_to_drop)}"])
 
         for plugin in self.plugins:
             if progress_callback:
@@ -107,6 +162,14 @@ class DATADOC:
                 if progress_callback:
                     progress_callback(plugin.name, "skipped", [])
 
+        # Phase final: Drop any constant columns created during pipeline
+        df_transformed, dropped_constants = self._drop_constant_columns(df_transformed)
+        if dropped_constants:
+            self._dropped_columns.extend(dropped_constants)
+            if progress_callback:
+                progress_callback("ColumnFilter", "applied",
+                    [f"Dropped constant columns: {', '.join(dropped_constants)}"])
+
         return df_transformed
 
     def compare(self, clean_df: pl.DataFrame) -> Dict[str, Any]:
@@ -230,7 +293,7 @@ Respond in plain text formatted nicely with bullet points and paragraphs where a
             
         try:
             planner_response = self._generate_ai_plan(model, goal, api_key)
-        except Exception as e:
+        except (RuntimeError, ValueError, TypeError) as e:
             if progress_callback:
                 progress_callback("AI Planner", "error", [f"LLM API Error: {e}"])
             raise RuntimeError(f"AI Planner failed: {e}")
@@ -238,6 +301,17 @@ Respond in plain text formatted nicely with bullet points and paragraphs where a
         df_transformed = self.df.clone()
         self._applied_plugins = []
         self._skipped_plugins = []
+        self._dropped_columns = []
+
+        # Phase 0: Drop ID and name columns before any plugin runs
+        roles = self._detect_column_roles(df_transformed)
+        cols_to_drop = [c for c, r in roles.items() if r in ("id", "name")]
+        if cols_to_drop:
+            df_transformed = df_transformed.drop(cols_to_drop)
+            self._dropped_columns.extend(cols_to_drop)
+            if progress_callback:
+                progress_callback("ColumnFilter", "applied",
+                    [f"Dropped non-feature columns: {', '.join(cols_to_drop)}"])
 
         if progress_callback:
             progress_callback("AI Planner", "applied", [f"Generated plan with {len(planner_response.plan)} steps."])
@@ -266,6 +340,14 @@ Respond in plain text formatted nicely with bullet points and paragraphs where a
                 self._skipped_plugins.append(p.name)
                 if progress_callback:
                     progress_callback(p.name, "skipped", ["Skipped by AI Planner"])
+
+        # Phase final: Drop any constant columns created during pipeline
+        df_transformed, dropped_constants = self._drop_constant_columns(df_transformed)
+        if dropped_constants:
+            self._dropped_columns.extend(dropped_constants)
+            if progress_callback:
+                progress_callback("ColumnFilter", "applied",
+                    [f"Dropped constant columns: {', '.join(dropped_constants)}"])
                     
         return df_transformed
 

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/datadoc/plugins/base.py

@@ -21,12 +21,13 @@ class BasePlugin(ABC):
         return 50
 
     @property
-    def supported_datatypes(self) -> list:
-        return ["numeric", "categorical"]
+    def dependencies(self) -> list[str]:
+        """List of plugin names this plugin depends on."""
+        return []
 
     @property
-    def dependencies(self) -> list:
-        """List of plugin names that must run before this one."""
+    def supported_datatypes(self) -> list[str]:
+        """List of datatypes this plugin can handle."""
         return []
 
     @abstractmethod
@@ -49,26 +50,19 @@ class BasePlugin(ABC):
         """Apply the engineering transformation and return the new dataframe."""
         pass
 
-    def validate(self, df: pl.DataFrame) -> bool:
-        """Validate that the transformation produced a valid result."""
-        if df.is_empty():
-            return False
-        return True
-
-    def rollback(self, original_df: pl.DataFrame) -> pl.DataFrame:
-        """Return the original dataframe, undoing any transformation."""
-        return original_df.clone()
-
     def explain(self) -> str:
         """Return a human-readable explanation of what this plugin does."""
         return f"{self.name} (v{self.version}): {self.description}"
 
-    def ai_explain(self, analysis_result: dict, goal: str, llm_reason: str) -> str:
-        """Return a human-readable explanation of why this plugin is applied under AI guidance."""
-        return f"{self.name} - AI Reason: {llm_reason}"
+    def validate(self, df: pl.DataFrame) -> bool:
+        """Validate that the plugin's transformation was applied correctly."""
+        return True
+
+    def rollback(self, df: pl.DataFrame) -> pl.DataFrame:
+        """Rollback the transformation (default: return df unchanged)."""
+        return df
 
     def estimate_runtime(self, df: pl.DataFrame) -> float:
-        """Estimate runtime in seconds based on dataset size."""
-        rows = df.height
-        # Simple linear estimate: ~1ms per 1000 rows
-        return round(rows / 1_000_000, 4)
+        """Estimate runtime in seconds based on dataframe size."""
+        return df.height * df.width * 1e-6
+

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/datadoc/plugins/datetime_feat.py

@@ -12,19 +12,22 @@ class DatetimePlugin(BasePlugin):
 
     @property
     def description(self) -> str:
-        return "Detects datetime-like columns and extracts year, month, day, dayofweek features."
+        return "Detects datetime-like columns and extracts year, month, day, dayofweek, and hour features. Drops constant-value features."
 
     @property
     def priority(self) -> int:
         return 30  # After missing values and outliers, before encoding
 
-    @property
-    def supported_datatypes(self) -> list:
-        return ["datetime"]
-
-    @property
-    def dependencies(self) -> list:
-        return ["MissingValuePlugin"]
+    @staticmethod
+    def _has_time_component(series: pl.Series) -> bool:
+        """Check if a datetime series has meaningful time components (not all midnight)."""
+        times = series.drop_nulls()
+        if times.len() == 0:
+            return False
+        hours = times.dt.hour()
+        minutes = times.dt.minute()
+        # If all hours and minutes are 0, there's no meaningful time info
+        return not ((hours == 0).all() and (minutes == 0).all())
 
     def analyze(self, df: pl.DataFrame) -> dict:
         datetime_cols = []
@@ -39,7 +42,8 @@ class DatetimePlugin(BasePlugin):
                         not_null_ratio = 1 - (parsed[col].null_count() / parsed.height)
                         if not_null_ratio > 0.5:
                             datetime_cols.append(col)
-                except Exception:
+                except (ValueError, TypeError, pl.exceptions.ComputeError, pl.exceptions.InvalidOperationError):
+                    # Column is not a valid datetime string; skipping
                     pass
 
         return {
@@ -53,7 +57,8 @@ class DatetimePlugin(BasePlugin):
             cols = analysis_result["datetime_columns"]
             recs.append(
                 f"Datetime columns detected: {', '.join(cols)}. "
-                f"Recommendation: Extract year, month, day, day_of_week features and drop original."
+                f"Recommendation: Extract year, month, day, day_of_week, hour features. "
+                f"Drop original column and any constant features."
             )
         return recs
 
@@ -67,12 +72,23 @@ for col in datetime_cols:
     if df[col].dtype == pl.String:
         df = df.with_columns(pl.col(col).str.to_datetime(strict=False).alias(col))
     
-    df = df.with_columns([
+    new_cols = [
         pl.col(col).dt.year().alias(col + '_year'),
         pl.col(col).dt.month().alias(col + '_month'),
         pl.col(col).dt.day().alias(col + '_day'),
-        pl.col(col).dt.weekday().alias(col + '_dayofweek')
-    ]).drop(col)"""
+        pl.col(col).dt.weekday().alias(col + '_dayofweek'),
+    ]
+    # Add hour if time component exists
+    hours = df[col].dt.hour()
+    if not ((hours == 0).all()):
+        new_cols.append(pl.col(col).dt.hour().alias(col + '_hour'))
+    
+    df = df.with_columns(new_cols).drop(col)
+    
+    # Drop any constant features (e.g. year when all dates are same year)
+    for c in [c for c in df.columns if c.startswith(col + '_')]:
+        if df[c].drop_nulls().n_unique() <= 1:
+            df = df.drop(c)"""
 
     def apply(self, df: pl.DataFrame) -> pl.DataFrame:
         df_clean = df.clone()
@@ -81,22 +97,32 @@ for col in datetime_cols:
         for col in dt_cols:
             if df_clean[col].dtype == pl.String:
                 df_clean = df_clean.with_columns(pl.col(col).str.to_datetime(strict=False).alias(col))
-                
-            df_clean = df_clean.with_columns([
+
+            new_cols = [
                 pl.col(col).dt.year().alias(col + '_year'),
                 pl.col(col).dt.month().alias(col + '_month'),
                 pl.col(col).dt.day().alias(col + '_day'),
-                pl.col(col).dt.weekday().alias(col + '_dayofweek')
-            ]).drop(col)
+                pl.col(col).dt.weekday().alias(col + '_dayofweek'),
+            ]
 
-        return df_clean
+            # Add hour if time component exists
+            if self._has_time_component(df_clean[col]):
+                new_cols.append(pl.col(col).dt.hour().alias(col + '_hour'))
+
+            df_clean = df_clean.with_columns(new_cols).drop(col)
 
-    def validate(self, df: pl.DataFrame) -> bool:
-        return True
+            # Drop constant datetime features (e.g. year=2023 for all rows)
+            for c in [c for c in df_clean.columns if c.startswith(col + '_')]:
+                if df_clean[c].drop_nulls().n_unique() <= 1:
+                    df_clean = df_clean.drop(c)
+
+        return df_clean
 
     def explain(self) -> str:
         return (
             "DatetimePlugin detects columns containing dates/times and extracts "
-            "year, month, day, and day_of_week as new numeric features. "
-            "The original datetime column is dropped."
+            "year, month, day, day_of_week, and hour as new numeric features. "
+            "The original datetime column is dropped, along with any constant features "
+            "(e.g., year when all dates are in the same year)."
         )
+

datadoc_cli-0.3.0/datadoc/plugins/encoders.py

@@ -0,0 +1,95 @@
+import polars as pl
+from datadoc.plugins.base import BasePlugin
+
+class CategoricalEncoderPlugin(BasePlugin):
+    @property
+    def name(self) -> str:
+        return "CategoricalEncoderPlugin"
+
+    @property
+    def version(self) -> str:
+        return "0.1.0"
+
+    @property
+    def description(self) -> str:
+        return "Drops high-cardinality identifier columns and applies One-Hot Encoding (drop_first=True) to low-cardinality categoricals."
+
+    @property
+    def priority(self) -> int:
+        return 40  # After missing values and outliers
+
+    def analyze(self, df: pl.DataFrame) -> dict:
+        str_cols = [col for col in df.columns if df[col].dtype == pl.String]
+
+        # Identifier columns: every value is unique (e.g. Name, Email)
+        id_cols = [col for col in str_cols if df[col].drop_nulls().n_unique() >= df.height]
+
+        # Encodable categorical: low cardinality (2-9 unique), not an identifier
+        valid_cats = [
+            col for col in str_cols
+            if col not in id_cols and 1 < df[col].n_unique() < 10
+        ]
+        cardinality = {col: df[col].n_unique() for col in valid_cats}
+
+        return {
+            "has_categorical": len(valid_cats) > 0 or len(id_cols) > 0,
+            "categorical_columns": valid_cats,
+            "identifier_columns": id_cols,
+            "cardinality": cardinality,
+        }
+
+    def recommend(self, analysis_result: dict) -> list[str]:
+        recs = []
+        id_cols = analysis_result.get("identifier_columns", [])
+        if id_cols:
+            recs.append(
+                f"Identifier columns detected: {', '.join(id_cols)}. "
+                f"Recommendation: Drop these (every value is unique, no ML signal)."
+            )
+        if analysis_result.get("categorical_columns"):
+            info = analysis_result.get("cardinality", {})
+            detail = ", ".join([f"{c} ({v} unique)" for c, v in info.items()])
+            recs.append(
+                f"Categorical columns found: {detail}. "
+                f"Recommendation: Apply One-Hot Encoding."
+            )
+        return recs
+
+    def generate_code(self, analysis_result: dict) -> str:
+        lines = []
+        id_cols = analysis_result.get("identifier_columns", [])
+        if id_cols:
+            lines.append(f"# Drop identifier columns")
+            lines.append(f"df = df.drop({id_cols})")
+
+        cat_cols = analysis_result.get("categorical_columns", [])
+        if cat_cols:
+            cols_str = str(cat_cols)
+            lines.append(f"# Categorical Encoding (One-Hot)")
+            lines.append(f"cat_cols = {cols_str}")
+            lines.append(f"df = df.to_dummies(columns=cat_cols, drop_first=True)")
+
+        return "\n".join(lines)
+
+    def apply(self, df: pl.DataFrame) -> pl.DataFrame:
+        df_clean = df.clone()
+        analysis = self.analyze(df_clean)
+
+        # Drop identifier columns first
+        id_cols = analysis.get("identifier_columns", [])
+        if id_cols:
+            df_clean = df_clean.drop(id_cols)
+
+        # One-hot encode low-cardinality categoricals
+        cat_cols = analysis.get("categorical_columns", [])
+        if cat_cols:
+            df_clean = df_clean.to_dummies(columns=cat_cols, drop_first=True)
+        return df_clean
+
+    def explain(self) -> str:
+        return (
+            "CategoricalEncoderPlugin detects text/string columns. "
+            "High-cardinality identifier columns (every value unique) are dropped. "
+            "Low-cardinality columns (< 10 unique values) are One-Hot Encoded with drop_first=True."
+        )
+

{datadoc_cli-0.2.0 → datadoc_cli-0.3.0}/datadoc/plugins/missing_values.py

@@ -18,9 +18,7 @@ class MissingValuePlugin(BasePlugin):
     def priority(self) -> int:
         return 10  # Should run first
 
-    @property
-    def supported_datatypes(self) -> list:
-        return ["numeric", "categorical"]
+
 
     def analyze(self, df: pl.DataFrame) -> dict:
         cols_with_missing = {c: df[c].null_count() for c in df.columns if df[c].null_count() > 0}
@@ -64,8 +62,7 @@ for col in df.columns:
                     df_clean = df_clean.with_columns(pl.col(col).fill_null(pl.col(col).drop_nulls().mode().first()))
         return df_clean
 
-    def validate(self, df: pl.DataFrame) -> bool:
-        return bool(sum(df[c].null_count() for c in df.columns) == 0)
+
 
     def explain(self) -> str:
         return (