recursive-cleaner 0.8.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

backends/__init__.py

@@ -1,5 +1,6 @@
 """Backend implementations for Recursive Data Cleaner."""
 
 from .mlx_backend import MLXBackend
+from .openai_backend import OpenAIBackend
 
-__all__ = ["MLXBackend"]
+__all__ = ["MLXBackend", "OpenAIBackend"]

backends/openai_backend.py

@@ -0,0 +1,71 @@
+"""OpenAI-compatible backend for Recursive Data Cleaner."""
+
+import os
+
+
+class OpenAIBackend:
+    """
+    OpenAI-compatible backend implementation.
+
+    Works with OpenAI API, LM Studio, Ollama, and other OpenAI-compatible servers.
+    Conforms to the LLMBackend protocol.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.7,
+    ):
+        """
+        Initialize the OpenAI backend.
+
+        Args:
+            model: Model name (e.g., "gpt-4o", "gpt-3.5-turbo")
+            api_key: API key (defaults to OPENAI_API_KEY env var, or "not-needed" for local)
+            base_url: API base URL (defaults to OpenAI's API)
+            max_tokens: Maximum tokens to generate
+            temperature: Sampling temperature
+        """
+        try:
+            import openai
+        except ImportError:
+            raise ImportError(
+                "OpenAI SDK not installed. Install with: pip install openai"
+            )
+
+        self.model = model
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+
+        # Resolve API key: explicit > env var > "not-needed" for local servers
+        if api_key is not None:
+            resolved_key = api_key
+        else:
+            resolved_key = os.environ.get("OPENAI_API_KEY", "not-needed")
+
+        # Create client
+        self._client = openai.OpenAI(
+            api_key=resolved_key,
+            base_url=base_url,
+        )
+
+    def generate(self, prompt: str) -> str:
+        """
+        Generate a response from the LLM.
+
+        Args:
+            prompt: The input prompt
+
+        Returns:
+            The generated text response
+        """
+        response = self._client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+        )
+        return response.choices[0].message.content or ""

recursive_cleaner/__init__.py

@@ -1,5 +1,6 @@
 """Recursive Data Cleaner - LLM-powered incremental data cleaning pipeline."""
 
+from recursive_cleaner.apply import apply_cleaning
 from recursive_cleaner.cleaner import DataCleaner
 from recursive_cleaner.context import build_context
 from recursive_cleaner.dependencies import resolve_dependencies
@@ -24,6 +25,7 @@ from recursive_cleaner.tui import HAS_RICH, TUIRenderer
 from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
 
 __all__ = [
+    "apply_cleaning",
     "CleanerError",
     "ParseError",
     "MaxIterationsError",

recursive_cleaner/__main__.py

@@ -0,0 +1,8 @@
+"""Entry point for python -m recursive_cleaner."""
+
+import sys
+
+from recursive_cleaner.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

recursive_cleaner/apply.py

@@ -0,0 +1,483 @@
+"""Apply cleaning functions to data files."""
+
+import csv
+import importlib.util
+import json
+from pathlib import Path
+from typing import Callable
+
+from .parsers import MARKITDOWN_EXTENSIONS
+
+# Text formats that should be converted to markdown (excludes spreadsheets)
+TEXT_MARKITDOWN_EXTENSIONS = MARKITDOWN_EXTENSIONS - {".xlsx", ".xls", ".ods"}
+
+
+def load_cleaning_module(functions_path: str):
+    """
+    Dynamically import a cleaning_functions.py file.
+
+    Args:
+        functions_path: Path to the cleaning functions file
+
+    Returns:
+        The imported module
+
+    Raises:
+        FileNotFoundError: If the functions file doesn't exist
+        ImportError: If the module cannot be imported
+    """
+    path = Path(functions_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Functions file not found: {functions_path}")
+
+    spec = importlib.util.spec_from_file_location("cleaning_module", path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from: {functions_path}")
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def get_default_output_path(input_path: str, force_ext: str | None = None) -> str:
+    """
+    Generate default output path: input.cleaned.ext
+
+    Args:
+        input_path: Path to the input file
+        force_ext: Override the output extension (e.g., ".xlsx" for .xls files)
+
+    Returns:
+        Path string for the output file
+    """
+    path = Path(input_path)
+    suffix = path.suffix.lower()
+
+    if force_ext:
+        ext = force_ext
+    elif suffix == ".xls":
+        # .xls files are written as .xlsx
+        ext = ".xlsx"
+    elif suffix == ".txt" or suffix in TEXT_MARKITDOWN_EXTENSIONS:
+        # Text formats output as markdown
+        ext = ".md"
+    else:
+        ext = path.suffix
+
+    return str(path.with_suffix(f".cleaned{ext}"))
+
+
+def apply_to_jsonl(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Stream JSONL: read line, clean, write line.
+
+    Args:
+        input_path: Path to input JSONL file
+        output_path: Path for output JSONL file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    records_processed = 0
+
+    with open(input_path, "r", encoding="utf-8") as infile, \
+         open(output_path, "w", encoding="utf-8") as outfile:
+        for line in infile:
+            line = line.strip()
+            if not line:
+                continue
+
+            record = json.loads(line)
+            cleaned = clean_fn(record)
+            outfile.write(json.dumps(cleaned) + "\n")
+
+            records_processed += 1
+            if on_progress:
+                on_progress({"type": "apply_progress", "records_processed": records_processed})
+
+    return records_processed
+
+
+def apply_to_csv(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Stream CSV: DictReader to clean each row, DictWriter to output.
+
+    Args:
+        input_path: Path to input CSV file
+        output_path: Path for output CSV file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    records_processed = 0
+
+    with open(input_path, "r", encoding="utf-8", newline="") as infile:
+        reader = csv.DictReader(infile)
+        fieldnames = reader.fieldnames
+
+        if not fieldnames:
+            return 0
+
+        with open(output_path, "w", encoding="utf-8", newline="") as outfile:
+            writer = csv.DictWriter(outfile, fieldnames=fieldnames)
+            writer.writeheader()
+
+            for row in reader:
+                cleaned = clean_fn(row)
+                writer.writerow(cleaned)
+
+                records_processed += 1
+                if on_progress:
+                    on_progress({"type": "apply_progress", "records_processed": records_processed})
+
+    return records_processed
+
+
+def apply_to_json(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch JSON array: load all, clean each, write array.
+
+    Args:
+        input_path: Path to input JSON file
+        output_path: Path for output JSON file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    with open(input_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        # Single object - wrap, clean, unwrap
+        cleaned = clean_fn(data)
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(cleaned, f, indent=2)
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": 1})
+        return 1
+
+    cleaned_data = []
+    for i, record in enumerate(data):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(cleaned_data, f, indent=2)
+
+    return len(cleaned_data)
+
+
+def apply_to_parquet(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch Parquet: load as list of dicts, clean each, write back.
+
+    Args:
+        input_path: Path to input Parquet file
+        output_path: Path for output Parquet file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+
+    Raises:
+        ImportError: If pyarrow is not installed
+    """
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for parquet files. "
+            "Install with: pip install recursive-cleaner[parquet]"
+        )
+
+    table = pq.read_table(input_path)
+    records = table.to_pylist()
+
+    cleaned_data = []
+    for i, record in enumerate(records):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    # Write back as parquet
+    cleaned_table = pa.Table.from_pylist(cleaned_data)
+    pq.write_table(cleaned_table, output_path)
+
+    return len(cleaned_data)
+
+
+def apply_to_excel(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch Excel: load as list of dicts, clean each, write back.
+
+    Args:
+        input_path: Path to input Excel file (.xlsx or .xls)
+        output_path: Path for output Excel file (.xlsx)
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+
+    Raises:
+        ImportError: If openpyxl (or xlrd for .xls) is not installed
+    """
+    suffix = Path(input_path).suffix.lower()
+
+    if suffix == ".xls":
+        # Use xlrd for .xls files
+        try:
+            import xlrd
+        except ImportError:
+            raise ImportError(
+                "xlrd is required for .xls files. "
+                "Install with: pip install recursive-cleaner[excel]"
+            )
+
+        workbook = xlrd.open_workbook(input_path)
+        sheet = workbook.sheet_by_index(0)
+
+        if sheet.nrows < 1:
+            return 0
+
+        # First row is headers
+        headers = [str(sheet.cell_value(0, col)) for col in range(sheet.ncols)]
+        records = []
+        for row_idx in range(1, sheet.nrows):
+            row_data = {}
+            for col_idx, header in enumerate(headers):
+                row_data[header] = sheet.cell_value(row_idx, col_idx)
+            records.append(row_data)
+    else:
+        # Use openpyxl for .xlsx files
+        try:
+            from openpyxl import load_workbook
+        except ImportError:
+            raise ImportError(
+                "openpyxl is required for .xlsx files. "
+                "Install with: pip install recursive-cleaner[excel]"
+            )
+
+        workbook = load_workbook(input_path, read_only=True)
+        sheet = workbook.active
+
+        rows = list(sheet.iter_rows(values_only=True))
+        if not rows:
+            return 0
+
+        # First row is headers
+        headers = [str(h) if h is not None else "" for h in rows[0]]
+        records = []
+        for row in rows[1:]:
+            row_data = {}
+            for col_idx, header in enumerate(headers):
+                value = row[col_idx] if col_idx < len(row) else None
+                row_data[header] = value
+            records.append(row_data)
+
+        workbook.close()
+
+    # Clean records
+    cleaned_data = []
+    for i, record in enumerate(records):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    # Write back as xlsx using openpyxl
+    try:
+        from openpyxl import Workbook
+    except ImportError:
+        raise ImportError(
+            "openpyxl is required for writing Excel files. "
+            "Install with: pip install recursive-cleaner[excel]"
+        )
+
+    wb = Workbook()
+    ws = wb.active
+
+    if cleaned_data:
+        # Write headers
+        fieldnames = list(cleaned_data[0].keys())
+        ws.append(fieldnames)
+
+        # Write data rows
+        for record in cleaned_data:
+            ws.append([record.get(k) for k in fieldnames])
+
+    wb.save(output_path)
+
+    return len(cleaned_data)
+
+
+def apply_to_text(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Process text/document files: extract text, clean, write as markdown.
+
+    Args:
+        input_path: Path to input file (.txt or markitdown format)
+        output_path: Path for output markdown file
+        clean_fn: Cleaning function to apply to the text
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed (always 1 for text)
+
+    Raises:
+        ImportError: If markitdown is not installed (for non-.txt files)
+    """
+    suffix = Path(input_path).suffix.lower()
+
+    if suffix == ".txt":
+        # Plain text - read directly
+        with open(input_path, "r", encoding="utf-8") as f:
+            content = f.read()
+    else:
+        # Use markitdown for other formats
+        try:
+            from markitdown import MarkItDown
+        except ImportError:
+            raise ImportError(
+                "markitdown is required for this file type. "
+                "Install with: pip install recursive-cleaner[markitdown]"
+            )
+
+        md = MarkItDown()
+        result = md.convert(input_path)
+        content = result.text_content
+
+    # Clean the text content
+    cleaned = clean_fn(content)
+
+    # Write as markdown
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(cleaned)
+
+    if on_progress:
+        on_progress({"type": "apply_progress", "records_processed": 1})
+
+    return 1
+
+
+def apply_cleaning(
+    input_path: str,
+    functions_path: str,
+    output_path: str | None = None,
+    on_progress: Callable[[dict], None] | None = None,
+) -> str:
+    """
+    Apply cleaning functions to a data file.
+
+    Args:
+        input_path: Path to input data file
+        functions_path: Path to cleaning_functions.py
+        output_path: Path for output file (default: input.cleaned.ext)
+        on_progress: Optional progress callback
+
+    Returns:
+        Path to output file
+
+    Raises:
+        FileNotFoundError: If input or functions file not found
+        ImportError: If functions file cannot be imported
+        ValueError: If input format is unsupported
+    """
+    # Validate input file exists
+    if not Path(input_path).exists():
+        raise FileNotFoundError(f"Input file not found: {input_path}")
+
+    # Load cleaning module
+    module = load_cleaning_module(functions_path)
+
+    # Get the clean_data function
+    if not hasattr(module, "clean_data"):
+        raise ImportError(f"Functions file missing clean_data() function: {functions_path}")
+
+    clean_fn = module.clean_data
+
+    # Determine output path
+    suffix = Path(input_path).suffix.lower()
+    if output_path is None:
+        output_path = get_default_output_path(input_path)
+
+    # Route by format
+    format_handlers = {
+        ".jsonl": apply_to_jsonl,
+        ".csv": apply_to_csv,
+        ".json": apply_to_json,
+        ".parquet": apply_to_parquet,
+        ".xlsx": apply_to_excel,
+        ".xls": apply_to_excel,
+    }
+
+    handler = format_handlers.get(suffix)
+
+    # Check for text formats (.txt and markitdown extensions, excluding spreadsheets)
+    if handler is None:
+        if suffix == ".txt" or suffix in TEXT_MARKITDOWN_EXTENSIONS:
+            handler = apply_to_text
+
+    if handler is None:
+        raise ValueError(f"Unsupported format: {suffix}")
+
+    # Emit start event (total_records unknown for streaming formats)
+    if on_progress:
+        on_progress({"type": "apply_start", "total_records": None})
+
+    # Apply cleaning
+    total_records = handler(input_path, output_path, clean_fn, on_progress)
+
+    # Emit complete event
+    if on_progress:
+        on_progress({
+            "type": "apply_complete",
+            "total_records": total_records,
+            "output_path": output_path,
+        })
+
+    return output_path

recursive_cleaner/cleaner.py

@@ -63,6 +63,7 @@ class DataCleaner:
         dry_run: bool = False,
         auto_parse: bool = False,
         tui: bool = False,
+        output_path: str = "cleaning_functions.py",
     ):
         self.backend = llm_backend
         self.file_path = file_path
@@ -88,6 +89,7 @@ class DataCleaner:
         self.dry_run = dry_run
         self.auto_parse = auto_parse
         self.tui = tui
+        self.output_path = output_path
         self.functions: list[dict] = []  # List of {name, docstring, code}
         self._tui_renderer = None  # TUIRenderer instance when tui=True
         self._generated_parser: callable | None = None  # LLM-generated parser for unknown formats
@@ -520,7 +522,7 @@ class DataCleaner:
                 "quality_delta": 0.0,  # Could be calculated from metrics
                 "latency_total_ms": latency_summary.get("total_ms", 0.0),
                 "llm_calls": latency_summary.get("call_count", 0),
-                "output_file": "cleaning_functions.py",
+                "output_file": self.output_path,
             })
             self._tui_renderer.stop()
 
@@ -687,11 +689,11 @@ class DataCleaner:
         self._emit("chunk_done", chunk_index=chunk_idx)
 
     def _write_output(self) -> None:
-        """Write generated functions to cleaning_functions.py."""
+        """Write generated functions to output file."""
         from .output import write_cleaning_file
 
         try:
-            write_cleaning_file(self.functions)
+            write_cleaning_file(self.functions, self.output_path)
         except OutputValidationError as e:
             if not self.tui:
                 print(f"  Error: {e}")
@@ -707,7 +709,7 @@ class DataCleaner:
                     if not self.tui:
                         print(f"  Skipping invalid function: {f['name']}")
             if valid_functions:
-                write_cleaning_file(valid_functions)
+                write_cleaning_file(valid_functions, self.output_path)
             elif not self.tui:
                 print("  No valid functions to write.")
 

recursive_cleaner/cli.py

@@ -0,0 +1,395 @@
+"""CLI interface for Recursive Data Cleaner."""
+
+import argparse
+import os
+import sys
+
+
+def create_backend(provider: str, model: str, base_url: str | None, api_key: str | None):
+    """
+    Factory function to create the appropriate backend.
+
+    Args:
+        provider: Backend provider ("mlx" or "openai")
+        model: Model name/path
+        base_url: Optional API base URL (for openai-compatible servers)
+        api_key: Optional API key
+
+    Returns:
+        LLMBackend instance
+
+    Raises:
+        SystemExit: With code 2 if provider is invalid or import fails
+    """
+    if provider == "mlx":
+        try:
+            from backends import MLXBackend
+            return MLXBackend(model_path=model)
+        except ImportError:
+            print("Error: MLX backend requires mlx-lm. Install with: pip install mlx-lm", file=sys.stderr)
+            sys.exit(2)
+    elif provider == "openai":
+        try:
+            from backends import OpenAIBackend
+            return OpenAIBackend(model=model, api_key=api_key, base_url=base_url)
+        except ImportError as e:
+            print(f"Error: {e}", file=sys.stderr)
+            sys.exit(2)
+    else:
+        print(f"Error: Unknown provider '{provider}'. Use 'mlx' or 'openai'.", file=sys.stderr)
+        sys.exit(2)
+
+
+def read_instructions(value: str) -> str:
+    """
+    Read instructions from inline text or file.
+
+    Args:
+        value: Instructions string or @file.txt path
+
+    Returns:
+        Instructions text
+    """
+    if value.startswith("@"):
+        file_path = value[1:]
+        try:
+            with open(file_path, "r") as f:
+                return f.read().strip()
+        except FileNotFoundError:
+            print(f"Error: Instructions file not found: {file_path}", file=sys.stderr)
+            sys.exit(1)
+        except IOError as e:
+            print(f"Error reading instructions file: {e}", file=sys.stderr)
+            sys.exit(1)
+    elif value == "-":
+        return sys.stdin.read().strip()
+    return value
+
+
+def cmd_generate(args) -> int:
+    """Handle the generate command."""
+    from recursive_cleaner import DataCleaner
+
+    # Check if file exists
+    if not os.path.exists(args.file):
+        print(f"Error: File not found: {args.file}", file=sys.stderr)
+        return 1
+
+    # Create backend
+    backend = create_backend(args.provider, args.model, args.base_url, args.api_key)
+
+    # Read instructions
+    instructions = read_instructions(args.instructions) if args.instructions else ""
+
+    # Create progress callback for non-TUI mode
+    def on_progress(event):
+        if not args.tui:
+            event_type = event.get("type", "")
+            if event_type == "function_generated":
+                print(f"  Generated: {event.get('function_name', '')}")
+
+    try:
+        cleaner = DataCleaner(
+            llm_backend=backend,
+            file_path=args.file,
+            chunk_size=args.chunk_size,
+            instructions=instructions,
+            max_iterations=args.max_iterations,
+            mode=args.mode,
+            state_file=args.state_file,
+            report_path=args.report if args.report else None,
+            tui=args.tui,
+            optimize=args.optimize,
+            track_metrics=args.track_metrics,
+            early_termination=args.early_termination,
+            on_progress=on_progress if not args.tui else None,
+            output_path=args.output,
+        )
+        cleaner.run()
+        return 0
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 3
+
+
+def cmd_analyze(args) -> int:
+    """Handle the analyze command (dry-run mode)."""
+    from recursive_cleaner import DataCleaner
+
+    # Check if file exists
+    if not os.path.exists(args.file):
+        print(f"Error: File not found: {args.file}", file=sys.stderr)
+        return 1
+
+    # Create backend
+    backend = create_backend(args.provider, args.model, args.base_url, args.api_key)
+
+    # Read instructions
+    instructions = read_instructions(args.instructions) if args.instructions else ""
+
+    # Progress callback for analysis output
+    def on_progress(event):
+        if not args.tui:
+            event_type = event.get("type", "")
+            if event_type == "issues_detected":
+                issues = event.get("issues", [])
+                chunk_idx = event.get("chunk_index", 0)
+                unsolved = [i for i in issues if not i.get("solved", False)]
+                print(f"Chunk {chunk_idx + 1}: {len(issues)} issues ({len(unsolved)} unsolved)")
+
+    try:
+        cleaner = DataCleaner(
+            llm_backend=backend,
+            file_path=args.file,
+            chunk_size=args.chunk_size,
+            instructions=instructions,
+            max_iterations=args.max_iterations,
+            mode=args.mode,
+            dry_run=True,
+            tui=args.tui,
+            on_progress=on_progress if not args.tui else None,
+        )
+        cleaner.run()
+        return 0
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 3
+
+
+def cmd_apply(args) -> int:
+    """Handle the apply command."""
+    from recursive_cleaner.apply import apply_cleaning
+
+    # Check if input file exists
+    if not os.path.exists(args.file):
+        print(f"Error: File not found: {args.file}", file=sys.stderr)
+        return 1
+
+    # Check if functions file exists
+    if not os.path.exists(args.functions):
+        print(f"Error: Functions file not found: {args.functions}", file=sys.stderr)
+        return 1
+
+    try:
+        output_path = apply_cleaning(
+            input_path=args.file,
+            functions_path=args.functions,
+            output_path=args.output,
+        )
+        print(f"Cleaned data written to: {output_path}")
+        return 0
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    except ImportError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 2
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 3
+
+
+def cmd_resume(args) -> int:
+    """Handle the resume command."""
+    from recursive_cleaner import DataCleaner
+
+    # Check if state file exists
+    if not os.path.exists(args.state_file):
+        print(f"Error: State file not found: {args.state_file}", file=sys.stderr)
+        return 1
+
+    # Create backend
+    backend = create_backend(args.provider, args.model, args.base_url, args.api_key)
+
+    try:
+        cleaner = DataCleaner.resume(args.state_file, backend)
+        cleaner.run()
+        return 0
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+    except ValueError as e:
+        print(f"Error: Invalid state file: {e}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 3
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create the argument parser with all subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="recursive-cleaner",
+        description="LLM-powered incremental data cleaning pipeline",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # --- generate command ---
+    gen_parser = subparsers.add_parser(
+        "generate",
+        help="Generate cleaning functions from data file",
+    )
+    gen_parser.add_argument("file", metavar="FILE", help="Path to input data file")
+    gen_parser.add_argument(
+        "-p", "--provider", required=True, choices=["mlx", "openai"],
+        help="LLM provider (mlx or openai)"
+    )
+    gen_parser.add_argument(
+        "-m", "--model", required=True, help="Model name/path"
+    )
+    gen_parser.add_argument(
+        "-i", "--instructions", default="",
+        help="Cleaning instructions (text or @file.txt)"
+    )
+    gen_parser.add_argument(
+        "--base-url", help="API base URL (for openai-compatible servers)"
+    )
+    gen_parser.add_argument(
+        "--api-key", help="API key (or use OPENAI_API_KEY env var)"
+    )
+    gen_parser.add_argument(
+        "--chunk-size", type=int, default=50, help="Items per chunk (default: 50)"
+    )
+    gen_parser.add_argument(
+        "--max-iterations", type=int, default=5,
+        help="Max iterations per chunk (default: 5)"
+    )
+    gen_parser.add_argument(
+        "--mode", choices=["auto", "structured", "text"], default="auto",
+        help="Processing mode (default: auto)"
+    )
+    gen_parser.add_argument(
+        "-o", "--output", default="cleaning_functions.py",
+        help="Output file path (default: cleaning_functions.py)"
+    )
+    gen_parser.add_argument(
+        "--report", default="cleaning_report.md",
+        help="Report file path (empty to disable, default: cleaning_report.md)"
+    )
+    gen_parser.add_argument(
+        "--state-file", help="Checkpoint file for resume"
+    )
+    gen_parser.add_argument(
+        "--tui", action="store_true", help="Enable Rich terminal dashboard"
+    )
+    gen_parser.add_argument(
+        "--optimize", action="store_true", help="Consolidate redundant functions"
+    )
+    gen_parser.add_argument(
+        "--track-metrics", action="store_true", help="Measure before/after quality"
+    )
+    gen_parser.add_argument(
+        "--early-termination", action="store_true",
+        help="Stop on pattern saturation"
+    )
+    gen_parser.set_defaults(func=cmd_generate)
+
+    # --- analyze command ---
+    analyze_parser = subparsers.add_parser(
+        "analyze",
+        help="Dry-run analysis without generating functions",
+    )
+    analyze_parser.add_argument("file", metavar="FILE", help="Path to input data file")
+    analyze_parser.add_argument(
+        "-p", "--provider", required=True, choices=["mlx", "openai"],
+        help="LLM provider (mlx or openai)"
+    )
+    analyze_parser.add_argument(
+        "-m", "--model", required=True, help="Model name/path"
+    )
+    analyze_parser.add_argument(
+        "-i", "--instructions", default="",
+        help="Cleaning instructions (text or @file.txt)"
+    )
+    analyze_parser.add_argument(
+        "--base-url", help="API base URL (for openai-compatible servers)"
+    )
+    analyze_parser.add_argument(
+        "--api-key", help="API key (or use OPENAI_API_KEY env var)"
+    )
+    analyze_parser.add_argument(
+        "--chunk-size", type=int, default=50, help="Items per chunk (default: 50)"
+    )
+    analyze_parser.add_argument(
+        "--max-iterations", type=int, default=5,
+        help="Max iterations per chunk (default: 5)"
+    )
+    analyze_parser.add_argument(
+        "--mode", choices=["auto", "structured", "text"], default="auto",
+        help="Processing mode (default: auto)"
+    )
+    analyze_parser.add_argument(
+        "--tui", action="store_true", help="Enable Rich terminal dashboard"
+    )
+    analyze_parser.set_defaults(func=cmd_analyze)
+
+    # --- resume command ---
+    resume_parser = subparsers.add_parser(
+        "resume",
+        help="Resume from checkpoint file",
+    )
+    resume_parser.add_argument(
+        "state_file", metavar="STATE_FILE", help="Path to checkpoint JSON file"
+    )
+    resume_parser.add_argument(
+        "-p", "--provider", required=True, choices=["mlx", "openai"],
+        help="LLM provider (mlx or openai)"
+    )
+    resume_parser.add_argument(
+        "-m", "--model", required=True, help="Model name/path"
+    )
+    resume_parser.add_argument(
+        "--base-url", help="API base URL (for openai-compatible servers)"
+    )
+    resume_parser.add_argument(
+        "--api-key", help="API key (or use OPENAI_API_KEY env var)"
+    )
+    resume_parser.set_defaults(func=cmd_resume)
+
+    # --- apply command ---
+    apply_parser = subparsers.add_parser(
+        "apply",
+        help="Apply cleaning functions to data file",
+    )
+    apply_parser.add_argument("file", metavar="FILE", help="Path to input data file")
+    apply_parser.add_argument(
+        "-f", "--functions", required=True,
+        help="Path to cleaning_functions.py"
+    )
+    apply_parser.add_argument(
+        "-o", "--output", help="Output file path (default: <input>.cleaned.<ext>)"
+    )
+    apply_parser.set_defaults(func=cmd_apply)
+
+    return parser
+
+
+def main(args: list[str] | None = None) -> int:
+    """
+    Main entry point for the CLI.
+
+    Args:
+        args: Command-line arguments (defaults to sys.argv[1:])
+
+    Returns:
+        Exit code (0=success, 1=general error, 2=backend error, 3=validation error)
+    """
+    parser = create_parser()
+    parsed = parser.parse_args(args)
+
+    if parsed.command is None:
+        parser.print_help()
+        return 0
+
+    return parsed.func(parsed)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

recursive_cleaner/tui.py

@@ -505,19 +505,28 @@ class TUIRenderer:
         )
         self._layout["left_panel"].update(left_panel)
 
-    def _parse_response_for_display(self, response: str) -> str:
-        """Parse LLM XML response into readable format for transmission log.
+    def _colorize_transmission(self, response: str) -> "Text":
+        """Parse LLM XML response into colorized Rich Text for transmission log.
+
+        Color scheme:
+        - Issues (solved): dim
+        - Issues (unsolved): bright_white with cycling accent (blue/magenta/cyan/yellow)
+        - Function names: green
+        - Docstrings: italic
+        - Status clean: green
+        - Status needs_more_work: yellow
 
         Args:
             response: Raw LLM response text (XML format)
 
         Returns:
-            Formatted string for display showing issues, function being
-            generated, and chunk status.
+            Rich Text object with colors applied.
         """
         import re
 
-        lines = []
+        ISSUE_COLORS = ["blue", "magenta", "cyan", "yellow"]
+        text = Text()
+        unsolved_index = 0
 
         try:
             # Find all issues
@@ -525,53 +534,63 @@ class TUIRenderer:
             issues = re.findall(issue_pattern, response, re.DOTALL)
 
             if issues:
-                lines.append("ISSUES DETECTED:")
+                text.append("ISSUES DETECTED:\n", style="bold cyan")
                 for issue_id, solved, desc in issues[:8]:  # Limit to 8 issues
-                    marker = "\u2713" if solved == "true" else "\u2717"  # checkmark or X
                     desc_clean = desc.strip()[:40]  # Truncate description
-                    lines.append(f"  {marker} {desc_clean}")
+                    if solved == "true":
+                        text.append("  \u2713 ", style="green")
+                        text.append(f"{desc_clean}\n", style="dim")
+                    else:
+                        accent = ISSUE_COLORS[unsolved_index % len(ISSUE_COLORS)]
+                        text.append("  \u2717 ", style=accent)
+                        text.append(f"{desc_clean}\n", style="bright_white")
+                        unsolved_index += 1
                 if len(issues) > 8:
-                    lines.append(f"  (+{len(issues) - 8} more)")
-                lines.append("")
+                    text.append(f"  (+{len(issues) - 8} more)\n", style="dim")
+                text.append("\n")
 
             # Find function being generated
             name_match = re.search(r'<name>([^<]+)</name>', response)
             docstring_match = re.search(r'<docstring>([^<]+)</docstring>', response, re.DOTALL)
 
             if name_match:
-                lines.append(f"GENERATING: {name_match.group(1).strip()}")
+                text.append("GENERATING: ", style="bold cyan")
+                text.append(f"{name_match.group(1).strip()}\n", style="green bold")
                 if docstring_match:
                     doc = docstring_match.group(1).strip()[:60]
-                    lines.append(f'  "{doc}..."')
-                lines.append("")
+                    text.append(f'  "{doc}..."\n', style="italic")
+                text.append("\n")
 
             # Find chunk status
             status_match = re.search(r'<chunk_status>([^<]+)</chunk_status>', response)
             if status_match:
                 status = status_match.group(1).strip()
-                lines.append(f"STATUS: {status.upper()}")
-
-            if lines:
-                return "\n".join(lines)
+                text.append("STATUS: ", style="bold cyan")
+                if status == "clean":
+                    text.append(status.upper(), style="green bold")
+                else:
+                    text.append(status.upper().replace("_", " "), style="yellow bold")
+
+            if text.plain:
+                return text
         except Exception:
             pass
 
         # Fallback: show truncated raw response
-        return response[:500] + "..." if len(response) > 500 else response
+        fallback = response[:500] + "..." if len(response) > 500 else response
+        return Text(fallback, style="dim cyan")
 
     def _refresh_right_panel(self) -> None:
-        """Refresh the right panel with parsed transmission log."""
+        """Refresh the right panel with colorized transmission log."""
         if not HAS_RICH or self._layout is None:
             return
 
-        # Get last response and parse for display
+        # Get last response and colorize for display
         response = self._state.last_response
         if not response:
-            display_text = "(Awaiting transmission...)"
+            log_text = Text("(Awaiting transmission...)", style="dim cyan")
         else:
-            display_text = self._parse_response_for_display(response)
-
-        log_text = Text(display_text, style="dim cyan")
+            log_text = self._colorize_transmission(response)
 
         right_panel = Panel(
             log_text,

{recursive_cleaner-0.8.0.dist-info → recursive_cleaner-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 0.8.0
+Version: 1.0.0
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
@@ -9,7 +9,7 @@ Author: Gary Tran
 License-Expression: MIT
 License-File: LICENSE
 Keywords: automation,data-cleaning,data-quality,etl,llm,machine-learning
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
@@ -26,10 +26,15 @@ Requires-Dist: tenacity>=8.0
 Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.0.0; extra == 'excel'
+Requires-Dist: xlrd>=2.0.0; extra == 'excel'
 Provides-Extra: markitdown
 Requires-Dist: markitdown>=0.1.0; extra == 'markitdown'
 Provides-Extra: mlx
 Requires-Dist: mlx-lm>=0.10.0; extra == 'mlx'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=14.0.0; extra == 'parquet'
 Provides-Extra: tui
@@ -140,6 +145,91 @@ cleaner.run()  # Generates cleaning_functions.py
 - **Token Estimation**: Track estimated input/output tokens across the run
 - **Graceful Fallback**: Works without Rich installed (falls back to callbacks)
 
+### CLI (v0.9.0)
+- **Command Line Interface**: Use without writing Python code
+- **Multiple Backends**: MLX (Apple Silicon) and OpenAI-compatible (OpenAI, LM Studio, Ollama)
+- **Four Commands**: `generate`, `analyze` (dry-run), `resume`, `apply`
+
+### Apply Mode (v1.0.0)
+- **Apply Cleaning Functions**: Apply generated functions to full datasets
+- **Data Formats**: JSONL, CSV, JSON, Parquet, Excel (.xlsx/.xls) output same format
+- **Text Formats**: PDF, Word, HTML, etc. output as Markdown
+- **Streaming**: Memory-efficient line-by-line processing for JSONL/CSV
+- **Colored TUI**: Enhanced transmission log with syntax-highlighted XML parsing
+
+## Command Line Interface
+
+After installation, the `recursive-cleaner` command is available:
+
+```bash
+# Generate cleaning functions with MLX (Apple Silicon)
+recursive-cleaner generate data.jsonl \
+  --provider mlx \
+  --model "lmstudio-community/Qwen3-80B-MLX-4bit" \
+  --instructions "Normalize phone numbers to E.164" \
+  --output cleaning_functions.py
+
+# Use OpenAI
+export OPENAI_API_KEY=your-key
+recursive-cleaner generate data.jsonl \
+  --provider openai \
+  --model gpt-4o \
+  --instructions "Fix date formats"
+
+# Use LM Studio or Ollama (OpenAI-compatible)
+recursive-cleaner generate data.jsonl \
+  --provider openai \
+  --model "qwen/qwen3-vl-30b" \
+  --base-url http://localhost:1234/v1 \
+  --instructions "Normalize prices"
+
+# Dry-run analysis
+recursive-cleaner analyze data.jsonl \
+  --provider openai \
+  --model gpt-4o \
+  --instructions @instructions.txt
+
+# Resume from checkpoint
+recursive-cleaner resume cleaning_state.json \
+  --provider mlx \
+  --model "model-path"
+
+# Apply cleaning functions to data
+recursive-cleaner apply data.jsonl \
+  --functions cleaning_functions.py \
+  --output cleaned_data.jsonl
+
+# Apply to Excel (outputs same format)
+recursive-cleaner apply sales.xlsx \
+  --functions cleaning_functions.py
+
+# Apply to PDF (outputs markdown)
+recursive-cleaner apply document.pdf \
+  --functions cleaning_functions.py \
+  --output cleaned.md
+```
+
+### CLI Options
+
+```
+recursive-cleaner generate <FILE> [OPTIONS]
+
+Required:
+  FILE                      Input data file
+  -p, --provider {mlx,openai}  LLM provider
+  -m, --model MODEL         Model name/path
+
+Optional:
+  -i, --instructions TEXT   Cleaning instructions (or @file.txt)
+  --base-url URL            API URL for OpenAI-compatible servers
+  --chunk-size N            Items per chunk (default: 50)
+  --max-iterations N        Max iterations per chunk (default: 5)
+  -o, --output PATH         Output file (default: cleaning_functions.py)
+  --tui                     Enable Rich dashboard
+  --optimize                Consolidate redundant functions
+  --track-metrics           Measure before/after quality
+```
+
 ## Configuration
 
 ```python
@@ -270,6 +360,7 @@ cleaner.run()
 
 ```
 recursive_cleaner/
+├── cli.py              # Command line interface
 ├── cleaner.py          # Main DataCleaner class
 ├── context.py          # Docstring registry with FIFO eviction
 ├── dependencies.py     # Topological sort for function ordering
@@ -286,6 +377,10 @@ recursive_cleaner/
 ├── validation.py       # Runtime validation + holdout
 └── vendor/
     └── chunker.py      # Vendored sentence-aware chunker
+
+backends/
+├── mlx_backend.py      # MLX-LM backend for Apple Silicon
+└── openai_backend.py   # OpenAI-compatible backend
 ```
 
 ## Testing
@@ -294,14 +389,14 @@ recursive_cleaner/
 pytest tests/ -v
 ```
 
-465 tests covering all features. Test datasets in `test_cases/`:
+548 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data
 
 ## Philosophy
 
-- **Simplicity over extensibility**: ~3,000 lines that do one thing well
+- **Simplicity over extensibility**: ~5,000 lines that do one thing well
 - **stdlib over dependencies**: Only `tenacity` required
 - **Retry over recover**: On error, retry with error in prompt
 - **Wu wei**: Let the LLM make decisions about data it understands
@@ -310,6 +405,7 @@ pytest tests/ -v
 
 | Version | Features |
 |---------|----------|
+| v0.9.0 | CLI tool with MLX and OpenAI-compatible backends (LM Studio, Ollama) |
 | v0.8.0 | Terminal UI with Rich dashboard, mission control aesthetic, transmission log |
 | v0.7.0 | Markitdown (20+ formats), Parquet support, LLM-generated parsers |
 | v0.6.0 | Latency metrics, import consolidation, cleaning report, dry-run mode |

{recursive_cleaner-0.8.0.dist-info → recursive_cleaner-1.0.0.dist-info}/RECORD

@@ -1,7 +1,11 @@
-backends/__init__.py,sha256=FUgODeYSGBvT0-z6myVby6YeAHG0nEUgWLITBKobUew,121
+backends/__init__.py,sha256=vWcPASV0GGEAydzOSjdrknkSHoGbSs4edtuv9HIzBhI,180
 backends/mlx_backend.py,sha256=0U6IqmDHyk4vjKzytvEcQvSUBryQTgFtsNOcpwFNKk8,2945
-recursive_cleaner/__init__.py,sha256=v0bNQ3H0d7n6cTOkuxuqG9bmnX9yeZBLZ_AfFM7edHI,1789
-recursive_cleaner/cleaner.py,sha256=vZTMwaLlCmuh1qy3c-puEZrwS5gXt0u28d5iweQXbms,29801
+backends/openai_backend.py,sha256=vKWsXKltBv_tJDoQfQ_7KVMZDfomhFFN2vl1oZ1KGbQ,2057
+recursive_cleaner/__init__.py,sha256=xCFlkqmmBoa7ntUZQnRQxVMv9iLeOvmboDS_j2EHfZI,1862
+recursive_cleaner/__main__.py,sha256=WXmMaL_myHPsG_qXAhZDufD43Ydsd25RV2IPeW2Kg08,152
+recursive_cleaner/apply.py,sha256=hjeljhZNiOuwz9m09RYVLl_z_9tet7LwubH6cb_Wy6Y,13855
+recursive_cleaner/cleaner.py,sha256=kPOQ44hgiJzABiqdmjg2hqd7Ot9uxKUSOe8_jz0UBQc,29911
+recursive_cleaner/cli.py,sha256=Sk_qYKxSn1PiPmMLKkyj9VxsseHaSXmSlGazxfmkTFc,12807
 recursive_cleaner/context.py,sha256=avMXRDxLd7nd8CKWtvPHQy1MFhBKiA0aUVVJIlWoLZ4,824
 recursive_cleaner/dependencies.py,sha256=vlYeoGL517v3yUSWN0wYDuIs9OOuQwM_dCBADrlitW8,2080
 recursive_cleaner/errors.py,sha256=hwRJF8NSmWy_FZHCxcZDZxLQ0zqvo5dX8ImkB9mrOYc,433
@@ -14,12 +18,13 @@ recursive_cleaner/prompt.py,sha256=ep0eOXz_XbhH3HduJ76LvzVSftonhcv4GLEecIqd3lY,6
 recursive_cleaner/report.py,sha256=AWWneRjvl76ccLlExdkKJeY3GVFUG_LtmzVIJJT5cFI,4629
 recursive_cleaner/response.py,sha256=3w0mLnqEPdB4daMSF0mtTcG0PTP-utb1HFtKuYA1ljw,9064
 recursive_cleaner/schema.py,sha256=w2hcEdApR15KVI9SFWB3VfumMoHFwn1YJrktdfgPo8M,3925
-recursive_cleaner/tui.py,sha256=FwG_uCwqUcvch5dRZmV-ba2JXD0XJkm9roXzPQ9iUSo,21633
+recursive_cleaner/tui.py,sha256=zuiFPtMh3K-sC1CWZoaoUmgZ3rESkl10gYcqMzpVqiM,22598
 recursive_cleaner/types.py,sha256=-GdCmsfHd3rfdfCi5c-RXqX4TyuCSHgA__3AF3bMhoQ,290
 recursive_cleaner/validation.py,sha256=-KAolhw3GQyhHwmh0clEj8xqPD5O-R2AO5rx7vubIME,6442
 recursive_cleaner/vendor/__init__.py,sha256=E87TjmjRzu8ty39nqThvBwM611yXlLKQZ6KGY_zp3Dk,117
 recursive_cleaner/vendor/chunker.py,sha256=pDDbfF6FoSmUji0-RG4MletPxJ-VybGw0yfnhh0aMSQ,6730
-recursive_cleaner-0.8.0.dist-info/METADATA,sha256=rVABzjvUZ-uzk35o5evbIlkRIbgEb29QPKSCoMI4_fs,11072
-recursive_cleaner-0.8.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-recursive_cleaner-0.8.0.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
-recursive_cleaner-0.8.0.dist-info/RECORD,,
+recursive_cleaner-1.0.0.dist-info/METADATA,sha256=L86ATNd8JxmPp32HKaO6PPwkmq4sIE3Mdvgx3pmUulE,14285
+recursive_cleaner-1.0.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+recursive_cleaner-1.0.0.dist-info/entry_points.txt,sha256=S5nbi0rnifpShxdXGExeZnd65UZfp8K7DNyuKPST6nk,65
+recursive_cleaner-1.0.0.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
+recursive_cleaner-1.0.0.dist-info/RECORD,,

recursive_cleaner-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+recursive-cleaner = recursive_cleaner.cli:main