recursive-cleaner 0.8.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

backends/__init__.py

@@ -1,5 +1,6 @@
 """Backend implementations for Recursive Data Cleaner."""
 
 from .mlx_backend import MLXBackend
+from .openai_backend import OpenAIBackend
 
-__all__ = ["MLXBackend"]
+__all__ = ["MLXBackend", "OpenAIBackend"]

backends/openai_backend.py

@@ -0,0 +1,71 @@
+"""OpenAI-compatible backend for Recursive Data Cleaner."""
+
+import os
+
+
+class OpenAIBackend:
+    """
+    OpenAI-compatible backend implementation.
+
+    Works with OpenAI API, LM Studio, Ollama, and other OpenAI-compatible servers.
+    Conforms to the LLMBackend protocol.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        max_tokens: int = 4096,
+        temperature: float = 0.7,
+    ):
+        """
+        Initialize the OpenAI backend.
+
+        Args:
+            model: Model name (e.g., "gpt-4o", "gpt-3.5-turbo")
+            api_key: API key (defaults to OPENAI_API_KEY env var, or "not-needed" for local)
+            base_url: API base URL (defaults to OpenAI's API)
+            max_tokens: Maximum tokens to generate
+            temperature: Sampling temperature
+        """
+        try:
+            import openai
+        except ImportError:
+            raise ImportError(
+                "OpenAI SDK not installed. Install with: pip install openai"
+            )
+
+        self.model = model
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+
+        # Resolve API key: explicit > env var > "not-needed" for local servers
+        if api_key is not None:
+            resolved_key = api_key
+        else:
+            resolved_key = os.environ.get("OPENAI_API_KEY", "not-needed")
+
+        # Create client
+        self._client = openai.OpenAI(
+            api_key=resolved_key,
+            base_url=base_url,
+        )
+
+    def generate(self, prompt: str) -> str:
+        """
+        Generate a response from the LLM.
+
+        Args:
+            prompt: The input prompt
+
+        Returns:
+            The generated text response
+        """
+        response = self._client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+        )
+        return response.choices[0].message.content or ""

recursive_cleaner/__init__.py

@@ -1,5 +1,6 @@
 """Recursive Data Cleaner - LLM-powered incremental data cleaning pipeline."""
 
+from recursive_cleaner.apply import apply_cleaning
 from recursive_cleaner.cleaner import DataCleaner
 from recursive_cleaner.context import build_context
 from recursive_cleaner.dependencies import resolve_dependencies
@@ -21,9 +22,10 @@ from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
 from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.tui import HAS_RICH, TUIRenderer
-from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
+from recursive_cleaner.validation import check_code_safety, extract_modified_fields, extract_sample_data, validate_function
 
 __all__ = [
+    "apply_cleaning",
     "CleanerError",
     "ParseError",
     "MaxIterationsError",
@@ -41,6 +43,7 @@ __all__ = [
     "validate_function",
     "extract_sample_data",
     "check_code_safety",
+    "extract_modified_fields",
     "resolve_dependencies",
     "QualityMetrics",
     "measure_quality",

recursive_cleaner/__main__.py

@@ -0,0 +1,8 @@
+"""Entry point for python -m recursive_cleaner."""
+
+import sys
+
+from recursive_cleaner.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

recursive_cleaner/apply.py

@@ -0,0 +1,483 @@
+"""Apply cleaning functions to data files."""
+
+import csv
+import importlib.util
+import json
+from pathlib import Path
+from typing import Callable
+
+from .parsers import MARKITDOWN_EXTENSIONS
+
+# Text formats that should be converted to markdown (excludes spreadsheets)
+TEXT_MARKITDOWN_EXTENSIONS = MARKITDOWN_EXTENSIONS - {".xlsx", ".xls", ".ods"}
+
+
+def load_cleaning_module(functions_path: str):
+    """
+    Dynamically import a cleaning_functions.py file.
+
+    Args:
+        functions_path: Path to the cleaning functions file
+
+    Returns:
+        The imported module
+
+    Raises:
+        FileNotFoundError: If the functions file doesn't exist
+        ImportError: If the module cannot be imported
+    """
+    path = Path(functions_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Functions file not found: {functions_path}")
+
+    spec = importlib.util.spec_from_file_location("cleaning_module", path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from: {functions_path}")
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def get_default_output_path(input_path: str, force_ext: str | None = None) -> str:
+    """
+    Generate default output path: input.cleaned.ext
+
+    Args:
+        input_path: Path to the input file
+        force_ext: Override the output extension (e.g., ".xlsx" for .xls files)
+
+    Returns:
+        Path string for the output file
+    """
+    path = Path(input_path)
+    suffix = path.suffix.lower()
+
+    if force_ext:
+        ext = force_ext
+    elif suffix == ".xls":
+        # .xls files are written as .xlsx
+        ext = ".xlsx"
+    elif suffix == ".txt" or suffix in TEXT_MARKITDOWN_EXTENSIONS:
+        # Text formats output as markdown
+        ext = ".md"
+    else:
+        ext = path.suffix
+
+    return str(path.with_suffix(f".cleaned{ext}"))
+
+
+def apply_to_jsonl(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Stream JSONL: read line, clean, write line.
+
+    Args:
+        input_path: Path to input JSONL file
+        output_path: Path for output JSONL file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    records_processed = 0
+
+    with open(input_path, "r", encoding="utf-8") as infile, \
+         open(output_path, "w", encoding="utf-8") as outfile:
+        for line in infile:
+            line = line.strip()
+            if not line:
+                continue
+
+            record = json.loads(line)
+            cleaned = clean_fn(record)
+            outfile.write(json.dumps(cleaned) + "\n")
+
+            records_processed += 1
+            if on_progress:
+                on_progress({"type": "apply_progress", "records_processed": records_processed})
+
+    return records_processed
+
+
+def apply_to_csv(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Stream CSV: DictReader to clean each row, DictWriter to output.
+
+    Args:
+        input_path: Path to input CSV file
+        output_path: Path for output CSV file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    records_processed = 0
+
+    with open(input_path, "r", encoding="utf-8", newline="") as infile:
+        reader = csv.DictReader(infile)
+        fieldnames = reader.fieldnames
+
+        if not fieldnames:
+            return 0
+
+        with open(output_path, "w", encoding="utf-8", newline="") as outfile:
+            writer = csv.DictWriter(outfile, fieldnames=fieldnames)
+            writer.writeheader()
+
+            for row in reader:
+                cleaned = clean_fn(row)
+                writer.writerow(cleaned)
+
+                records_processed += 1
+                if on_progress:
+                    on_progress({"type": "apply_progress", "records_processed": records_processed})
+
+    return records_processed
+
+
+def apply_to_json(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch JSON array: load all, clean each, write array.
+
+    Args:
+        input_path: Path to input JSON file
+        output_path: Path for output JSON file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+    """
+    with open(input_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        # Single object - wrap, clean, unwrap
+        cleaned = clean_fn(data)
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(cleaned, f, indent=2)
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": 1})
+        return 1
+
+    cleaned_data = []
+    for i, record in enumerate(data):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(cleaned_data, f, indent=2)
+
+    return len(cleaned_data)
+
+
+def apply_to_parquet(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch Parquet: load as list of dicts, clean each, write back.
+
+    Args:
+        input_path: Path to input Parquet file
+        output_path: Path for output Parquet file
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+
+    Raises:
+        ImportError: If pyarrow is not installed
+    """
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for parquet files. "
+            "Install with: pip install recursive-cleaner[parquet]"
+        )
+
+    table = pq.read_table(input_path)
+    records = table.to_pylist()
+
+    cleaned_data = []
+    for i, record in enumerate(records):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    # Write back as parquet
+    cleaned_table = pa.Table.from_pylist(cleaned_data)
+    pq.write_table(cleaned_table, output_path)
+
+    return len(cleaned_data)
+
+
+def apply_to_excel(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Batch Excel: load as list of dicts, clean each, write back.
+
+    Args:
+        input_path: Path to input Excel file (.xlsx or .xls)
+        output_path: Path for output Excel file (.xlsx)
+        clean_fn: Cleaning function to apply to each record
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed
+
+    Raises:
+        ImportError: If openpyxl (or xlrd for .xls) is not installed
+    """
+    suffix = Path(input_path).suffix.lower()
+
+    if suffix == ".xls":
+        # Use xlrd for .xls files
+        try:
+            import xlrd
+        except ImportError:
+            raise ImportError(
+                "xlrd is required for .xls files. "
+                "Install with: pip install recursive-cleaner[excel]"
+            )
+
+        workbook = xlrd.open_workbook(input_path)
+        sheet = workbook.sheet_by_index(0)
+
+        if sheet.nrows < 1:
+            return 0
+
+        # First row is headers
+        headers = [str(sheet.cell_value(0, col)) for col in range(sheet.ncols)]
+        records = []
+        for row_idx in range(1, sheet.nrows):
+            row_data = {}
+            for col_idx, header in enumerate(headers):
+                row_data[header] = sheet.cell_value(row_idx, col_idx)
+            records.append(row_data)
+    else:
+        # Use openpyxl for .xlsx files
+        try:
+            from openpyxl import load_workbook
+        except ImportError:
+            raise ImportError(
+                "openpyxl is required for .xlsx files. "
+                "Install with: pip install recursive-cleaner[excel]"
+            )
+
+        workbook = load_workbook(input_path, read_only=True)
+        sheet = workbook.active
+
+        rows = list(sheet.iter_rows(values_only=True))
+        if not rows:
+            return 0
+
+        # First row is headers
+        headers = [str(h) if h is not None else "" for h in rows[0]]
+        records = []
+        for row in rows[1:]:
+            row_data = {}
+            for col_idx, header in enumerate(headers):
+                value = row[col_idx] if col_idx < len(row) else None
+                row_data[header] = value
+            records.append(row_data)
+
+        workbook.close()
+
+    # Clean records
+    cleaned_data = []
+    for i, record in enumerate(records):
+        cleaned = clean_fn(record)
+        cleaned_data.append(cleaned)
+
+        if on_progress:
+            on_progress({"type": "apply_progress", "records_processed": i + 1})
+
+    # Write back as xlsx using openpyxl
+    try:
+        from openpyxl import Workbook
+    except ImportError:
+        raise ImportError(
+            "openpyxl is required for writing Excel files. "
+            "Install with: pip install recursive-cleaner[excel]"
+        )
+
+    wb = Workbook()
+    ws = wb.active
+
+    if cleaned_data:
+        # Write headers
+        fieldnames = list(cleaned_data[0].keys())
+        ws.append(fieldnames)
+
+        # Write data rows
+        for record in cleaned_data:
+            ws.append([record.get(k) for k in fieldnames])
+
+    wb.save(output_path)
+
+    return len(cleaned_data)
+
+
+def apply_to_text(
+    input_path: str,
+    output_path: str,
+    clean_fn: Callable,
+    on_progress: Callable[[dict], None] | None = None,
+) -> int:
+    """
+    Process text/document files: extract text, clean, write as markdown.
+
+    Args:
+        input_path: Path to input file (.txt or markitdown format)
+        output_path: Path for output markdown file
+        clean_fn: Cleaning function to apply to the text
+        on_progress: Optional progress callback
+
+    Returns:
+        Number of records processed (always 1 for text)
+
+    Raises:
+        ImportError: If markitdown is not installed (for non-.txt files)
+    """
+    suffix = Path(input_path).suffix.lower()
+
+    if suffix == ".txt":
+        # Plain text - read directly
+        with open(input_path, "r", encoding="utf-8") as f:
+            content = f.read()
+    else:
+        # Use markitdown for other formats
+        try:
+            from markitdown import MarkItDown
+        except ImportError:
+            raise ImportError(
+                "markitdown is required for this file type. "
+                "Install with: pip install recursive-cleaner[markitdown]"
+            )
+
+        md = MarkItDown()
+        result = md.convert(input_path)
+        content = result.text_content
+
+    # Clean the text content
+    cleaned = clean_fn(content)
+
+    # Write as markdown
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(cleaned)
+
+    if on_progress:
+        on_progress({"type": "apply_progress", "records_processed": 1})
+
+    return 1
+
+
+def apply_cleaning(
+    input_path: str,
+    functions_path: str,
+    output_path: str | None = None,
+    on_progress: Callable[[dict], None] | None = None,
+) -> str:
+    """
+    Apply cleaning functions to a data file.
+
+    Args:
+        input_path: Path to input data file
+        functions_path: Path to cleaning_functions.py
+        output_path: Path for output file (default: input.cleaned.ext)
+        on_progress: Optional progress callback
+
+    Returns:
+        Path to output file
+
+    Raises:
+        FileNotFoundError: If input or functions file not found
+        ImportError: If functions file cannot be imported
+        ValueError: If input format is unsupported
+    """
+    # Validate input file exists
+    if not Path(input_path).exists():
+        raise FileNotFoundError(f"Input file not found: {input_path}")
+
+    # Load cleaning module
+    module = load_cleaning_module(functions_path)
+
+    # Get the clean_data function
+    if not hasattr(module, "clean_data"):
+        raise ImportError(f"Functions file missing clean_data() function: {functions_path}")
+
+    clean_fn = module.clean_data
+
+    # Determine output path
+    suffix = Path(input_path).suffix.lower()
+    if output_path is None:
+        output_path = get_default_output_path(input_path)
+
+    # Route by format
+    format_handlers = {
+        ".jsonl": apply_to_jsonl,
+        ".csv": apply_to_csv,
+        ".json": apply_to_json,
+        ".parquet": apply_to_parquet,
+        ".xlsx": apply_to_excel,
+        ".xls": apply_to_excel,
+    }
+
+    handler = format_handlers.get(suffix)
+
+    # Check for text formats (.txt and markitdown extensions, excluding spreadsheets)
+    if handler is None:
+        if suffix == ".txt" or suffix in TEXT_MARKITDOWN_EXTENSIONS:
+            handler = apply_to_text
+
+    if handler is None:
+        raise ValueError(f"Unsupported format: {suffix}")
+
+    # Emit start event (total_records unknown for streaming formats)
+    if on_progress:
+        on_progress({"type": "apply_start", "total_records": None})
+
+    # Apply cleaning
+    total_records = handler(input_path, output_path, clean_fn, on_progress)
+
+    # Emit complete event
+    if on_progress:
+        on_progress({
+            "type": "apply_complete",
+            "total_records": total_records,
+            "output_path": output_path,
+        })
+
+    return output_path

recursive_cleaner/cleaner.py

@@ -17,7 +17,7 @@ from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
 from .types import LLMBackend
-from .validation import check_code_safety, extract_sample_data, split_holdout, validate_function
+from .validation import check_code_safety, extract_modified_fields, extract_sample_data, split_holdout, validate_function
 
 STATE_VERSION = "0.5.0"
 
@@ -63,6 +63,7 @@ class DataCleaner:
         dry_run: bool = False,
         auto_parse: bool = False,
         tui: bool = False,
+        output_path: str = "cleaning_functions.py",
     ):
         self.backend = llm_backend
         self.file_path = file_path
@@ -88,6 +89,7 @@ class DataCleaner:
         self.dry_run = dry_run
         self.auto_parse = auto_parse
         self.tui = tui
+        self.output_path = output_path
         self.functions: list[dict] = []  # List of {name, docstring, code}
         self._tui_renderer = None  # TUIRenderer instance when tui=True
         self._generated_parser: callable | None = None  # LLM-generated parser for unknown formats
@@ -108,6 +110,8 @@ class DataCleaner:
             "min_ms": float("inf"),
             "max_ms": 0.0,
         }
+        # Track fields already covered by generated functions (per chunk)
+        self._fields_covered: set[str] = set()
 
     def _emit(self, event_type: str, chunk_index: int = 0, **kwargs) -> None:
         """Emit a progress event to the callback, if set."""
@@ -520,7 +524,7 @@ class DataCleaner:
                 "quality_delta": 0.0,  # Could be calculated from metrics
                 "latency_total_ms": latency_summary.get("total_ms", 0.0),
                 "llm_calls": latency_summary.get("call_count", 0),
-                "output_file": "cleaning_functions.py",
+                "output_file": self.output_path,
             })
             self._tui_renderer.stop()
 
@@ -531,6 +535,8 @@ class DataCleaner:
         """Process a single chunk, iterating until clean or max iterations."""
         self._emit("chunk_start", chunk_index=chunk_idx)
         error_feedback = ""
+        # Reset fields covered for new chunk
+        self._fields_covered = set()
 
         # Dry run mode: just detect issues, don't generate functions
         if self.dry_run:
@@ -592,6 +598,20 @@ class DataCleaner:
                         print(f"  Safety check failed: {safety_error}")
                     continue
 
+                # Check for duplicate field coverage
+                new_fields = extract_modified_fields(result["code"])
+                overlap = new_fields & self._fields_covered
+                if overlap:
+                    field_list = ", ".join(sorted(overlap))
+                    error_feedback = f"You already generated a function for field(s): {field_list}. This issue is solved. Move on to the next unsolved issue."
+                    self._emit(
+                        "duplicate_field",
+                        chunk_index=chunk_idx,
+                        function_name=result["name"],
+                        fields=list(overlap),
+                    )
+                    continue
+
                 # Runtime validation if enabled
                 if self.validate_runtime:
                     # Use holdout data if available, else sample from generation chunk
@@ -626,6 +646,8 @@ class DataCleaner:
                     "docstring": result["docstring"],
                     "code": result["code"],
                 })
+                # Track fields covered by this function
+                self._fields_covered.update(new_fields)
                 # Track for saturation check
                 self._recent_new_function_count += 1
 
@@ -687,11 +709,11 @@ class DataCleaner:
         self._emit("chunk_done", chunk_index=chunk_idx)
 
     def _write_output(self) -> None:
-        """Write generated functions to cleaning_functions.py."""
+        """Write generated functions to output file."""
         from .output import write_cleaning_file
 
         try:
-            write_cleaning_file(self.functions)
+            write_cleaning_file(self.functions, self.output_path)
         except OutputValidationError as e:
             if not self.tui:
                 print(f"  Error: {e}")
@@ -707,7 +729,7 @@ class DataCleaner:
                     if not self.tui:
                         print(f"  Skipping invalid function: {f['name']}")
             if valid_functions:
-                write_cleaning_file(valid_functions)
+                write_cleaning_file(valid_functions, self.output_path)
             elif not self.tui:
                 print("  No valid functions to write.")