recursive-cleaner 0.6.1__py3-none-any.whl → 0.7.1__py3-none-any.whl

recursive_cleaner/__init__.py

@@ -16,9 +16,10 @@ from recursive_cleaner.optimizer import (
     group_by_salience,
 )
 from recursive_cleaner.output import write_cleaning_file
-from recursive_cleaner.parsers import chunk_file
+from recursive_cleaner.parsers import MARKITDOWN_EXTENSIONS, chunk_file, load_parquet, preprocess_with_markitdown
 from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
+from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
 
 __all__ = [
@@ -27,6 +28,9 @@ __all__ = [
     "MaxIterationsError",
     "OutputValidationError",
     "chunk_file",
+    "MARKITDOWN_EXTENSIONS",
+    "load_parquet",
+    "preprocess_with_markitdown",
     "parse_response",
     "extract_python_block",
     "build_context",
@@ -43,4 +47,6 @@ __all__ = [
     "extract_tags",
     "group_by_salience",
     "consolidate_with_agency",
+    "generate_parser",
+    "check_parser_safety",
 ]

recursive_cleaner/cleaner.py

@@ -12,7 +12,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential
 from .context import build_context
 from .errors import OutputValidationError, ParseError
 from .metrics import QualityMetrics, compare_quality, load_structured_data, measure_quality
-from .parsers import chunk_file
+from .parsers import MARKITDOWN_EXTENSIONS, chunk_file
 from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
@@ -61,6 +61,7 @@ class DataCleaner:
         saturation_check_interval: int = 20,
         report_path: str | None = "cleaning_report.md",
         dry_run: bool = False,
+        auto_parse: bool = False,
     ):
         self.backend = llm_backend
         self.file_path = file_path
@@ -84,7 +85,9 @@ class DataCleaner:
         self.saturation_check_interval = saturation_check_interval
         self.report_path = report_path
         self.dry_run = dry_run
+        self.auto_parse = auto_parse
         self.functions: list[dict] = []  # List of {name, docstring, code}
+        self._generated_parser: callable | None = None  # LLM-generated parser for unknown formats
         # Track recent function generation for saturation check
         self._recent_new_function_count = 0
         self._last_check_function_count = 0
@@ -319,27 +322,72 @@ class DataCleaner:
     def _detect_mode(self) -> Literal["structured", "text"]:
         """Detect mode from file extension."""
         suffix = Path(self.file_path).suffix.lower()
-        structured_extensions = {".jsonl", ".csv", ".json"}
+        # Markitdown formats are processed as text
+        if suffix in MARKITDOWN_EXTENSIONS:
+            return "text"
+        structured_extensions = {".jsonl", ".csv", ".json", ".parquet"}
         if suffix in structured_extensions:
             return "structured"
         return "text"
 
+    def _is_known_extension(self) -> bool:
+        """Check if file extension is natively supported."""
+        suffix = Path(self.file_path).suffix.lower()
+        known = {".jsonl", ".csv", ".json", ".parquet", ".txt"}
+        return suffix in known or suffix in MARKITDOWN_EXTENSIONS
+
+    def _load_with_auto_parser(self) -> list[str]:
+        """Load file using LLM-generated parser, return JSONL chunks."""
+        from .parser_generator import generate_parser
+
+        print(f"Unknown file format, generating parser...")
+        self._emit("parser_generation_start")
+
+        parser = generate_parser(self.backend, self.file_path)
+        self._generated_parser = parser
+
+        self._emit("parser_generation_complete")
+        print("Parser generated successfully.")
+
+        # Parse the file
+        records = parser(self.file_path)
+        if not records:
+            return []
+
+        # Convert to JSONL chunks
+        import json
+        chunks = []
+        for i in range(0, len(records), self.chunk_size):
+            chunk_records = records[i:i + self.chunk_size]
+            chunk_lines = [json.dumps(r) for r in chunk_records]
+            chunks.append("\n".join(chunk_lines))
+
+        return chunks
+
     def run(self) -> None:
         """Run the cleaning pipeline."""
-        # Resolve effective mode
-        if self.mode == "auto":
-            self._effective_mode = self._detect_mode()
+        # Check if we should use auto-parser for unknown formats
+        use_auto_parser = self.auto_parse and not self._is_known_extension()
+
+        if use_auto_parser:
+            # LLM generates parser, always structured mode
+            self._effective_mode = "structured"
+            chunks = self._load_with_auto_parser()
         else:
-            self._effective_mode = self.mode
+            # Resolve effective mode
+            if self.mode == "auto":
+                self._effective_mode = self._detect_mode()
+            else:
+                self._effective_mode = self.mode
 
-        chunks = chunk_file(
-            self.file_path,
-            self.chunk_size,
-            mode=self._effective_mode,
-            chunk_overlap=self.chunk_overlap,
-            sampling_strategy=self.sampling_strategy,
-            stratify_field=self.stratify_field,
-        )
+            chunks = chunk_file(
+                self.file_path,
+                self.chunk_size,
+                mode=self._effective_mode,
+                chunk_overlap=self.chunk_overlap,
+                sampling_strategy=self.sampling_strategy,
+                stratify_field=self.stratify_field,
+            )
 
         if not chunks:
             print("No data to process.")

recursive_cleaner/parser_generator.py

@@ -0,0 +1,123 @@
+"""LLM-generated parser for unknown file formats."""
+
+import ast
+import re
+from pathlib import Path
+
+from .types import LLMBackend
+
+# Dangerous patterns for parser code (allows 'open' since parsers need file I/O)
+_DANGEROUS_IMPORTS = frozenset({
+    "os", "subprocess", "sys", "shutil", "socket", "urllib",
+    "requests", "httplib", "ftplib", "smtplib", "pickle",
+})
+_DANGEROUS_CALLS = frozenset({"eval", "exec", "compile", "__import__"})
+
+PARSER_PROMPT = '''You are a data parsing expert. Generate a Python function to parse this file format.
+
+=== SAMPLE (first 4KB) ===
+{sample}
+
+=== TASK ===
+Generate a function with this EXACT signature:
+
+```python
+def parse_file(file_path: str) -> list[dict]:
+    """Parse the file into a list of records."""
+    # Your implementation
+```
+
+RULES:
+- Return list of dicts, one dict per logical record
+- Use only stdlib (xml.etree, json, re, csv)
+- Handle the ENTIRE file, not just this sample
+- Be defensive about malformed data
+- Include necessary imports inside or before the function
+'''
+
+
+def check_parser_safety(code: str) -> list[str]:
+    """Check parser code for dangerous patterns. Returns list of issues."""
+    issues = []
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        return [f"Syntax error: {e}"]
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                module = alias.name.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: {alias.name}")
+        if isinstance(node, ast.ImportFrom):
+            if node.module:
+                module = node.module.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: from {node.module}")
+        if isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name):
+                if node.func.id in _DANGEROUS_CALLS:
+                    issues.append(f"Dangerous call: {node.func.id}()")
+    return issues
+
+
+def extract_python_block(text: str) -> str:
+    """Extract code from ```python ... ``` block."""
+    match = re.search(r"```python\s*(.*?)\s*```", text, re.DOTALL)
+    return match.group(1).strip() if match else text.strip()
+
+
+def generate_parser(llm_backend: LLMBackend, file_path: str) -> callable:
+    """
+    Generate a parser function for an unknown file format.
+
+    Args:
+        llm_backend: LLM backend implementing generate(prompt) -> str
+        file_path: Path to the file to parse
+
+    Returns:
+        A callable parse_file(file_path) -> list[dict]
+
+    Raises:
+        ValueError: If generated code is unsafe, has invalid syntax,
+                   or doesn't return list of dicts
+    """
+    path = Path(file_path)
+    with open(path, "r", errors="replace") as f:
+        sample = f.read(4096)
+
+    prompt = PARSER_PROMPT.format(sample=sample)
+    response = llm_backend.generate(prompt)
+    code = extract_python_block(response)
+
+    # Validate syntax
+    try:
+        ast.parse(code)
+    except SyntaxError as e:
+        raise ValueError(f"Generated parser has invalid syntax: {e}")
+
+    # Security check
+    issues = check_parser_safety(code)
+    if issues:
+        raise ValueError(f"Generated parser contains dangerous code: {issues}")
+
+    # Execute to get function
+    namespace: dict = {}
+    exec(code, namespace)
+
+    if "parse_file" not in namespace:
+        raise ValueError("Generated code must define 'parse_file' function")
+
+    parser = namespace["parse_file"]
+
+    # Validate on actual file
+    result = parser(file_path)
+    if not isinstance(result, list):
+        raise ValueError(f"Parser must return list, got {type(result).__name__}")
+    if result and not isinstance(result[0], dict):
+        raise ValueError(
+            f"Parser must return list of dicts, got list of {type(result[0]).__name__}"
+        )
+
+    return parser

recursive_cleaner/parsers.py

@@ -17,6 +17,62 @@ try:
 except ImportError:
     _HAS_SENTENCE_CHUNKER = False
 
+# File extensions supported by markitdown for conversion to text
+MARKITDOWN_EXTENSIONS = {
+    ".pdf", ".docx", ".doc", ".xlsx", ".xls", ".pptx", ".ppt",
+    ".html", ".htm", ".epub", ".msg", ".rtf", ".odt", ".ods", ".odp"
+}
+
+
+def load_parquet(file_path: str) -> list[dict]:
+    """Load parquet file as list of dicts.
+
+    Args:
+        file_path: Path to the parquet file
+
+    Returns:
+        List of dictionaries, one per row
+
+    Raises:
+        ImportError: If pyarrow is not installed
+    """
+    try:
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for parquet files. "
+            "Install with: pip install recursive-cleaner[parquet]"
+        )
+
+    table = pq.read_table(file_path)
+    return table.to_pylist()
+
+
+def preprocess_with_markitdown(file_path: str) -> str:
+    """
+    Convert supported formats to text using markitdown.
+
+    Args:
+        file_path: Path to the file to convert
+
+    Returns:
+        Extracted text content from the file
+
+    Raises:
+        ImportError: If markitdown is not installed
+    """
+    try:
+        from markitdown import MarkItDown
+    except ImportError:
+        raise ImportError(
+            "markitdown is required for this file type. "
+            "Install with: pip install recursive-cleaner[markitdown]"
+        )
+
+    md = MarkItDown()
+    result = md.convert(file_path)
+    return result.text_content
+
 
 def chunk_file(
     file_path: str,
@@ -50,6 +106,25 @@ def chunk_file(
     if not path.exists():
         raise FileNotFoundError(f"File not found: {file_path}")
 
+    # Handle markitdown formats: preprocess to text, then chunk as text
+    if suffix in MARKITDOWN_EXTENSIONS:
+        content = preprocess_with_markitdown(file_path)
+        if not content.strip():
+            return []
+        # Markitdown output is always processed as text
+        if sampling_strategy != "sequential":
+            raise ValueError(
+                f"Text mode only supports 'sequential' sampling, got '{sampling_strategy}'"
+            )
+        return chunk_text_sentences(content, chunk_size, chunk_overlap)
+
+    # Handle parquet files: load as list of dicts, chunk like JSONL
+    if suffix == ".parquet":
+        records = load_parquet(file_path)
+        if not records:
+            return []
+        return _chunk_records(records, chunk_size, sampling_strategy, stratify_field)
+
     content = path.read_text(encoding="utf-8")
 
     if not content.strip():
@@ -79,7 +154,7 @@ def chunk_file(
 
 def _detect_mode(suffix: str) -> Literal["structured", "text"]:
     """Detect mode from file extension."""
-    structured_extensions = {".jsonl", ".csv", ".json"}
+    structured_extensions = {".jsonl", ".csv", ".json", ".parquet"}
     if suffix in structured_extensions:
         return "structured"
     return "text"
@@ -281,6 +356,61 @@ def _chunk_jsonl(
     return chunks
 
 
+def _chunk_records(
+    records: list[dict],
+    item_count: int,
+    sampling_strategy: Literal["sequential", "random", "stratified"] = "sequential",
+    stratify_field: str | None = None,
+) -> list[str]:
+    """Chunk a list of dicts by item count with optional sampling."""
+    if not records:
+        return []
+
+    # For seed computation, use JSON representation
+    seed = _compute_seed(json.dumps(records[0]))
+
+    # Apply sampling strategy
+    if sampling_strategy == "random":
+        records = _shuffle_records(records, seed)
+    elif sampling_strategy == "stratified" and stratify_field:
+        records = _stratified_sample_dicts(records, stratify_field, seed)
+
+    chunks = []
+    for i in range(0, len(records), item_count):
+        chunk_records = records[i:i + item_count]
+        # Convert to JSONL format for LLM context
+        chunk_lines = [json.dumps(r) for r in chunk_records]
+        chunks.append("\n".join(chunk_lines))
+
+    return chunks
+
+
+def _stratified_sample_dicts(records: list[dict], field: str, seed: int) -> list[dict]:
+    """Group dicts by field, interleave proportionally."""
+    groups: dict[str, list] = {}
+    for record in records:
+        key = str(record.get(field, "_missing_"))
+        if key not in groups:
+            groups[key] = []
+        groups[key].append(record)
+
+    # Shuffle within each group
+    rng = random.Random(seed)
+    for key in groups:
+        rng.shuffle(groups[key])
+
+    # Interleave from groups (round-robin)
+    result = []
+    group_lists = list(groups.values())
+    while any(group_lists):
+        for g in group_lists:
+            if g:
+                result.append(g.pop(0))
+        group_lists = [g for g in group_lists if g]
+
+    return result
+
+
 def _compute_seed(content: str) -> int:
     """Compute deterministic seed from content hash."""
     return int(hashlib.md5(content.encode("utf-8")).hexdigest()[:8], 16)

{recursive_cleaner-0.6.1.dist-info → recursive_cleaner-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 0.6.1
+Version: 0.7.1
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
@@ -26,8 +26,12 @@ Requires-Dist: tenacity>=8.0
 Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: markitdown
+Requires-Dist: markitdown>=0.1.0; extra == 'markitdown'
 Provides-Extra: mlx
 Requires-Dist: mlx-lm>=0.10.0; extra == 'mlx'
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=14.0.0; extra == 'parquet'
 Description-Content-Type: text/markdown
 
 # Recursive Data Cleaner
@@ -36,7 +40,7 @@ LLM-powered incremental data cleaning for massive datasets. Process files in chu
 
 ## How It Works
 
-1. **Chunk** your data (JSONL, CSV, JSON, or text)
+1. **Chunk** your data (JSONL, CSV, JSON, Parquet, PDF, Word, Excel, XML, and more)
 2. **Analyze** each chunk with an LLM to identify issues
 3. **Generate** one cleaning function per issue
 4. **Validate** functions on holdout data before accepting
@@ -55,6 +59,16 @@ For Apple Silicon (MLX backend):
 pip install -e ".[mlx]"
 ```
 
+For document conversion (PDF, Word, Excel, HTML, etc.):
+```bash
+pip install -e ".[markitdown]"
+```
+
+For Parquet files:
+```bash
+pip install -e ".[parquet]"
+```
+
 ## Quick Start
 
 ```python
@@ -107,6 +121,11 @@ cleaner.run()  # Generates cleaning_functions.py
 - **Cleaning Reports**: Markdown summary with functions, timing, quality delta
 - **Dry-Run Mode**: Analyze data without generating functions
 
+### Format Expansion (v0.7.0)
+- **Markitdown Integration**: Convert 20+ formats (PDF, Word, Excel, PowerPoint, HTML, EPUB, etc.) to text
+- **Parquet Support**: Load parquet files as structured data via pyarrow
+- **LLM-Generated Parsers**: Auto-generate parsers for XML and unknown formats (`auto_parse=True`)
+
 ## Configuration
 
 ```python
@@ -138,6 +157,9 @@ cleaner = DataCleaner(
     report_path="report.md",    # Markdown report output (None to disable)
     dry_run=False,              # Analyze without generating functions
 
+    # Format Expansion
+    auto_parse=False,           # LLM generates parser for unknown formats
+
     # Progress & State
     on_progress=callback,       # Progress event callback
     state_file="state.json",    # Enable resume on interrupt
@@ -231,20 +253,21 @@ cleaner.run()
 
 ```
 recursive_cleaner/
-├── cleaner.py       # Main DataCleaner class (~580 lines)
-├── context.py       # Docstring registry with FIFO eviction
-├── dependencies.py  # Topological sort for function ordering
-├── metrics.py       # Quality metrics before/after
-├── optimizer.py     # Two-pass consolidation with LLM agency
-├── output.py        # Function file generation + import consolidation
-├── parsers.py       # Chunking for JSONL/CSV/JSON/text + sampling
-├── prompt.py        # LLM prompt templates
-├── report.py        # Markdown report generation
-├── response.py      # XML/markdown parsing + agency dataclasses
-├── schema.py        # Schema inference
-├── validation.py    # Runtime validation + holdout
+├── cleaner.py          # Main DataCleaner class
+├── context.py          # Docstring registry with FIFO eviction
+├── dependencies.py     # Topological sort for function ordering
+├── metrics.py          # Quality metrics before/after
+├── optimizer.py        # Two-pass consolidation with LLM agency
+├── output.py           # Function file generation + import consolidation
+├── parser_generator.py # LLM-generated parsers for unknown formats
+├── parsers.py          # Chunking for all formats + sampling
+├── prompt.py           # LLM prompt templates
+├── report.py           # Markdown report generation
+├── response.py         # XML/markdown parsing + agency dataclasses
+├── schema.py           # Schema inference
+├── validation.py       # Runtime validation + holdout
 └── vendor/
-    └── chunker.py   # Vendored sentence-aware chunker
+    └── chunker.py      # Vendored sentence-aware chunker
 ```
 
 ## Testing
@@ -253,7 +276,7 @@ recursive_cleaner/
 pytest tests/ -v
 ```
 
-392 tests covering all features. Test datasets in `test_cases/`:
+432 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data
@@ -269,6 +292,7 @@ pytest tests/ -v
 
 | Version | Features |
 |---------|----------|
+| v0.7.0 | Markitdown (20+ formats), Parquet support, LLM-generated parsers |
 | v0.6.0 | Latency metrics, import consolidation, cleaning report, dry-run mode |
 | v0.5.1 | Dangerous code detection (AST-based security) |
 | v0.5.0 | Two-pass optimization, early termination, LLM agency |

{recursive_cleaner-0.6.1.dist-info → recursive_cleaner-0.7.1.dist-info}/RECORD

@@ -1,14 +1,15 @@
 backends/__init__.py,sha256=FUgODeYSGBvT0-z6myVby6YeAHG0nEUgWLITBKobUew,121
 backends/mlx_backend.py,sha256=0U6IqmDHyk4vjKzytvEcQvSUBryQTgFtsNOcpwFNKk8,2945
-recursive_cleaner/__init__.py,sha256=Wm_sQdh1mkJ-DByo4lMiAIiLWFRfj8G2Limr0eSG0s0,1416
-recursive_cleaner/cleaner.py,sha256=vSrogf8T1AquLakmqbpgvuFoTD6_AZp_hrG3vJxx9gk,24340
+recursive_cleaner/__init__.py,sha256=bG83PcmkxAYMC17FmKuyMJUrMnuukp32JO3rlCLyB-Q,1698
+recursive_cleaner/cleaner.py,sha256=J2X5bnk2OsWJyOn4BNR-cj0sqeKCylznfs_WEyMGxG8,26280
 recursive_cleaner/context.py,sha256=avMXRDxLd7nd8CKWtvPHQy1MFhBKiA0aUVVJIlWoLZ4,824
 recursive_cleaner/dependencies.py,sha256=vlYeoGL517v3yUSWN0wYDuIs9OOuQwM_dCBADrlitW8,2080
 recursive_cleaner/errors.py,sha256=hwRJF8NSmWy_FZHCxcZDZxLQ0zqvo5dX8ImkB9mrOYc,433
 recursive_cleaner/metrics.py,sha256=C6RlvZMTtVm7kdRhfMZu4xd-R-i1EQGMT5FCasPOO3A,5003
 recursive_cleaner/optimizer.py,sha256=lnQC9Y1ClkW4po1eYa2bnYYu4smiDuUpMPPX6EN1UQ0,9700
 recursive_cleaner/output.py,sha256=quTlZYtKZm9h37mbnwQmEjg0q8VQSZWEqwaHfhSAd3s,6106
-recursive_cleaner/parsers.py,sha256=39oMg0WGRFV_eRBzOfB7LIGXMP1cIDwdeYk4UOlw140,10595
+recursive_cleaner/parser_generator.py,sha256=enn6_okGWB2ddVkwI7ytndT04S4QEVAk6cbmb7shxcM,3905
+recursive_cleaner/parsers.py,sha256=HCS2UiVFhboq_go4DyWUygkJTkpfYkFj9_hqWiGIEXo,14572
 recursive_cleaner/prompt.py,sha256=ep0eOXz_XbhH3HduJ76LvzVSftonhcv4GLEecIqd3lY,6484
 recursive_cleaner/report.py,sha256=AWWneRjvl76ccLlExdkKJeY3GVFUG_LtmzVIJJT5cFI,4629
 recursive_cleaner/response.py,sha256=3w0mLnqEPdB4daMSF0mtTcG0PTP-utb1HFtKuYA1ljw,9064
@@ -17,7 +18,7 @@ recursive_cleaner/types.py,sha256=-GdCmsfHd3rfdfCi5c-RXqX4TyuCSHgA__3AF3bMhoQ,29
 recursive_cleaner/validation.py,sha256=-KAolhw3GQyhHwmh0clEj8xqPD5O-R2AO5rx7vubIME,6442
 recursive_cleaner/vendor/__init__.py,sha256=E87TjmjRzu8ty39nqThvBwM611yXlLKQZ6KGY_zp3Dk,117
 recursive_cleaner/vendor/chunker.py,sha256=pDDbfF6FoSmUji0-RG4MletPxJ-VybGw0yfnhh0aMSQ,6730
-recursive_cleaner-0.6.1.dist-info/METADATA,sha256=q7MqUiWUUZ2zlhujiR725ydPOKSIqCEG01ncXkI1NkA,9328
-recursive_cleaner-0.6.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-recursive_cleaner-0.6.1.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
-recursive_cleaner-0.6.1.dist-info/RECORD,,
+recursive_cleaner-0.7.1.dist-info/METADATA,sha256=X5_HVPMIPUULKKIgDvqhN0ZRQQBcZ1lupGb9frLdCSI,10258
+recursive_cleaner-0.7.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+recursive_cleaner-0.7.1.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
+recursive_cleaner-0.7.1.dist-info/RECORD,,