recursive-cleaner 0.6.1__tar.gz → 0.7.1__tar.gz

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 0.6.1
+Version: 0.7.1
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
@@ -26,8 +26,12 @@ Requires-Dist: tenacity>=8.0
 Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: markitdown
+Requires-Dist: markitdown>=0.1.0; extra == 'markitdown'
 Provides-Extra: mlx
 Requires-Dist: mlx-lm>=0.10.0; extra == 'mlx'
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=14.0.0; extra == 'parquet'
 Description-Content-Type: text/markdown
 
 # Recursive Data Cleaner
@@ -36,7 +40,7 @@ LLM-powered incremental data cleaning for massive datasets. Process files in chu
 
 ## How It Works
 
-1. **Chunk** your data (JSONL, CSV, JSON, or text)
+1. **Chunk** your data (JSONL, CSV, JSON, Parquet, PDF, Word, Excel, XML, and more)
 2. **Analyze** each chunk with an LLM to identify issues
 3. **Generate** one cleaning function per issue
 4. **Validate** functions on holdout data before accepting
@@ -55,6 +59,16 @@ For Apple Silicon (MLX backend):
 pip install -e ".[mlx]"
 ```
 
+For document conversion (PDF, Word, Excel, HTML, etc.):
+```bash
+pip install -e ".[markitdown]"
+```
+
+For Parquet files:
+```bash
+pip install -e ".[parquet]"
+```
+
 ## Quick Start
 
 ```python
@@ -107,6 +121,11 @@ cleaner.run()  # Generates cleaning_functions.py
 - **Cleaning Reports**: Markdown summary with functions, timing, quality delta
 - **Dry-Run Mode**: Analyze data without generating functions
 
+### Format Expansion (v0.7.0)
+- **Markitdown Integration**: Convert 20+ formats (PDF, Word, Excel, PowerPoint, HTML, EPUB, etc.) to text
+- **Parquet Support**: Load parquet files as structured data via pyarrow
+- **LLM-Generated Parsers**: Auto-generate parsers for XML and unknown formats (`auto_parse=True`)
+
 ## Configuration
 
 ```python
@@ -138,6 +157,9 @@ cleaner = DataCleaner(
     report_path="report.md",    # Markdown report output (None to disable)
     dry_run=False,              # Analyze without generating functions
 
+    # Format Expansion
+    auto_parse=False,           # LLM generates parser for unknown formats
+
     # Progress & State
     on_progress=callback,       # Progress event callback
     state_file="state.json",    # Enable resume on interrupt
@@ -231,20 +253,21 @@ cleaner.run()
 
 ```
 recursive_cleaner/
-├── cleaner.py       # Main DataCleaner class (~580 lines)
-├── context.py       # Docstring registry with FIFO eviction
-├── dependencies.py  # Topological sort for function ordering
-├── metrics.py       # Quality metrics before/after
-├── optimizer.py     # Two-pass consolidation with LLM agency
-├── output.py        # Function file generation + import consolidation
-├── parsers.py       # Chunking for JSONL/CSV/JSON/text + sampling
-├── prompt.py        # LLM prompt templates
-├── report.py        # Markdown report generation
-├── response.py      # XML/markdown parsing + agency dataclasses
-├── schema.py        # Schema inference
-├── validation.py    # Runtime validation + holdout
+├── cleaner.py          # Main DataCleaner class
+├── context.py          # Docstring registry with FIFO eviction
+├── dependencies.py     # Topological sort for function ordering
+├── metrics.py          # Quality metrics before/after
+├── optimizer.py        # Two-pass consolidation with LLM agency
+├── output.py           # Function file generation + import consolidation
+├── parser_generator.py # LLM-generated parsers for unknown formats
+├── parsers.py          # Chunking for all formats + sampling
+├── prompt.py           # LLM prompt templates
+├── report.py           # Markdown report generation
+├── response.py         # XML/markdown parsing + agency dataclasses
+├── schema.py           # Schema inference
+├── validation.py       # Runtime validation + holdout
 └── vendor/
-    └── chunker.py   # Vendored sentence-aware chunker
+    └── chunker.py      # Vendored sentence-aware chunker
 ```
 
 ## Testing
@@ -253,7 +276,7 @@ recursive_cleaner/
 pytest tests/ -v
 ```
 
-392 tests covering all features. Test datasets in `test_cases/`:
+432 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data
@@ -269,6 +292,7 @@ pytest tests/ -v
 
 | Version | Features |
 |---------|----------|
+| v0.7.0 | Markitdown (20+ formats), Parquet support, LLM-generated parsers |
 | v0.6.0 | Latency metrics, import consolidation, cleaning report, dry-run mode |
 | v0.5.1 | Dangerous code detection (AST-based security) |
 | v0.5.0 | Two-pass optimization, early termination, LLM agency |

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/README.md

@@ -4,7 +4,7 @@ LLM-powered incremental data cleaning for massive datasets. Process files in chu
 
 ## How It Works
 
-1. **Chunk** your data (JSONL, CSV, JSON, or text)
+1. **Chunk** your data (JSONL, CSV, JSON, Parquet, PDF, Word, Excel, XML, and more)
 2. **Analyze** each chunk with an LLM to identify issues
 3. **Generate** one cleaning function per issue
 4. **Validate** functions on holdout data before accepting
@@ -23,6 +23,16 @@ For Apple Silicon (MLX backend):
 pip install -e ".[mlx]"
 ```
 
+For document conversion (PDF, Word, Excel, HTML, etc.):
+```bash
+pip install -e ".[markitdown]"
+```
+
+For Parquet files:
+```bash
+pip install -e ".[parquet]"
+```
+
 ## Quick Start
 
 ```python
@@ -75,6 +85,11 @@ cleaner.run()  # Generates cleaning_functions.py
 - **Cleaning Reports**: Markdown summary with functions, timing, quality delta
 - **Dry-Run Mode**: Analyze data without generating functions
 
+### Format Expansion (v0.7.0)
+- **Markitdown Integration**: Convert 20+ formats (PDF, Word, Excel, PowerPoint, HTML, EPUB, etc.) to text
+- **Parquet Support**: Load parquet files as structured data via pyarrow
+- **LLM-Generated Parsers**: Auto-generate parsers for XML and unknown formats (`auto_parse=True`)
+
 ## Configuration
 
 ```python
@@ -106,6 +121,9 @@ cleaner = DataCleaner(
     report_path="report.md",    # Markdown report output (None to disable)
     dry_run=False,              # Analyze without generating functions
 
+    # Format Expansion
+    auto_parse=False,           # LLM generates parser for unknown formats
+
     # Progress & State
     on_progress=callback,       # Progress event callback
     state_file="state.json",    # Enable resume on interrupt
@@ -199,20 +217,21 @@ cleaner.run()
 
 ```
 recursive_cleaner/
-├── cleaner.py       # Main DataCleaner class (~580 lines)
-├── context.py       # Docstring registry with FIFO eviction
-├── dependencies.py  # Topological sort for function ordering
-├── metrics.py       # Quality metrics before/after
-├── optimizer.py     # Two-pass consolidation with LLM agency
-├── output.py        # Function file generation + import consolidation
-├── parsers.py       # Chunking for JSONL/CSV/JSON/text + sampling
-├── prompt.py        # LLM prompt templates
-├── report.py        # Markdown report generation
-├── response.py      # XML/markdown parsing + agency dataclasses
-├── schema.py        # Schema inference
-├── validation.py    # Runtime validation + holdout
+├── cleaner.py          # Main DataCleaner class
+├── context.py          # Docstring registry with FIFO eviction
+├── dependencies.py     # Topological sort for function ordering
+├── metrics.py          # Quality metrics before/after
+├── optimizer.py        # Two-pass consolidation with LLM agency
+├── output.py           # Function file generation + import consolidation
+├── parser_generator.py # LLM-generated parsers for unknown formats
+├── parsers.py          # Chunking for all formats + sampling
+├── prompt.py           # LLM prompt templates
+├── report.py           # Markdown report generation
+├── response.py         # XML/markdown parsing + agency dataclasses
+├── schema.py           # Schema inference
+├── validation.py       # Runtime validation + holdout
 └── vendor/
-    └── chunker.py   # Vendored sentence-aware chunker
+    └── chunker.py      # Vendored sentence-aware chunker
 ```
 
 ## Testing
@@ -221,7 +240,7 @@ recursive_cleaner/
 pytest tests/ -v
 ```
 
-392 tests covering all features. Test datasets in `test_cases/`:
+432 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data
@@ -237,6 +256,7 @@ pytest tests/ -v
 
 | Version | Features |
 |---------|----------|
+| v0.7.0 | Markitdown (20+ formats), Parquet support, LLM-generated parsers |
 | v0.6.0 | Latency metrics, import consolidation, cleaning report, dry-run mode |
 | v0.5.1 | Dangerous code detection (AST-based security) |
 | v0.5.0 | Two-pass optimization, early termination, LLM agency |

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/TODO.md

@@ -60,30 +60,42 @@ These patterns proved high-value with low implementation effort:
 
 ---
 
-## Future Considerations
+## Tier 5: Format Expansion & UI (v0.7.0) - PLANNED
+
+### Markitdown Integration
+- [ ] Add markitdown as optional dependency
+- [ ] Auto-convert 20+ formats: Excel, HTML, Word, PDF, PowerPoint, EPUB, etc.
+- [ ] Preprocessing step before chunking
+- **Approach**: `pip install recursive-cleaner[markitdown]`
+
+### Parquet Support
+- [ ] Native parser using pyarrow
+- [ ] Read as list of dicts (same as JSONL)
+- **Approach**: Optional dependency, ~10 lines of code
+
+### LLM-Generated Parsers
+- [ ] For XML and unknown formats
+- [ ] Send sample to LLM: "Generate a function to parse this into list of records"
+- [ ] Validate generated parser on sample before using
+- **Approach**: Wu wei - let LLM decide how to parse data it understands
+
+### Terminal UI (Textual)
+- [ ] Optional `[ui]` extra dependency
+- [ ] Live dashboard showing: chunk progress, function generation, latency sparkline
+- [ ] Pure terminal, no browser needed
+- **Approach**: `pip install recursive-cleaner[ui]`
 
-Ideas that might be valuable but need more thought.
+---
 
-### Confidence Scoring
-- LLM rates confidence in each generated function (high/medium/low)
-- Low confidence = flag for human review
-- **Question**: Does this actually help users, or just add noise?
+## Future Considerations
 
-### Before/After Examples
-- User provides expected input→output pairs
-- Validate generated functions match expectations
-- **Question**: How to handle functions that transform data differently but correctly?
+Ideas that might be valuable but need more thought.
 
 ### Multi-File Batch Mode
 - Process multiple files with shared function registry
 - Functions learned from file A applied to file B
 - **Question**: How to handle schema differences between files?
 
-### Summary Buffer Memory
-- Compress old function docstrings into summaries
-- Keep recent functions verbatim
-- **Question**: Does FIFO eviction already work well enough?
-
 ---
 
 ## Explicitly Deferred

recursive_cleaner-0.7.1/docs/contracts/v070-success-criteria.md

@@ -0,0 +1,13 @@
+# Success Criteria - v0.7.0 Format Expansion
+
+## Project-Level Success
+- [ ] Markitdown integration converts 20+ formats to text before chunking
+- [ ] Parquet files load as list of dicts like JSONL/CSV
+- [ ] LLM-generated parsers handle XML and unknown formats
+- [ ] All new formats integrate seamlessly with existing cleaning pipeline
+- [ ] Optional dependencies don't break core functionality when not installed
+- [ ] All 392 existing tests still pass
+
+## Phase Success Criteria
+
+[To be populated during planning]

recursive_cleaner-0.7.1/docs/workflow-state.md

@@ -0,0 +1,26 @@
+# Workflow State - v0.7.0 Format Expansion
+
+## Current Phase
+Research
+
+## Awaiting
+Subagent Completion (Research)
+
+## Blockers
+None
+
+## Progress
+- [ ] Research complete
+- [ ] Contracts approved
+- [ ] Plan approved
+- [ ] Phase 1: Markitdown integration
+- [ ] Phase 1 audit
+- [ ] Phase 2: Parquet support
+- [ ] Phase 2 audit
+- [ ] Phase 3: LLM-generated parsers
+- [ ] Phase 3 audit
+
+## Previous Version (v0.6.0)
+- **Tests**: 392 passing
+- **Lines**: 2,967 total
+- **Status**: Released on GitHub + PyPI

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "recursive-cleaner"
-version = "0.6.1"
+version = "0.7.1"
 description = "LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions"
 readme = "README.md"
 license = "MIT"
@@ -46,6 +46,12 @@ dev = [
 mlx = [
     "mlx-lm>=0.10.0",
 ]
+markitdown = [
+    "markitdown>=0.1.0",
+]
+parquet = [
+    "pyarrow>=14.0.0",
+]
 
 [project.urls]
 Homepage = "https://github.com/gaztrabisme/recursive-data-cleaner"

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/recursive_cleaner/__init__.py

@@ -16,9 +16,10 @@ from recursive_cleaner.optimizer import (
     group_by_salience,
 )
 from recursive_cleaner.output import write_cleaning_file
-from recursive_cleaner.parsers import chunk_file
+from recursive_cleaner.parsers import MARKITDOWN_EXTENSIONS, chunk_file, load_parquet, preprocess_with_markitdown
 from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
+from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
 
 __all__ = [
@@ -27,6 +28,9 @@ __all__ = [
     "MaxIterationsError",
     "OutputValidationError",
     "chunk_file",
+    "MARKITDOWN_EXTENSIONS",
+    "load_parquet",
+    "preprocess_with_markitdown",
     "parse_response",
     "extract_python_block",
     "build_context",
@@ -43,4 +47,6 @@ __all__ = [
     "extract_tags",
     "group_by_salience",
     "consolidate_with_agency",
+    "generate_parser",
+    "check_parser_safety",
 ]

{recursive_cleaner-0.6.1 → recursive_cleaner-0.7.1}/recursive_cleaner/cleaner.py

@@ -12,7 +12,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential
 from .context import build_context
 from .errors import OutputValidationError, ParseError
 from .metrics import QualityMetrics, compare_quality, load_structured_data, measure_quality
-from .parsers import chunk_file
+from .parsers import MARKITDOWN_EXTENSIONS, chunk_file
 from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
@@ -61,6 +61,7 @@ class DataCleaner:
         saturation_check_interval: int = 20,
         report_path: str | None = "cleaning_report.md",
         dry_run: bool = False,
+        auto_parse: bool = False,
     ):
         self.backend = llm_backend
         self.file_path = file_path
@@ -84,7 +85,9 @@ class DataCleaner:
         self.saturation_check_interval = saturation_check_interval
         self.report_path = report_path
         self.dry_run = dry_run
+        self.auto_parse = auto_parse
         self.functions: list[dict] = []  # List of {name, docstring, code}
+        self._generated_parser: callable | None = None  # LLM-generated parser for unknown formats
         # Track recent function generation for saturation check
         self._recent_new_function_count = 0
         self._last_check_function_count = 0
@@ -319,27 +322,72 @@ class DataCleaner:
     def _detect_mode(self) -> Literal["structured", "text"]:
         """Detect mode from file extension."""
         suffix = Path(self.file_path).suffix.lower()
-        structured_extensions = {".jsonl", ".csv", ".json"}
+        # Markitdown formats are processed as text
+        if suffix in MARKITDOWN_EXTENSIONS:
+            return "text"
+        structured_extensions = {".jsonl", ".csv", ".json", ".parquet"}
         if suffix in structured_extensions:
             return "structured"
         return "text"
 
+    def _is_known_extension(self) -> bool:
+        """Check if file extension is natively supported."""
+        suffix = Path(self.file_path).suffix.lower()
+        known = {".jsonl", ".csv", ".json", ".parquet", ".txt"}
+        return suffix in known or suffix in MARKITDOWN_EXTENSIONS
+
+    def _load_with_auto_parser(self) -> list[str]:
+        """Load file using LLM-generated parser, return JSONL chunks."""
+        from .parser_generator import generate_parser
+
+        print(f"Unknown file format, generating parser...")
+        self._emit("parser_generation_start")
+
+        parser = generate_parser(self.backend, self.file_path)
+        self._generated_parser = parser
+
+        self._emit("parser_generation_complete")
+        print("Parser generated successfully.")
+
+        # Parse the file
+        records = parser(self.file_path)
+        if not records:
+            return []
+
+        # Convert to JSONL chunks
+        import json
+        chunks = []
+        for i in range(0, len(records), self.chunk_size):
+            chunk_records = records[i:i + self.chunk_size]
+            chunk_lines = [json.dumps(r) for r in chunk_records]
+            chunks.append("\n".join(chunk_lines))
+
+        return chunks
+
     def run(self) -> None:
         """Run the cleaning pipeline."""
-        # Resolve effective mode
-        if self.mode == "auto":
-            self._effective_mode = self._detect_mode()
+        # Check if we should use auto-parser for unknown formats
+        use_auto_parser = self.auto_parse and not self._is_known_extension()
+
+        if use_auto_parser:
+            # LLM generates parser, always structured mode
+            self._effective_mode = "structured"
+            chunks = self._load_with_auto_parser()
         else:
-            self._effective_mode = self.mode
+            # Resolve effective mode
+            if self.mode == "auto":
+                self._effective_mode = self._detect_mode()
+            else:
+                self._effective_mode = self.mode
 
-        chunks = chunk_file(
-            self.file_path,
-            self.chunk_size,
-            mode=self._effective_mode,
-            chunk_overlap=self.chunk_overlap,
-            sampling_strategy=self.sampling_strategy,
-            stratify_field=self.stratify_field,
-        )
+            chunks = chunk_file(
+                self.file_path,
+                self.chunk_size,
+                mode=self._effective_mode,
+                chunk_overlap=self.chunk_overlap,
+                sampling_strategy=self.sampling_strategy,
+                stratify_field=self.stratify_field,
+            )
 
         if not chunks:
             print("No data to process.")

recursive_cleaner-0.7.1/recursive_cleaner/parser_generator.py

@@ -0,0 +1,123 @@
+"""LLM-generated parser for unknown file formats."""
+
+import ast
+import re
+from pathlib import Path
+
+from .types import LLMBackend
+
+# Dangerous patterns for parser code (allows 'open' since parsers need file I/O)
+_DANGEROUS_IMPORTS = frozenset({
+    "os", "subprocess", "sys", "shutil", "socket", "urllib",
+    "requests", "httplib", "ftplib", "smtplib", "pickle",
+})
+_DANGEROUS_CALLS = frozenset({"eval", "exec", "compile", "__import__"})
+
+PARSER_PROMPT = '''You are a data parsing expert. Generate a Python function to parse this file format.
+
+=== SAMPLE (first 4KB) ===
+{sample}
+
+=== TASK ===
+Generate a function with this EXACT signature:
+
+```python
+def parse_file(file_path: str) -> list[dict]:
+    """Parse the file into a list of records."""
+    # Your implementation
+```
+
+RULES:
+- Return list of dicts, one dict per logical record
+- Use only stdlib (xml.etree, json, re, csv)
+- Handle the ENTIRE file, not just this sample
+- Be defensive about malformed data
+- Include necessary imports inside or before the function
+'''
+
+
+def check_parser_safety(code: str) -> list[str]:
+    """Check parser code for dangerous patterns. Returns list of issues."""
+    issues = []
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        return [f"Syntax error: {e}"]
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                module = alias.name.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: {alias.name}")
+        if isinstance(node, ast.ImportFrom):
+            if node.module:
+                module = node.module.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: from {node.module}")
+        if isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name):
+                if node.func.id in _DANGEROUS_CALLS:
+                    issues.append(f"Dangerous call: {node.func.id}()")
+    return issues
+
+
+def extract_python_block(text: str) -> str:
+    """Extract code from ```python ... ``` block."""
+    match = re.search(r"```python\s*(.*?)\s*```", text, re.DOTALL)
+    return match.group(1).strip() if match else text.strip()
+
+
+def generate_parser(llm_backend: LLMBackend, file_path: str) -> callable:
+    """
+    Generate a parser function for an unknown file format.
+
+    Args:
+        llm_backend: LLM backend implementing generate(prompt) -> str
+        file_path: Path to the file to parse
+
+    Returns:
+        A callable parse_file(file_path) -> list[dict]
+
+    Raises:
+        ValueError: If generated code is unsafe, has invalid syntax,
+                   or doesn't return list of dicts
+    """
+    path = Path(file_path)
+    with open(path, "r", errors="replace") as f:
+        sample = f.read(4096)
+
+    prompt = PARSER_PROMPT.format(sample=sample)
+    response = llm_backend.generate(prompt)
+    code = extract_python_block(response)
+
+    # Validate syntax
+    try:
+        ast.parse(code)
+    except SyntaxError as e:
+        raise ValueError(f"Generated parser has invalid syntax: {e}")
+
+    # Security check
+    issues = check_parser_safety(code)
+    if issues:
+        raise ValueError(f"Generated parser contains dangerous code: {issues}")
+
+    # Execute to get function
+    namespace: dict = {}
+    exec(code, namespace)
+
+    if "parse_file" not in namespace:
+        raise ValueError("Generated code must define 'parse_file' function")
+
+    parser = namespace["parse_file"]
+
+    # Validate on actual file
+    result = parser(file_path)
+    if not isinstance(result, list):
+        raise ValueError(f"Parser must return list, got {type(result).__name__}")
+    if result and not isinstance(result[0], dict):
+        raise ValueError(
+            f"Parser must return list of dicts, got list of {type(result[0]).__name__}"
+        )
+
+    return parser