recursive-cleaner 0.6.0__tar.gz → 0.7.0__tar.gz

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 0.6.0
+Version: 0.7.0
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Issues, https://github.com/gaztrabisme/recursive-data-cleaner/issues
-Author: Gary Tou
+Author: Gary Tran
 License-Expression: MIT
 License-File: LICENSE
 Keywords: automation,data-cleaning,data-quality,etl,llm,machine-learning
@@ -26,8 +26,12 @@ Requires-Dist: tenacity>=8.0
 Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: markitdown
+Requires-Dist: markitdown>=0.1.0; extra == 'markitdown'
 Provides-Extra: mlx
 Requires-Dist: mlx-lm>=0.10.0; extra == 'mlx'
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=14.0.0; extra == 'parquet'
 Description-Content-Type: text/markdown
 
 # Recursive Data Cleaner
@@ -277,6 +281,11 @@ pytest tests/ -v
 | v0.2.0 | Runtime validation, schema inference, callbacks, incremental saves |
 | v0.1.0 | Core pipeline, chunking, docstring registry |
 
+## Acknowledgments
+
+- Sentence-aware text chunking adapted from [Chonkie](https://github.com/chonkie-inc/chonkie) (MIT License)
+- Development assisted by [Claude Code](https://claude.ai/claude-code)
+
 ## License
 
 MIT

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/README.md

@@ -245,6 +245,11 @@ pytest tests/ -v
 | v0.2.0 | Runtime validation, schema inference, callbacks, incremental saves |
 | v0.1.0 | Core pipeline, chunking, docstring registry |
 
+## Acknowledgments
+
+- Sentence-aware text chunking adapted from [Chonkie](https://github.com/chonkie-inc/chonkie) (MIT License)
+- Development assisted by [Claude Code](https://claude.ai/claude-code)
+
 ## License
 
 MIT

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/TODO.md

@@ -60,30 +60,42 @@ These patterns proved high-value with low implementation effort:
 
 ---
 
-## Future Considerations
+## Tier 5: Format Expansion & UI (v0.7.0) - PLANNED
+
+### Markitdown Integration
+- [ ] Add markitdown as optional dependency
+- [ ] Auto-convert 20+ formats: Excel, HTML, Word, PDF, PowerPoint, EPUB, etc.
+- [ ] Preprocessing step before chunking
+- **Approach**: `pip install recursive-cleaner[markitdown]`
+
+### Parquet Support
+- [ ] Native parser using pyarrow
+- [ ] Read as list of dicts (same as JSONL)
+- **Approach**: Optional dependency, ~10 lines of code
+
+### LLM-Generated Parsers
+- [ ] For XML and unknown formats
+- [ ] Send sample to LLM: "Generate a function to parse this into list of records"
+- [ ] Validate generated parser on sample before using
+- **Approach**: Wu wei - let LLM decide how to parse data it understands
+
+### Terminal UI (Textual)
+- [ ] Optional `[ui]` extra dependency
+- [ ] Live dashboard showing: chunk progress, function generation, latency sparkline
+- [ ] Pure terminal, no browser needed
+- **Approach**: `pip install recursive-cleaner[ui]`
 
-Ideas that might be valuable but need more thought.
+---
 
-### Confidence Scoring
-- LLM rates confidence in each generated function (high/medium/low)
-- Low confidence = flag for human review
-- **Question**: Does this actually help users, or just add noise?
+## Future Considerations
 
-### Before/After Examples
-- User provides expected input→output pairs
-- Validate generated functions match expectations
-- **Question**: How to handle functions that transform data differently but correctly?
+Ideas that might be valuable but need more thought.
 
 ### Multi-File Batch Mode
 - Process multiple files with shared function registry
 - Functions learned from file A applied to file B
 - **Question**: How to handle schema differences between files?
 
-### Summary Buffer Memory
-- Compress old function docstrings into summaries
-- Keep recent functions verbatim
-- **Question**: Does FIFO eviction already work well enough?
-
 ---
 
 ## Explicitly Deferred

recursive_cleaner-0.7.0/docs/contracts/v070-success-criteria.md

@@ -0,0 +1,13 @@
+# Success Criteria - v0.7.0 Format Expansion
+
+## Project-Level Success
+- [ ] Markitdown integration converts 20+ formats to text before chunking
+- [ ] Parquet files load as list of dicts like JSONL/CSV
+- [ ] LLM-generated parsers handle XML and unknown formats
+- [ ] All new formats integrate seamlessly with existing cleaning pipeline
+- [ ] Optional dependencies don't break core functionality when not installed
+- [ ] All 392 existing tests still pass
+
+## Phase Success Criteria
+
+[To be populated during planning]

recursive_cleaner-0.7.0/docs/workflow-state.md

@@ -0,0 +1,26 @@
+# Workflow State - v0.7.0 Format Expansion
+
+## Current Phase
+Research
+
+## Awaiting
+Subagent Completion (Research)
+
+## Blockers
+None
+
+## Progress
+- [ ] Research complete
+- [ ] Contracts approved
+- [ ] Plan approved
+- [ ] Phase 1: Markitdown integration
+- [ ] Phase 1 audit
+- [ ] Phase 2: Parquet support
+- [ ] Phase 2 audit
+- [ ] Phase 3: LLM-generated parsers
+- [ ] Phase 3 audit
+
+## Previous Version (v0.6.0)
+- **Tests**: 392 passing
+- **Lines**: 2,967 total
+- **Status**: Released on GitHub + PyPI

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
 
 [project]
 name = "recursive-cleaner"
-version = "0.6.0"
+version = "0.7.0"
 description = "LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions"
 readme = "README.md"
 license = "MIT"
 requires-python = ">=3.10"
 authors = [
-    { name = "Gary Tou" },
+    { name = "Gary Tran" },
 ]
 keywords = [
     "data-cleaning",
@@ -46,6 +46,12 @@ dev = [
 mlx = [
     "mlx-lm>=0.10.0",
 ]
+markitdown = [
+    "markitdown>=0.1.0",
+]
+parquet = [
+    "pyarrow>=14.0.0",
+]
 
 [project.urls]
 Homepage = "https://github.com/gaztrabisme/recursive-data-cleaner"

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/recursive_cleaner/__init__.py

@@ -16,9 +16,10 @@ from recursive_cleaner.optimizer import (
     group_by_salience,
 )
 from recursive_cleaner.output import write_cleaning_file
-from recursive_cleaner.parsers import chunk_file
+from recursive_cleaner.parsers import MARKITDOWN_EXTENSIONS, chunk_file, load_parquet, preprocess_with_markitdown
 from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
+from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
 
 __all__ = [
@@ -27,6 +28,9 @@ __all__ = [
     "MaxIterationsError",
     "OutputValidationError",
     "chunk_file",
+    "MARKITDOWN_EXTENSIONS",
+    "load_parquet",
+    "preprocess_with_markitdown",
     "parse_response",
     "extract_python_block",
     "build_context",
@@ -43,4 +47,6 @@ __all__ = [
     "extract_tags",
     "group_by_salience",
     "consolidate_with_agency",
+    "generate_parser",
+    "check_parser_safety",
 ]

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/recursive_cleaner/cleaner.py

@@ -12,7 +12,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential
 from .context import build_context
 from .errors import OutputValidationError, ParseError
 from .metrics import QualityMetrics, compare_quality, load_structured_data, measure_quality
-from .parsers import chunk_file
+from .parsers import MARKITDOWN_EXTENSIONS, chunk_file
 from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
@@ -61,6 +61,7 @@ class DataCleaner:
         saturation_check_interval: int = 20,
         report_path: str | None = "cleaning_report.md",
         dry_run: bool = False,
+        auto_parse: bool = False,
     ):
         self.backend = llm_backend
         self.file_path = file_path
@@ -84,7 +85,9 @@ class DataCleaner:
         self.saturation_check_interval = saturation_check_interval
         self.report_path = report_path
         self.dry_run = dry_run
+        self.auto_parse = auto_parse
         self.functions: list[dict] = []  # List of {name, docstring, code}
+        self._generated_parser: callable | None = None  # LLM-generated parser for unknown formats
         # Track recent function generation for saturation check
         self._recent_new_function_count = 0
         self._last_check_function_count = 0
@@ -319,27 +322,72 @@ class DataCleaner:
     def _detect_mode(self) -> Literal["structured", "text"]:
         """Detect mode from file extension."""
         suffix = Path(self.file_path).suffix.lower()
-        structured_extensions = {".jsonl", ".csv", ".json"}
+        # Markitdown formats are processed as text
+        if suffix in MARKITDOWN_EXTENSIONS:
+            return "text"
+        structured_extensions = {".jsonl", ".csv", ".json", ".parquet"}
         if suffix in structured_extensions:
             return "structured"
         return "text"
 
+    def _is_known_extension(self) -> bool:
+        """Check if file extension is natively supported."""
+        suffix = Path(self.file_path).suffix.lower()
+        known = {".jsonl", ".csv", ".json", ".parquet", ".txt"}
+        return suffix in known or suffix in MARKITDOWN_EXTENSIONS
+
+    def _load_with_auto_parser(self) -> list[str]:
+        """Load file using LLM-generated parser, return JSONL chunks."""
+        from .parser_generator import generate_parser
+
+        print(f"Unknown file format, generating parser...")
+        self._emit("parser_generation_start")
+
+        parser = generate_parser(self.backend, self.file_path)
+        self._generated_parser = parser
+
+        self._emit("parser_generation_complete")
+        print("Parser generated successfully.")
+
+        # Parse the file
+        records = parser(self.file_path)
+        if not records:
+            return []
+
+        # Convert to JSONL chunks
+        import json
+        chunks = []
+        for i in range(0, len(records), self.chunk_size):
+            chunk_records = records[i:i + self.chunk_size]
+            chunk_lines = [json.dumps(r) for r in chunk_records]
+            chunks.append("\n".join(chunk_lines))
+
+        return chunks
+
     def run(self) -> None:
         """Run the cleaning pipeline."""
-        # Resolve effective mode
-        if self.mode == "auto":
-            self._effective_mode = self._detect_mode()
+        # Check if we should use auto-parser for unknown formats
+        use_auto_parser = self.auto_parse and not self._is_known_extension()
+
+        if use_auto_parser:
+            # LLM generates parser, always structured mode
+            self._effective_mode = "structured"
+            chunks = self._load_with_auto_parser()
         else:
-            self._effective_mode = self.mode
+            # Resolve effective mode
+            if self.mode == "auto":
+                self._effective_mode = self._detect_mode()
+            else:
+                self._effective_mode = self.mode
 
-        chunks = chunk_file(
-            self.file_path,
-            self.chunk_size,
-            mode=self._effective_mode,
-            chunk_overlap=self.chunk_overlap,
-            sampling_strategy=self.sampling_strategy,
-            stratify_field=self.stratify_field,
-        )
+            chunks = chunk_file(
+                self.file_path,
+                self.chunk_size,
+                mode=self._effective_mode,
+                chunk_overlap=self.chunk_overlap,
+                sampling_strategy=self.sampling_strategy,
+                stratify_field=self.stratify_field,
+            )
 
         if not chunks:
             print("No data to process.")

recursive_cleaner-0.7.0/recursive_cleaner/parser_generator.py

@@ -0,0 +1,123 @@
+"""LLM-generated parser for unknown file formats."""
+
+import ast
+import re
+from pathlib import Path
+
+from .types import LLMBackend
+
+# Dangerous patterns for parser code (allows 'open' since parsers need file I/O)
+_DANGEROUS_IMPORTS = frozenset({
+    "os", "subprocess", "sys", "shutil", "socket", "urllib",
+    "requests", "httplib", "ftplib", "smtplib", "pickle",
+})
+_DANGEROUS_CALLS = frozenset({"eval", "exec", "compile", "__import__"})
+
+PARSER_PROMPT = '''You are a data parsing expert. Generate a Python function to parse this file format.
+
+=== SAMPLE (first 4KB) ===
+{sample}
+
+=== TASK ===
+Generate a function with this EXACT signature:
+
+```python
+def parse_file(file_path: str) -> list[dict]:
+    """Parse the file into a list of records."""
+    # Your implementation
+```
+
+RULES:
+- Return list of dicts, one dict per logical record
+- Use only stdlib (xml.etree, json, re, csv)
+- Handle the ENTIRE file, not just this sample
+- Be defensive about malformed data
+- Include necessary imports inside or before the function
+'''
+
+
+def check_parser_safety(code: str) -> list[str]:
+    """Check parser code for dangerous patterns. Returns list of issues."""
+    issues = []
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        return [f"Syntax error: {e}"]
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                module = alias.name.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: {alias.name}")
+        if isinstance(node, ast.ImportFrom):
+            if node.module:
+                module = node.module.split(".")[0]
+                if module in _DANGEROUS_IMPORTS:
+                    issues.append(f"Dangerous import: from {node.module}")
+        if isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name):
+                if node.func.id in _DANGEROUS_CALLS:
+                    issues.append(f"Dangerous call: {node.func.id}()")
+    return issues
+
+
+def extract_python_block(text: str) -> str:
+    """Extract code from ```python ... ``` block."""
+    match = re.search(r"```python\s*(.*?)\s*```", text, re.DOTALL)
+    return match.group(1).strip() if match else text.strip()
+
+
+def generate_parser(llm_backend: LLMBackend, file_path: str) -> callable:
+    """
+    Generate a parser function for an unknown file format.
+
+    Args:
+        llm_backend: LLM backend implementing generate(prompt) -> str
+        file_path: Path to the file to parse
+
+    Returns:
+        A callable parse_file(file_path) -> list[dict]
+
+    Raises:
+        ValueError: If generated code is unsafe, has invalid syntax,
+                   or doesn't return list of dicts
+    """
+    path = Path(file_path)
+    with open(path, "r", errors="replace") as f:
+        sample = f.read(4096)
+
+    prompt = PARSER_PROMPT.format(sample=sample)
+    response = llm_backend.generate(prompt)
+    code = extract_python_block(response)
+
+    # Validate syntax
+    try:
+        ast.parse(code)
+    except SyntaxError as e:
+        raise ValueError(f"Generated parser has invalid syntax: {e}")
+
+    # Security check
+    issues = check_parser_safety(code)
+    if issues:
+        raise ValueError(f"Generated parser contains dangerous code: {issues}")
+
+    # Execute to get function
+    namespace: dict = {}
+    exec(code, namespace)
+
+    if "parse_file" not in namespace:
+        raise ValueError("Generated code must define 'parse_file' function")
+
+    parser = namespace["parse_file"]
+
+    # Validate on actual file
+    result = parser(file_path)
+    if not isinstance(result, list):
+        raise ValueError(f"Parser must return list, got {type(result).__name__}")
+    if result and not isinstance(result[0], dict):
+        raise ValueError(
+            f"Parser must return list of dicts, got list of {type(result[0]).__name__}"
+        )
+
+    return parser

{recursive_cleaner-0.6.0 → recursive_cleaner-0.7.0}/recursive_cleaner/parsers.py

@@ -17,6 +17,62 @@ try:
 except ImportError:
     _HAS_SENTENCE_CHUNKER = False
 
+# File extensions supported by markitdown for conversion to text
+MARKITDOWN_EXTENSIONS = {
+    ".pdf", ".docx", ".doc", ".xlsx", ".xls", ".pptx", ".ppt",
+    ".html", ".htm", ".epub", ".msg", ".rtf", ".odt", ".ods", ".odp"
+}
+
+
+def load_parquet(file_path: str) -> list[dict]:
+    """Load parquet file as list of dicts.
+
+    Args:
+        file_path: Path to the parquet file
+
+    Returns:
+        List of dictionaries, one per row
+
+    Raises:
+        ImportError: If pyarrow is not installed
+    """
+    try:
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for parquet files. "
+            "Install with: pip install recursive-cleaner[parquet]"
+        )
+
+    table = pq.read_table(file_path)
+    return table.to_pylist()
+
+
+def preprocess_with_markitdown(file_path: str) -> str:
+    """
+    Convert supported formats to text using markitdown.
+
+    Args:
+        file_path: Path to the file to convert
+
+    Returns:
+        Extracted text content from the file
+
+    Raises:
+        ImportError: If markitdown is not installed
+    """
+    try:
+        from markitdown import MarkItDown
+    except ImportError:
+        raise ImportError(
+            "markitdown is required for this file type. "
+            "Install with: pip install recursive-cleaner[markitdown]"
+        )
+
+    md = MarkItDown()
+    result = md.convert(file_path)
+    return result.text_content
+
 
 def chunk_file(
     file_path: str,
@@ -50,6 +106,25 @@ def chunk_file(
     if not path.exists():
         raise FileNotFoundError(f"File not found: {file_path}")
 
+    # Handle markitdown formats: preprocess to text, then chunk as text
+    if suffix in MARKITDOWN_EXTENSIONS:
+        content = preprocess_with_markitdown(file_path)
+        if not content.strip():
+            return []
+        # Markitdown output is always processed as text
+        if sampling_strategy != "sequential":
+            raise ValueError(
+                f"Text mode only supports 'sequential' sampling, got '{sampling_strategy}'"
+            )
+        return chunk_text_sentences(content, chunk_size, chunk_overlap)
+
+    # Handle parquet files: load as list of dicts, chunk like JSONL
+    if suffix == ".parquet":
+        records = load_parquet(file_path)
+        if not records:
+            return []
+        return _chunk_records(records, chunk_size, sampling_strategy, stratify_field)
+
     content = path.read_text(encoding="utf-8")
 
     if not content.strip():
@@ -79,7 +154,7 @@ def chunk_file(
 
 def _detect_mode(suffix: str) -> Literal["structured", "text"]:
     """Detect mode from file extension."""
-    structured_extensions = {".jsonl", ".csv", ".json"}
+    structured_extensions = {".jsonl", ".csv", ".json", ".parquet"}
     if suffix in structured_extensions:
         return "structured"
     return "text"
@@ -281,6 +356,61 @@ def _chunk_jsonl(
     return chunks
 
 
+def _chunk_records(
+    records: list[dict],
+    item_count: int,
+    sampling_strategy: Literal["sequential", "random", "stratified"] = "sequential",
+    stratify_field: str | None = None,
+) -> list[str]:
+    """Chunk a list of dicts by item count with optional sampling."""
+    if not records:
+        return []
+
+    # For seed computation, use JSON representation
+    seed = _compute_seed(json.dumps(records[0]))
+
+    # Apply sampling strategy
+    if sampling_strategy == "random":
+        records = _shuffle_records(records, seed)
+    elif sampling_strategy == "stratified" and stratify_field:
+        records = _stratified_sample_dicts(records, stratify_field, seed)
+
+    chunks = []
+    for i in range(0, len(records), item_count):
+        chunk_records = records[i:i + item_count]
+        # Convert to JSONL format for LLM context
+        chunk_lines = [json.dumps(r) for r in chunk_records]
+        chunks.append("\n".join(chunk_lines))
+
+    return chunks
+
+
+def _stratified_sample_dicts(records: list[dict], field: str, seed: int) -> list[dict]:
+    """Group dicts by field, interleave proportionally."""
+    groups: dict[str, list] = {}
+    for record in records:
+        key = str(record.get(field, "_missing_"))
+        if key not in groups:
+            groups[key] = []
+        groups[key].append(record)
+
+    # Shuffle within each group
+    rng = random.Random(seed)
+    for key in groups:
+        rng.shuffle(groups[key])
+
+    # Interleave from groups (round-robin)
+    result = []
+    group_lists = list(groups.values())
+    while any(group_lists):
+        for g in group_lists:
+            if g:
+                result.append(g.pop(0))
+        group_lists = [g for g in group_lists if g]
+
+    return result
+
+
 def _compute_seed(content: str) -> int:
     """Compute deterministic seed from content hash."""
     return int(hashlib.md5(content.encode("utf-8")).hexdigest()[:8], 16)