recursive-cleaner 0.6.0__py3-none-any.whl

recursive_cleaner/schema.py

@@ -0,0 +1,117 @@
+"""Schema inference for data files."""
+
+import csv
+import json
+from io import StringIO
+from pathlib import Path
+
+
+def infer_schema(file_path: str, sample_size: int = 10) -> dict:
+    """
+    Infer data schema from first N records.
+
+    Args:
+        file_path: Path to data file
+        sample_size: Number of records to sample
+
+    Returns:
+        {"fields": [...], "types": {...}, "samples": {...}, "nullable": {...}}
+    """
+    path = Path(file_path)
+    suffix = path.suffix.lower()
+
+    if not path.exists():
+        return {"fields": [], "types": {}, "samples": {}, "nullable": {}}
+
+    content = path.read_text(encoding="utf-8")
+    if not content.strip():
+        return {"fields": [], "types": {}, "samples": {}, "nullable": {}}
+
+    if suffix == ".jsonl":
+        return _infer_jsonl(content, sample_size)
+    elif suffix == ".csv":
+        return _infer_csv(content, sample_size)
+    elif suffix == ".json":
+        return _infer_json(content, sample_size)
+    else:
+        return {"fields": [], "types": {}, "samples": {}, "nullable": {}}
+
+
+def _infer_jsonl(content: str, sample_size: int) -> dict:
+    """Infer schema from JSONL content."""
+    lines = [line.strip() for line in content.strip().split("\n") if line.strip()]
+    records = [json.loads(line) for line in lines[:sample_size]]
+    return _infer_from_records(records)
+
+
+def _infer_csv(content: str, sample_size: int) -> dict:
+    """Infer schema from CSV content."""
+    reader = csv.DictReader(StringIO(content))
+    records = [row for _, row in zip(range(sample_size), reader)]
+    return _infer_from_records(records)
+
+
+def _infer_json(content: str, sample_size: int) -> dict:
+    """Infer schema from JSON content."""
+    data = json.loads(content)
+    if isinstance(data, list):
+        records = data[:sample_size]
+    elif isinstance(data, dict):
+        records = [data]
+    else:
+        return {"fields": [], "types": {}, "samples": {}, "nullable": {}}
+    return _infer_from_records(records)
+
+
+def _infer_from_records(records: list[dict]) -> dict:
+    """Build schema from list of record dicts."""
+    if not records:
+        return {"fields": [], "types": {}, "samples": {}, "nullable": {}}
+
+    fields = list(dict.fromkeys(k for r in records for k in r.keys()))
+    types = {}
+    samples = {}
+    nullable = {}
+
+    for field in fields:
+        values = [r.get(field) for r in records if field in r]
+        nullable[field] = any(v is None for v in values)
+        non_null = [v for v in values if v is not None]
+        samples[field] = non_null[:3]
+        types[field] = _infer_type(non_null)
+
+    return {"fields": fields, "types": types, "samples": samples, "nullable": nullable}
+
+
+def _infer_type(values: list) -> str:
+    """Infer type from list of non-null values."""
+    if not values:
+        return "unknown"
+    type_map = {str: "str", int: "int", float: "float", bool: "bool", list: "list", dict: "dict"}
+    seen = set()
+    for v in values:
+        for py_type, name in type_map.items():
+            if type(v) is py_type:
+                seen.add(name)
+                break
+        else:
+            seen.add("unknown")
+    if len(seen) == 1:
+        return seen.pop()
+    return "mixed"
+
+
+def format_schema_for_prompt(schema: dict) -> str:
+    """Format schema dict as human-readable string for prompt injection."""
+    if not schema.get("fields"):
+        return ""
+    lines = ["Fields detected:"]
+    for field in schema["fields"]:
+        ftype = schema["types"].get(field, "unknown")
+        is_nullable = schema["nullable"].get(field, False)
+        samples = schema["samples"].get(field, [])
+        type_str = f"{ftype}, nullable" if is_nullable else ftype
+        sample_strs = [repr(s) if isinstance(s, str) else str(s) for s in samples]
+        sample_part = ", ".join(sample_strs) if sample_strs else "no samples"
+        lines.append(f"- {field} ({type_str}): {sample_part}")
+    return "\n".join(lines)

recursive_cleaner/types.py

@@ -0,0 +1,11 @@
+"""Type definitions for the recursive cleaner pipeline."""
+
+from typing import Protocol
+
+
+class LLMBackend(Protocol):
+    """Protocol for LLM backend implementations."""
+
+    def generate(self, prompt: str) -> str:
+        """Generate a response from the LLM given a prompt."""
+        ...

recursive_cleaner/validation.py

@@ -0,0 +1,202 @@
+"""Runtime validation for generated cleaning functions."""
+
+import ast
+import json
+import re
+from typing import Literal
+
+# Modules that could access filesystem, run commands, or exfiltrate data
+DANGEROUS_IMPORTS = frozenset({
+    "os",
+    "subprocess",
+    "sys",
+    "shutil",
+    "pathlib",
+    "socket",
+    "urllib",
+    "requests",
+    "httplib",
+    "ftplib",
+    "smtplib",
+    "pickle",
+})
+
+# Built-in functions that allow arbitrary code execution
+DANGEROUS_CALLS = frozenset({
+    "eval",
+    "exec",
+    "compile",
+    "__import__",
+    "open",  # Data cleaning functions receive data as args, shouldn't need file I/O
+})
+
+
+def check_code_safety(code: str) -> tuple[bool, str | None]:
+    """
+    Check if generated code contains dangerous patterns.
+
+    Catches common LLM mistakes like importing os or using eval().
+    Not a security sandbox - won't catch obfuscated/adversarial code.
+
+    Args:
+        code: Python source code to check
+
+    Returns:
+        (True, None) if code appears safe
+        (False, error_message) if dangerous pattern detected
+    """
+    try:
+        tree = ast.parse(code)
+    except SyntaxError as e:
+        return False, f"Syntax error: {e}"
+
+    for node in ast.walk(tree):
+        # Check: import os, import subprocess, etc.
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                module = alias.name.split(".")[0]
+                if module in DANGEROUS_IMPORTS:
+                    return False, f"Dangerous import: {alias.name}"
+
+        # Check: from os import path, from subprocess import run, etc.
+        if isinstance(node, ast.ImportFrom):
+            if node.module:
+                module = node.module.split(".")[0]
+                if module in DANGEROUS_IMPORTS:
+                    return False, f"Dangerous import: from {node.module}"
+
+        # Check: eval(...), exec(...), open(...), etc.
+        if isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name):
+                if node.func.id in DANGEROUS_CALLS:
+                    return False, f"Dangerous function call: {node.func.id}()"
+
+    return True, None
+
+
+def split_holdout(
+    chunk: str, holdout_ratio: float, mode: Literal["structured", "text"] = "structured"
+) -> tuple[str, str]:
+    """
+    Split chunk into generation and holdout portions.
+
+    Args:
+        chunk: Raw chunk string (JSONL for structured, plain text for text mode)
+        holdout_ratio: Fraction to hold out (0.0-0.5)
+        mode: "structured" splits by JSONL records, "text" splits at sentence boundary
+
+    Returns:
+        (generation_data, holdout_data) - both as strings
+    """
+    if holdout_ratio <= 0:
+        return chunk, ""
+
+    if mode == "structured":
+        lines = [ln for ln in chunk.strip().split("\n") if ln.strip()]
+        if not lines:
+            return chunk, ""
+        holdout_count = max(1, int(len(lines) * holdout_ratio))
+        split_idx = len(lines) - holdout_count
+        return "\n".join(lines[:split_idx]), "\n".join(lines[split_idx:])
+    else:
+        # Text mode: split at sentence boundary
+        sentences = re.split(r"(?<=[.!?])\s+", chunk.strip())
+        if len(sentences) <= 1:
+            return chunk, ""
+        holdout_count = max(1, int(len(sentences) * holdout_ratio))
+        split_idx = len(sentences) - holdout_count
+        return " ".join(sentences[:split_idx]), " ".join(sentences[split_idx:])
+
+
+def validate_function(
+    code: str,
+    sample_data: list[dict] | str,
+    function_name: str,
+    mode: Literal["structured", "text"] = "structured",
+) -> tuple[bool, str | None]:
+    """
+    Execute generated function on sample data to catch runtime errors.
+
+    Args:
+        code: The Python source code of the function
+        sample_data: List of data records (structured) or text string (text mode)
+        function_name: Name of the function to call
+        mode: "structured" for dict records, "text" for string input
+
+    Returns:
+        (True, None) if function executes without error
+        (False, error_message) if function raises an exception
+    """
+    # Handle empty data
+    if mode == "text":
+        if not sample_data or (isinstance(sample_data, str) and not sample_data.strip()):
+            return True, None
+    else:
+        if not sample_data:
+            return True, None
+
+    # Create isolated namespace and execute the code
+    namespace: dict = {}
+    try:
+        exec(code, namespace)
+    except Exception as e:
+        return False, f"Code compilation failed: {type(e).__name__}: {e}"
+
+    # Get the function from namespace
+    func = namespace.get(function_name)
+    if func is None:
+        return False, f"Function '{function_name}' not found in code"
+
+    if mode == "text":
+        # Text mode: sample_data is a string
+        try:
+            result = func(sample_data)
+            # Verify function returns a string
+            if not isinstance(result, str):
+                return False, f"Function must return str, got {type(result).__name__}"
+        except Exception as e:
+            return False, f"Runtime error on text input: {type(e).__name__}: {e}"
+    else:
+        # Structured mode: sample_data is list[dict]
+        for i, record in enumerate(sample_data):
+            try:
+                func(record)
+            except Exception as e:
+                return False, f"Runtime error on sample {i}: {type(e).__name__}: {e}"
+
+    return True, None
+
+
+def extract_sample_data(
+    chunk: str, max_samples: int = 3, mode: Literal["structured", "text"] = "structured"
+) -> list[dict] | str:
+    """
+    Extract sample data from a chunk string.
+
+    Args:
+        chunk: Raw chunk string
+        max_samples: Maximum number of samples to extract (structured mode only)
+        mode: "structured" for JSONL parsing, "text" for raw string
+
+    Returns:
+        List of parsed JSON objects (structured) or the chunk string (text)
+    """
+    if mode == "text":
+        # Text mode: return the chunk as-is for validation
+        return chunk
+
+    # Structured mode: parse JSONL
+    samples = []
+    for line in chunk.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+            if isinstance(obj, dict):
+                samples.append(obj)
+                if len(samples) >= max_samples:
+                    break
+        except json.JSONDecodeError:
+            continue
+    return samples

recursive_cleaner/vendor/__init__.py

@@ -0,0 +1,4 @@
+"""Vendored third-party code."""
+from .chunker import Chunk, SentenceChunker
+
+__all__ = ["Chunk", "SentenceChunker"]

recursive_cleaner/vendor/chunker.py

@@ -0,0 +1,187 @@
+"""Minimal sentence-aware text chunker extracted from chonkie.
+
+Sentence chunking algorithm adapted from Chonkie
+https://github.com/chonkie-inc/chonkie
+Copyright (c) 2025 Chonkie
+Licensed under the MIT License
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from bisect import bisect_left
+from dataclasses import dataclass
+from itertools import accumulate
+from typing import Literal, Optional, Union
+
+
+@dataclass
+class Chunk:
+    """A chunk of text with position metadata."""
+
+    text: str
+    start_index: int
+    end_index: int
+    token_count: int  # In our case, character count
+
+    def __len__(self) -> int:
+        return len(self.text)
+
+
+class SentenceChunker:
+    """Split text into chunks based on sentence boundaries.
+
+    Args:
+        chunk_size: Maximum characters per chunk
+        chunk_overlap: Characters to overlap between chunks
+        min_sentences_per_chunk: Minimum sentences per chunk
+        min_characters_per_sentence: Minimum characters for valid sentence
+        delim: Sentence boundary delimiters
+        include_delim: Where to include delimiter ("prev", "next", or None)
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 4000,
+        chunk_overlap: int = 200,
+        min_sentences_per_chunk: int = 1,
+        min_characters_per_sentence: int = 12,
+        delim: Union[str, list[str]] = [". ", "! ", "? ", "\n"],
+        include_delim: Optional[Literal["prev", "next"]] = "prev",
+    ):
+        if chunk_size <= 0:
+            raise ValueError("chunk_size must be positive")
+        if chunk_overlap >= chunk_size:
+            raise ValueError("chunk_overlap must be less than chunk_size")
+        if min_sentences_per_chunk < 1:
+            raise ValueError("min_sentences_per_chunk must be at least 1")
+
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.min_sentences_per_chunk = min_sentences_per_chunk
+        self.min_characters_per_sentence = min_characters_per_sentence
+        self.delim = [delim] if isinstance(delim, str) else delim
+        self.include_delim = include_delim
+        self._sep = "\x00"  # Internal separator (null char)
+
+    def _split_into_sentences(self, text: str) -> list[str]:
+        """Split text into sentences based on delimiters."""
+        t = text
+        for d in self.delim:
+            if self.include_delim == "prev":
+                t = t.replace(d, d + self._sep)
+            elif self.include_delim == "next":
+                t = t.replace(d, self._sep + d)
+            else:
+                t = t.replace(d, self._sep)
+
+        splits = [s for s in t.split(self._sep) if s]
+
+        # Merge short splits with previous sentence
+        sentences = []
+        current = ""
+        for s in splits:
+            if len(s) < self.min_characters_per_sentence:
+                current += s
+            elif current:
+                current += s
+                sentences.append(current)
+                current = ""
+            else:
+                sentences.append(s)
+
+            if len(current) >= self.min_characters_per_sentence:
+                sentences.append(current)
+                current = ""
+
+        if current:
+            sentences.append(current)
+
+        return sentences
+
+    def chunk(self, text: str) -> list[Chunk]:
+        """Split text into overlapping chunks based on sentences."""
+        if not text.strip():
+            return []
+
+        # Split into sentences with positions and character counts
+        sentence_texts = self._split_into_sentences(text)
+        if not sentence_texts:
+            return []
+
+        # Calculate positions
+        positions = []
+        current_pos = 0
+        for sent in sentence_texts:
+            positions.append(current_pos)
+            current_pos += len(sent)
+
+        # Character counts (our "tokens")
+        char_counts = [len(s) for s in sentence_texts]
+
+        # Cumulative character counts for bisect
+        char_sums = list(accumulate([0] + char_counts))
+
+        chunks = []
+        pos = 0
+
+        while pos < len(sentence_texts):
+            # Find split point using bisect
+            target = char_sums[pos] + self.chunk_size
+            split_idx = bisect_left(char_sums, target) - 1
+            split_idx = max(split_idx, pos + 1)  # At least one sentence
+            split_idx = min(split_idx, len(sentence_texts))
+
+            # Handle minimum sentences requirement
+            if split_idx - pos < self.min_sentences_per_chunk:
+                if pos + self.min_sentences_per_chunk <= len(sentence_texts):
+                    split_idx = pos + self.min_sentences_per_chunk
+                else:
+                    split_idx = len(sentence_texts)
+
+            # Create chunk
+            chunk_sentences = sentence_texts[pos:split_idx]
+            chunk_text = "".join(chunk_sentences)
+            chunks.append(
+                Chunk(
+                    text=chunk_text,
+                    start_index=positions[pos],
+                    end_index=positions[pos] + len(chunk_text),
+                    token_count=len(chunk_text),
+                )
+            )
+
+            # Calculate next position with overlap
+            if self.chunk_overlap > 0 and split_idx < len(sentence_texts):
+                overlap_chars = 0
+                overlap_idx = split_idx - 1
+
+                while overlap_idx > pos and overlap_chars < self.chunk_overlap:
+                    next_chars = overlap_chars + char_counts[overlap_idx]
+                    if next_chars > self.chunk_overlap:
+                        break
+                    overlap_chars = next_chars
+                    overlap_idx -= 1
+
+                pos = overlap_idx + 1
+            else:
+                pos = split_idx
+
+        return chunks