recursive-cleaner 1.0.0__tar.gz → 1.0.1__tar.gz

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/CLAUDE.md

@@ -4,7 +4,8 @@
 
 | Version | Status | Date |
 |---------|--------|------|
-| v1.0.0 | **Implemented** | 2025-01-30 |
+| v1.0.1 | **Implemented** | 2025-02-05 |
+| v1.0.0 | Implemented | 2025-01-30 |
 | v0.9.0 | Implemented | 2025-01-19 |
 | v0.8.0 | Implemented | 2025-01-19 |
 | v0.7.0 | Implemented | 2025-01-17 |
@@ -16,9 +17,10 @@
 | v0.2.0 | Implemented | 2025-01-14 |
 | v0.1.0 | Implemented | 2025-01-14 |
 
-**Current State**: v1.0.0 complete. 548 tests passing.
+**Current State**: v1.0.1 complete. 555 tests passing.
 
 ### Version History
+- **v1.0.1**: Return type validation, prompt signature clarity, duplicate field detection
 - **v1.0.0**: Apply mode for applying cleaning functions to data, Excel support, TUI color enhancement
 - **v0.9.0**: CLI tool with MLX and OpenAI-compatible backends (LM Studio, Ollama)
 - **v0.8.0**: Terminal UI with Rich dashboard, mission control aesthetic, transmission log

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 1.0.0
+Version: 1.0.1
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
@@ -389,7 +389,7 @@ backends/
 pytest tests/ -v
 ```
 
-548 tests covering all features. Test datasets in `test_cases/`:
+555 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/README.md

@@ -346,7 +346,7 @@ backends/
 pytest tests/ -v
 ```
 
-548 tests covering all features. Test datasets in `test_cases/`:
+555 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "recursive-cleaner"
-version = "1.0.0"
+version = "1.0.1"
 description = "LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions"
 readme = "README.md"
 license = "MIT"

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/recursive_cleaner/__init__.py

@@ -22,7 +22,7 @@ from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
 from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.tui import HAS_RICH, TUIRenderer
-from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
+from recursive_cleaner.validation import check_code_safety, extract_modified_fields, extract_sample_data, validate_function
 
 __all__ = [
     "apply_cleaning",
@@ -43,6 +43,7 @@ __all__ = [
     "validate_function",
     "extract_sample_data",
     "check_code_safety",
+    "extract_modified_fields",
     "resolve_dependencies",
     "QualityMetrics",
     "measure_quality",

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/recursive_cleaner/cleaner.py

@@ -17,7 +17,7 @@ from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
 from .types import LLMBackend
-from .validation import check_code_safety, extract_sample_data, split_holdout, validate_function
+from .validation import check_code_safety, extract_modified_fields, extract_sample_data, split_holdout, validate_function
 
 STATE_VERSION = "0.5.0"
 
@@ -110,6 +110,8 @@ class DataCleaner:
             "min_ms": float("inf"),
             "max_ms": 0.0,
         }
+        # Track fields already covered by generated functions (per chunk)
+        self._fields_covered: set[str] = set()
 
     def _emit(self, event_type: str, chunk_index: int = 0, **kwargs) -> None:
         """Emit a progress event to the callback, if set."""
@@ -533,6 +535,8 @@ class DataCleaner:
         """Process a single chunk, iterating until clean or max iterations."""
         self._emit("chunk_start", chunk_index=chunk_idx)
         error_feedback = ""
+        # Reset fields covered for new chunk
+        self._fields_covered = set()
 
         # Dry run mode: just detect issues, don't generate functions
         if self.dry_run:
@@ -594,6 +598,20 @@ class DataCleaner:
                         print(f"  Safety check failed: {safety_error}")
                     continue
 
+                # Check for duplicate field coverage
+                new_fields = extract_modified_fields(result["code"])
+                overlap = new_fields & self._fields_covered
+                if overlap:
+                    field_list = ", ".join(sorted(overlap))
+                    error_feedback = f"You already generated a function for field(s): {field_list}. This issue is solved. Move on to the next unsolved issue."
+                    self._emit(
+                        "duplicate_field",
+                        chunk_index=chunk_idx,
+                        function_name=result["name"],
+                        fields=list(overlap),
+                    )
+                    continue
+
                 # Runtime validation if enabled
                 if self.validate_runtime:
                     # Use holdout data if available, else sample from generation chunk
@@ -628,6 +646,8 @@ class DataCleaner:
                     "docstring": result["docstring"],
                     "code": result["code"],
                 })
+                # Track fields covered by this function
+                self._fields_covered.update(new_fields)
                 # Track for saturation check
                 self._recent_new_function_count += 1
 

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/recursive_cleaner/prompt.py

@@ -52,7 +52,8 @@ CONSOLIDATION_TEMPLATE = '''You are reviewing cleaning functions for consolidati
       </docstring>
       <code>
 ```python
-def merged_function_name(record):
+def merged_function_name(record: dict) -> dict:
+    # Modify fields, return record
     ...
 ```
       </code>
@@ -108,9 +109,10 @@ Tags: domain, action, detail
     </docstring>
     <code>
 ```python
-def function_name(data):
-    # Complete implementation
-    pass
+def function_name(record: dict) -> dict:
+    # Modify field(s) in the record
+    record["field"] = cleaned_value
+    return record
 ```
     </code>
   </function_to_generate>
@@ -120,6 +122,8 @@ def function_name(data):
 
 RULES:
 - ONE function per response
+- Function signature: takes a dict (one record), returns the modified dict
+- Modify fields directly on the record, then return it
 - If all issues solved: <chunk_status>clean</chunk_status>, omit <function_to_generate>
 - Include imports inside the function or document needed imports in docstring
 - Function must be idempotent (safe to run multiple times)

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/recursive_cleaner/validation.py

@@ -160,7 +160,10 @@ def validate_function(
         # Structured mode: sample_data is list[dict]
         for i, record in enumerate(sample_data):
             try:
-                func(record)
+                result = func(record)
+                # Verify function returns a dict (not string, int, etc.)
+                if not isinstance(result, dict):
+                    return False, f"Function must return dict, got {type(result).__name__}"
             except Exception as e:
                 return False, f"Runtime error on sample {i}: {type(e).__name__}: {e}"
 
@@ -200,3 +203,39 @@ def extract_sample_data(
         except json.JSONDecodeError:
             continue
     return samples
+
+
+def extract_modified_fields(code: str) -> set[str]:
+    """
+    Extract field names that are modified via record["field"] = ... pattern.
+
+    Args:
+        code: Python source code of the function
+
+    Returns:
+        Set of field names that are assigned to
+    """
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return set()
+
+    fields = set()
+    # Common parameter names for the data/record argument
+    data_names = {"record", "data"}
+
+    for node in ast.walk(tree):
+        # Look for assignment statements
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                # Check if target is a subscript: record["field"] or data["field"]
+                if isinstance(target, ast.Subscript):
+                    # The value should be a Name node (record or data)
+                    if isinstance(target.value, ast.Name):
+                        if target.value.id in data_names:
+                            # The slice should be a string constant
+                            if isinstance(target.slice, ast.Constant):
+                                if isinstance(target.slice.value, str):
+                                    fields.add(target.slice.value)
+
+    return fields

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/tests/test_holdout.py

@@ -246,7 +246,7 @@ def test_cleaner_holdout_validation_uses_holdout_data(tmp_path):
     # Records where only the holdout has a specific key
     test_file.write_text('{"name":"A"}\n{"name":"B"}\n{"name":"C"}\n{"name":"D"}\n{"special":"E"}\n')
 
-    # Function that accesses "special" key - should fail on holdout
+    # Function that accesses "special" key - should pass on holdout (which has it)
     response_with_special_access = '''
 <cleaning_analysis>
   <issues_detected>
@@ -258,7 +258,8 @@ def test_cleaner_holdout_validation_uses_holdout_data(tmp_path):
     <code>
 ```python
 def get_special(data):
-    return data["special"]
+    data["found_special"] = data["special"]
+    return data
 ```
     </code>
   </function_to_generate>

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/tests/test_text_mode.py

@@ -261,7 +261,7 @@ class TestTextPromptTemplate:
         )
 
         # Should have structured-specific content
-        assert "def function_name(data):" in prompt
+        assert "def function_name(record: dict) -> dict:" in prompt
         # Should have schema section
         assert "=== DATA SCHEMA ===" in prompt
 

{recursive_cleaner-1.0.0 → recursive_cleaner-1.0.1}/tests/test_validation.py

@@ -2,6 +2,7 @@
 
 import pytest
 from recursive_cleaner import DataCleaner, validate_function, extract_sample_data, check_code_safety
+from recursive_cleaner.validation import extract_modified_fields
 
 
 class MockLLM:
@@ -22,7 +23,8 @@ def test_validate_function_accepts_working_code():
     """Function that works on sample data is accepted."""
     code = '''
 def process_data(data):
-    return data.get("name", "unknown")
+    data["processed"] = True
+    return data
 '''
     sample_data = [{"name": "Alice"}, {"name": "Bob"}]
     valid, error = validate_function(code, sample_data, "process_data")
@@ -30,6 +32,18 @@ def process_data(data):
     assert error is None
 
 
+def test_validate_function_rejects_wrong_return_type():
+    """Function that returns non-dict is rejected."""
+    code = '''
+def bad_return(data):
+    return data.get("name", "unknown")
+'''
+    sample_data = [{"name": "Alice"}]
+    valid, error = validate_function(code, sample_data, "bad_return")
+    assert valid is False
+    assert "must return dict" in error
+
+
 def test_validate_function_rejects_key_error():
     """Function with KeyError on nonexistent key is rejected."""
     code = '''
@@ -157,7 +171,8 @@ RESPONSE_WITH_GOOD_FUNCTION = '''
     <code>
 ```python
 def good_processor(data):
-    return data.get("name", "unknown")
+    data["processed"] = True
+    return data
 ```
     </code>
   </function_to_generate>
@@ -516,7 +531,8 @@ RESPONSE_SAFE_FUNCTION = '''
     <code>
 ```python
 def safe_processor(data):
-    return data.get("value", 0) * 2
+    data["doubled"] = data.get("value", 0) * 2
+    return data
 ```
     </code>
   </function_to_generate>
@@ -557,3 +573,168 @@ def test_cleaner_rejects_dangerous_code_and_retries(tmp_path):
     safety_events = [e for e in events if e["type"] == "safety_failed"]
     assert len(safety_events) == 1
     assert "os" in safety_events[0]["error"]
+
+
+# Tests for extract_modified_fields
+
+
+def test_extract_modified_fields_simple():
+    """Extract single field from record assignment."""
+    code = '''
+def fix_phone(record: dict) -> dict:
+    record["phone"] = normalize(record["phone"])
+    return record
+'''
+    fields = extract_modified_fields(code)
+    assert fields == {"phone"}
+
+
+def test_extract_modified_fields_multiple():
+    """Extract multiple fields from function."""
+    code = '''
+def clean_record(record: dict) -> dict:
+    record["phone"] = normalize(record["phone"])
+    record["email"] = record["email"].lower()
+    return record
+'''
+    fields = extract_modified_fields(code)
+    assert fields == {"phone", "email"}
+
+
+def test_extract_modified_fields_data_param():
+    """Works with 'data' parameter name too."""
+    code = '''
+def fix_status(data: dict) -> dict:
+    data["status"] = data["status"].lower()
+    return data
+'''
+    fields = extract_modified_fields(code)
+    assert fields == {"status"}
+
+
+def test_extract_modified_fields_no_assignments():
+    """Returns empty set when no field assignments."""
+    code = '''
+def passthrough(record: dict) -> dict:
+    return record
+'''
+    fields = extract_modified_fields(code)
+    assert fields == set()
+
+
+def test_extract_modified_fields_nested_not_extracted():
+    """Only extracts direct record assignments, not nested."""
+    code = '''
+def process(record: dict) -> dict:
+    temp = {}
+    temp["key"] = "value"
+    record["result"] = temp
+    return record
+'''
+    fields = extract_modified_fields(code)
+    assert fields == {"result"}
+
+
+# Integration test: duplicate field detection in DataCleaner
+
+
+def test_cleaner_rejects_duplicate_field_with_feedback(tmp_path):
+    """Cleaner rejects functions that modify already-covered fields."""
+    test_file = tmp_path / "test.jsonl"
+    test_file.write_text('{"phone": "555-1234", "status": "active"}\n')
+
+    # First response: phone function (should be accepted)
+    response_phone = '''
+<cleaning_analysis>
+  <issues_detected>
+    <issue id="1" solved="false">Phone needs normalization</issue>
+  </issues_detected>
+  <function_to_generate>
+    <name>normalize_phone</name>
+    <docstring>Normalize phone numbers.</docstring>
+    <code>
+```python
+def normalize_phone(record: dict) -> dict:
+    record["phone"] = record["phone"].replace("-", "")
+    return record
+```
+    </code>
+  </function_to_generate>
+  <chunk_status>needs_more_work</chunk_status>
+</cleaning_analysis>
+'''
+
+    # Second response: another phone function (should be rejected as duplicate)
+    response_phone_again = '''
+<cleaning_analysis>
+  <issues_detected>
+    <issue id="1" solved="false">Phone still needs work</issue>
+  </issues_detected>
+  <function_to_generate>
+    <name>fix_phone_format</name>
+    <docstring>Fix phone format.</docstring>
+    <code>
+```python
+def fix_phone_format(record: dict) -> dict:
+    record["phone"] = "+1" + record["phone"]
+    return record
+```
+    </code>
+  </function_to_generate>
+  <chunk_status>needs_more_work</chunk_status>
+</cleaning_analysis>
+'''
+
+    # Third response: status function (should be accepted - different field)
+    response_status = '''
+<cleaning_analysis>
+  <issues_detected>
+    <issue id="1" solved="true">Phone handled</issue>
+    <issue id="2" solved="false">Status needs fixing</issue>
+  </issues_detected>
+  <function_to_generate>
+    <name>fix_status</name>
+    <docstring>Fix status field.</docstring>
+    <code>
+```python
+def fix_status(record: dict) -> dict:
+    record["status"] = record["status"].lower()
+    return record
+```
+    </code>
+  </function_to_generate>
+  <chunk_status>needs_more_work</chunk_status>
+</cleaning_analysis>
+'''
+
+    response_clean = '''
+<cleaning_analysis>
+  <issues_detected>
+    <issue id="1" solved="true">Phone handled</issue>
+    <issue id="2" solved="true">Status handled</issue>
+  </issues_detected>
+  <chunk_status>clean</chunk_status>
+</cleaning_analysis>
+'''
+
+    mock_llm = MockLLM([response_phone, response_phone_again, response_status, response_clean])
+
+    cleaner = DataCleaner(
+        llm_backend=mock_llm,
+        file_path=str(test_file),
+        chunk_size=10,
+        validate_runtime=True,
+    )
+    cleaner.run()
+
+    # Should have 2 functions: normalize_phone and fix_status
+    # fix_phone_format should have been rejected as duplicate
+    assert len(cleaner.functions) == 2
+    function_names = [f["name"] for f in cleaner.functions]
+    assert "normalize_phone" in function_names
+    assert "fix_status" in function_names
+    assert "fix_phone_format" not in function_names
+
+    # The retry prompt should mention "already generated" or "duplicate"
+    assert any("phone" in call.lower() and ("already" in call.lower() or "duplicate" in call.lower())
+               for call in mock_llm.calls[2:])  # Check prompts after first acceptance