recursive-cleaner 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

recursive_cleaner/__init__.py

@@ -22,7 +22,7 @@ from recursive_cleaner.prompt import build_prompt
 from recursive_cleaner.response import extract_python_block, parse_response
 from recursive_cleaner.parser_generator import check_parser_safety, generate_parser
 from recursive_cleaner.tui import HAS_RICH, TUIRenderer
-from recursive_cleaner.validation import check_code_safety, extract_sample_data, validate_function
+from recursive_cleaner.validation import check_code_safety, extract_modified_fields, extract_sample_data, validate_function
 
 __all__ = [
     "apply_cleaning",
@@ -43,6 +43,7 @@ __all__ = [
     "validate_function",
     "extract_sample_data",
     "check_code_safety",
+    "extract_modified_fields",
     "resolve_dependencies",
     "QualityMetrics",
     "measure_quality",

recursive_cleaner/cleaner.py

@@ -17,7 +17,7 @@ from .prompt import build_prompt
 from .response import parse_response
 from .schema import format_schema_for_prompt, infer_schema
 from .types import LLMBackend
-from .validation import check_code_safety, extract_sample_data, split_holdout, validate_function
+from .validation import check_code_safety, extract_modified_fields, extract_sample_data, split_holdout, validate_function
 
 STATE_VERSION = "0.5.0"
 
@@ -110,6 +110,8 @@ class DataCleaner:
             "min_ms": float("inf"),
             "max_ms": 0.0,
         }
+        # Track fields already covered by generated functions (per chunk)
+        self._fields_covered: set[str] = set()
 
     def _emit(self, event_type: str, chunk_index: int = 0, **kwargs) -> None:
         """Emit a progress event to the callback, if set."""
@@ -533,6 +535,8 @@ class DataCleaner:
         """Process a single chunk, iterating until clean or max iterations."""
         self._emit("chunk_start", chunk_index=chunk_idx)
         error_feedback = ""
+        # Reset fields covered for new chunk
+        self._fields_covered = set()
 
         # Dry run mode: just detect issues, don't generate functions
         if self.dry_run:
@@ -594,6 +598,20 @@ class DataCleaner:
                         print(f"  Safety check failed: {safety_error}")
                     continue
 
+                # Check for duplicate field coverage
+                new_fields = extract_modified_fields(result["code"])
+                overlap = new_fields & self._fields_covered
+                if overlap:
+                    field_list = ", ".join(sorted(overlap))
+                    error_feedback = f"You already generated a function for field(s): {field_list}. This issue is solved. Move on to the next unsolved issue."
+                    self._emit(
+                        "duplicate_field",
+                        chunk_index=chunk_idx,
+                        function_name=result["name"],
+                        fields=list(overlap),
+                    )
+                    continue
+
                 # Runtime validation if enabled
                 if self.validate_runtime:
                     # Use holdout data if available, else sample from generation chunk
@@ -628,6 +646,8 @@ class DataCleaner:
                     "docstring": result["docstring"],
                     "code": result["code"],
                 })
+                # Track fields covered by this function
+                self._fields_covered.update(new_fields)
                 # Track for saturation check
                 self._recent_new_function_count += 1
 

recursive_cleaner/prompt.py

@@ -52,7 +52,8 @@ CONSOLIDATION_TEMPLATE = '''You are reviewing cleaning functions for consolidati
       </docstring>
       <code>
 ```python
-def merged_function_name(record):
+def merged_function_name(record: dict) -> dict:
+    # Modify fields, return record
     ...
 ```
       </code>
@@ -108,9 +109,10 @@ Tags: domain, action, detail
     </docstring>
     <code>
 ```python
-def function_name(data):
-    # Complete implementation
-    pass
+def function_name(record: dict) -> dict:
+    # Modify field(s) in the record
+    record["field"] = cleaned_value
+    return record
 ```
     </code>
   </function_to_generate>
@@ -120,6 +122,8 @@ def function_name(data):
 
 RULES:
 - ONE function per response
+- Function signature: takes a dict (one record), returns the modified dict
+- Modify fields directly on the record, then return it
 - If all issues solved: <chunk_status>clean</chunk_status>, omit <function_to_generate>
 - Include imports inside the function or document needed imports in docstring
 - Function must be idempotent (safe to run multiple times)

recursive_cleaner/validation.py

@@ -160,7 +160,10 @@ def validate_function(
         # Structured mode: sample_data is list[dict]
         for i, record in enumerate(sample_data):
             try:
-                func(record)
+                result = func(record)
+                # Verify function returns a dict (not string, int, etc.)
+                if not isinstance(result, dict):
+                    return False, f"Function must return dict, got {type(result).__name__}"
             except Exception as e:
                 return False, f"Runtime error on sample {i}: {type(e).__name__}: {e}"
 
@@ -200,3 +203,39 @@ def extract_sample_data(
         except json.JSONDecodeError:
             continue
     return samples
+
+
+def extract_modified_fields(code: str) -> set[str]:
+    """
+    Extract field names that are modified via record["field"] = ... pattern.
+
+    Args:
+        code: Python source code of the function
+
+    Returns:
+        Set of field names that are assigned to
+    """
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return set()
+
+    fields = set()
+    # Common parameter names for the data/record argument
+    data_names = {"record", "data"}
+
+    for node in ast.walk(tree):
+        # Look for assignment statements
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                # Check if target is a subscript: record["field"] or data["field"]
+                if isinstance(target, ast.Subscript):
+                    # The value should be a Name node (record or data)
+                    if isinstance(target.value, ast.Name):
+                        if target.value.id in data_names:
+                            # The slice should be a string constant
+                            if isinstance(target.slice, ast.Constant):
+                                if isinstance(target.slice.value, str):
+                                    fields.add(target.slice.value)
+
+    return fields

{recursive_cleaner-1.0.0.dist-info → recursive_cleaner-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recursive-cleaner
-Version: 1.0.0
+Version: 1.0.1
 Summary: LLM-powered incremental data cleaning pipeline that processes massive datasets in chunks and generates Python cleaning functions
 Project-URL: Homepage, https://github.com/gaztrabisme/recursive-data-cleaner
 Project-URL: Repository, https://github.com/gaztrabisme/recursive-data-cleaner
@@ -389,7 +389,7 @@ backends/
 pytest tests/ -v
 ```
 
-548 tests covering all features. Test datasets in `test_cases/`:
+555 tests covering all features. Test datasets in `test_cases/`:
 - E-commerce product catalogs
 - Healthcare patient records
 - Financial transaction data

{recursive_cleaner-1.0.0.dist-info → recursive_cleaner-1.0.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 backends/__init__.py,sha256=vWcPASV0GGEAydzOSjdrknkSHoGbSs4edtuv9HIzBhI,180
 backends/mlx_backend.py,sha256=0U6IqmDHyk4vjKzytvEcQvSUBryQTgFtsNOcpwFNKk8,2945
 backends/openai_backend.py,sha256=vKWsXKltBv_tJDoQfQ_7KVMZDfomhFFN2vl1oZ1KGbQ,2057
-recursive_cleaner/__init__.py,sha256=xCFlkqmmBoa7ntUZQnRQxVMv9iLeOvmboDS_j2EHfZI,1862
+recursive_cleaner/__init__.py,sha256=-NesTf9deCVOxkadFuyfVl-IjfbEHlYcMNAaAW9kUuw,1918
 recursive_cleaner/__main__.py,sha256=WXmMaL_myHPsG_qXAhZDufD43Ydsd25RV2IPeW2Kg08,152
 recursive_cleaner/apply.py,sha256=hjeljhZNiOuwz9m09RYVLl_z_9tet7LwubH6cb_Wy6Y,13855
-recursive_cleaner/cleaner.py,sha256=kPOQ44hgiJzABiqdmjg2hqd7Ot9uxKUSOe8_jz0UBQc,29911
+recursive_cleaner/cleaner.py,sha256=lLe7LNaVYwukDhBTxLs8ezsQf7fes9m9OX7g9nGo760,30954
 recursive_cleaner/cli.py,sha256=Sk_qYKxSn1PiPmMLKkyj9VxsseHaSXmSlGazxfmkTFc,12807
 recursive_cleaner/context.py,sha256=avMXRDxLd7nd8CKWtvPHQy1MFhBKiA0aUVVJIlWoLZ4,824
 recursive_cleaner/dependencies.py,sha256=vlYeoGL517v3yUSWN0wYDuIs9OOuQwM_dCBADrlitW8,2080
@@ -14,17 +14,17 @@ recursive_cleaner/optimizer.py,sha256=lnQC9Y1ClkW4po1eYa2bnYYu4smiDuUpMPPX6EN1UQ
 recursive_cleaner/output.py,sha256=quTlZYtKZm9h37mbnwQmEjg0q8VQSZWEqwaHfhSAd3s,6106
 recursive_cleaner/parser_generator.py,sha256=enn6_okGWB2ddVkwI7ytndT04S4QEVAk6cbmb7shxcM,3905
 recursive_cleaner/parsers.py,sha256=HCS2UiVFhboq_go4DyWUygkJTkpfYkFj9_hqWiGIEXo,14572
-recursive_cleaner/prompt.py,sha256=ep0eOXz_XbhH3HduJ76LvzVSftonhcv4GLEecIqd3lY,6484
+recursive_cleaner/prompt.py,sha256=yqwUyB6Z51Oqhvxz3mNijZraXr-QEUYQ_ubyiryZSrU,6730
 recursive_cleaner/report.py,sha256=AWWneRjvl76ccLlExdkKJeY3GVFUG_LtmzVIJJT5cFI,4629
 recursive_cleaner/response.py,sha256=3w0mLnqEPdB4daMSF0mtTcG0PTP-utb1HFtKuYA1ljw,9064
 recursive_cleaner/schema.py,sha256=w2hcEdApR15KVI9SFWB3VfumMoHFwn1YJrktdfgPo8M,3925
 recursive_cleaner/tui.py,sha256=zuiFPtMh3K-sC1CWZoaoUmgZ3rESkl10gYcqMzpVqiM,22598
 recursive_cleaner/types.py,sha256=-GdCmsfHd3rfdfCi5c-RXqX4TyuCSHgA__3AF3bMhoQ,290
-recursive_cleaner/validation.py,sha256=-KAolhw3GQyhHwmh0clEj8xqPD5O-R2AO5rx7vubIME,6442
+recursive_cleaner/validation.py,sha256=IlXz5EhXaUb0mJlaH0ygFH1ePPWHVfgjL-5ZawyKicY,7910
 recursive_cleaner/vendor/__init__.py,sha256=E87TjmjRzu8ty39nqThvBwM611yXlLKQZ6KGY_zp3Dk,117
 recursive_cleaner/vendor/chunker.py,sha256=pDDbfF6FoSmUji0-RG4MletPxJ-VybGw0yfnhh0aMSQ,6730
-recursive_cleaner-1.0.0.dist-info/METADATA,sha256=L86ATNd8JxmPp32HKaO6PPwkmq4sIE3Mdvgx3pmUulE,14285
-recursive_cleaner-1.0.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-recursive_cleaner-1.0.0.dist-info/entry_points.txt,sha256=S5nbi0rnifpShxdXGExeZnd65UZfp8K7DNyuKPST6nk,65
-recursive_cleaner-1.0.0.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
-recursive_cleaner-1.0.0.dist-info/RECORD,,
+recursive_cleaner-1.0.1.dist-info/METADATA,sha256=qEmuiRPtRjuigM29FgjrkUUZm0YV91xNjuc7j16NhKU,14285
+recursive_cleaner-1.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+recursive_cleaner-1.0.1.dist-info/entry_points.txt,sha256=S5nbi0rnifpShxdXGExeZnd65UZfp8K7DNyuKPST6nk,65
+recursive_cleaner-1.0.1.dist-info/licenses/LICENSE,sha256=P8hRMK-UqRbQDsVN9nr901wpZcqwXEHr28DXhBUheF0,1064
+recursive_cleaner-1.0.1.dist-info/RECORD,,