xgen-doc2chunk 0.1.4__tar.gz → 0.1.51__tar.gz

{xgen_doc2chunk-0.1.4 → xgen_doc2chunk-0.1.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xgen-doc2chunk
-Version: 0.1.4
+Version: 0.1.51
 Summary: Convert raw documents into AI-understandable context with intelligent text extraction, table detection, and semantic chunking
 Project-URL: Homepage, https://github.com/master0419/doc2chunk
 Project-URL: Documentation, https://github.com/master0419/doc2chunk#readme

{xgen_doc2chunk-0.1.4 → xgen_doc2chunk-0.1.51}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xgen-doc2chunk"
-version = "0.1.4"
+version = "0.1.51"
 description = "Convert raw documents into AI-understandable context with intelligent text extraction, table detection, and semantic chunking"
 readme = "README.md"
 requires-python = ">=3.12"

{xgen_doc2chunk-0.1.4 → xgen_doc2chunk-0.1.51}/xgen_doc2chunk/core/processor/pdf_helpers/pdf_table_validator.py

@@ -170,7 +170,7 @@ class TableQualityValidator:
         if paragraph_count > 0:
             # High probability of not being a table if paragraph-style text exists
             paragraph_ratio = paragraph_count / max(1, filled_cells)
-            if paragraph_ratio > 0.25:  # Relaxed from 15% to 25%
+            if paragraph_ratio > 0.60:  # Relaxed from 25% to 60%
                 return False, 0.0, f"contains_paragraph_text({paragraph_count})"
             elif paragraph_ratio > 0.1:  # Relaxed from 5% to 10%
                 penalties.append(f"has_paragraph_cells({paragraph_count})")
@@ -379,15 +379,15 @@ class TableQualityValidator:
             if col1_empty_ratio >= 0.6 and col2_long_ratio >= 0.3:
                 return False, f"col1_empty({col1_empty_ratio:.0%})_col2_long({col2_long_ratio:.0%})"
         
-        # Pattern 2: Many paragraph-style entries in second column
-        if num_rows > 5 and col2_has_paragraphs >= 2:
-            return False, f"col2_paragraphs({col2_has_paragraphs})"
+        # # Pattern 2: Many paragraph-style entries in second column
+        # if num_rows > 5 and col2_has_paragraphs >= 2:
+        #     return False, f"col2_paragraphs({col2_has_paragraphs})"
         
-        # Pattern 3: If first column is short and second is long overall, likely body text not key-value
-        if num_rows > 10:
-            col1_short_ratio = (col1_empty_count + col1_short_count) / num_rows
-            if col1_short_ratio >= 0.8 and col2_long_count >= 5:
-                return False, f"asymmetric_cols(short1={col1_short_ratio:.0%}, long2={col2_long_count})"
+        # # Pattern 3: If first column is short and second is long overall, likely body text not key-value
+        # if num_rows > 10:
+        #     col1_short_ratio = (col1_empty_count + col1_short_count) / num_rows
+        #     if col1_short_ratio >= 0.8 and col2_long_count >= 5:
+        #         return False, f"asymmetric_cols(short1={col1_short_ratio:.0%}, long2={col2_long_count})"
         
         return True, "valid"