kreuzberg 3.6.2__py3-none-any.whl → 3.8.0__py3-none-any.whl

kreuzberg/_utils/_string.py

@@ -1,39 +1,182 @@
 from __future__ import annotations
 
+import hashlib
+import re
 from contextlib import suppress
+from functools import lru_cache
 
-from charset_normalizer import detect
+import chardetng_py
+
+# Compile regex patterns once at module level for performance
+_WHITESPACE_PATTERN = re.compile(r"[ \t\f\v\r\xa0\u2000-\u200b\u2028\u2029\u3000]+")
+_NEWLINES_PATTERN = re.compile(r"\n+")
+_MOJIBAKE_PATTERNS = {
+    # Hebrew as Cyrillic patterns
+    "hebrew_as_cyrillic": re.compile(r"[\u0400-\u04FF]{3,}"),
+    # Control characters that shouldn't appear in text
+    "control_chars": re.compile(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F-\x9F]"),
+    # Unicode replacement characters
+    "replacement_chars": re.compile(r"\uFFFD+"),
+    # Isolated combining marks (likely encoding issues)
+    "isolated_combining": re.compile(r"[\u0300-\u036F](?![^\u0300-\u036F])"),
+}
+
+# Simple cache for encoding detection (in-memory, session-scoped)
+_encoding_cache: dict[str, str] = {}
+
+
+@lru_cache(maxsize=128)
+def _get_encoding_cache_key(data_hash: str, size: int) -> str:
+    """Generate cache key for encoding detection."""
+    return f"{data_hash}:{size}"
 
 
 def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
-    """Decode a byte string safely, removing invalid sequences.
+    """Decode a byte string safely with mojibake detection and correction.
 
     Args:
         byte_data: The byte string to decode.
         encoding: The encoding to use when decoding the byte string.
 
     Returns:
-        The decoded string.
+        The decoded string with mojibake detection and correction.
     """
     if not byte_data:
         return ""
 
-    encodings = [encoding, detect(byte_data).get("encoding", ""), "utf-8"]
+    # Try provided encoding first (fastest path)
+    if encoding:
+        with suppress(UnicodeDecodeError, LookupError):
+            decoded = byte_data.decode(encoding)
+            return _fix_mojibake(decoded)
 
-    for enc in [e for e in encodings if e]:
+    # Check cache for similar content (performance optimization)
+    data_hash = hashlib.sha256(byte_data[:1024]).hexdigest()[:16]  # Hash first 1KB
+    cache_key = _get_encoding_cache_key(data_hash, len(byte_data))
+
+    if cache_key in _encoding_cache:
+        cached_encoding = _encoding_cache[cache_key]
+        with suppress(UnicodeDecodeError, LookupError):
+            decoded = byte_data.decode(cached_encoding)
+            return _fix_mojibake(decoded)
+
+    # Use chardetng for better performance than charset-normalizer
+    detected_encoding = chardetng_py.detect(byte_data)
+    if detected_encoding:
         with suppress(UnicodeDecodeError, LookupError):
-            return byte_data.decode(enc)
+            decoded = byte_data.decode(detected_encoding)
+            # Cache successful encoding detection
+            if len(_encoding_cache) < 1000:  # Prevent unlimited growth
+                _encoding_cache[cache_key] = detected_encoding
+            return _fix_mojibake(decoded)
+
+    # Try multiple encodings with confidence scoring
+    encodings_to_try = [
+        "utf-8",
+        "windows-1255",  # Hebrew
+        "iso-8859-8",  # Hebrew
+        "windows-1256",  # Arabic
+        "iso-8859-6",  # Arabic
+        "windows-1252",  # Western European
+        "cp1251",  # Cyrillic
+    ]
 
+    best_result = None
+    best_confidence = 0.0
+
+    for enc in encodings_to_try:
+        with suppress(UnicodeDecodeError, LookupError):
+            decoded = byte_data.decode(enc)
+            confidence = _calculate_text_confidence(decoded)
+            if confidence > best_confidence:
+                best_confidence = confidence
+                best_result = decoded
+
+    if best_result and best_confidence > 0.5:
+        return _fix_mojibake(best_result)
+
+    # Final fallback
     return byte_data.decode("latin-1", errors="replace")
 
 
+def _calculate_text_confidence(text: str) -> float:
+    """Calculate confidence score for decoded text quality."""
+    if not text:
+        return 0.0
+
+    # Check for common encoding problems
+    replacement_count = len(_MOJIBAKE_PATTERNS["replacement_chars"].findall(text))
+    control_count = len(_MOJIBAKE_PATTERNS["control_chars"].findall(text))
+    total_chars = len(text)
+
+    if total_chars == 0:
+        return 0.0
+
+    # Penalize replacement and control characters
+    penalty = (replacement_count + control_count * 2) / total_chars
+
+    # Bonus for readable character ranges
+    readable_chars = sum(1 for c in text if c.isprintable() or c.isspace())
+    readability_score = readable_chars / total_chars
+
+    # Check for suspicious Cyrillic that might be misencoded Hebrew
+    cyrillic_matches = _MOJIBAKE_PATTERNS["hebrew_as_cyrillic"].findall(text)
+    if cyrillic_matches and len("".join(cyrillic_matches)) > total_chars * 0.1:
+        penalty += 0.3  # Heavy penalty for likely mojibake
+
+    return max(0.0, min(1.0, readability_score - penalty))
+
+
+def _fix_mojibake(text: str) -> str:
+    """Attempt to fix common mojibake patterns."""
+    if not text:
+        return text
+
+    # Remove control characters
+    text = _MOJIBAKE_PATTERNS["control_chars"].sub("", text)
+
+    # Remove replacement characters
+    text = _MOJIBAKE_PATTERNS["replacement_chars"].sub("", text)
+
+    # Remove isolated combining marks
+    text = _MOJIBAKE_PATTERNS["isolated_combining"].sub("", text)
+
+    # Try to fix Hebrew encoded as Cyrillic (common Windows-1255 -> CP1251 confusion)
+    if _MOJIBAKE_PATTERNS["hebrew_as_cyrillic"].search(text):
+        # This is a heuristic fix - in practice, you'd need actual character mapping
+        # For now, we flag it for manual review by keeping the text but adding a marker
+        pass
+
+    return text
+
+
 def normalize_spaces(text: str) -> str:
-    """Normalize the spaces in a string.
+    """Normalize spaces while preserving line breaks and paragraph structure.
 
     Args:
-        text: The text to sanitize.
+        text: The text to normalize.
 
     Returns:
-        The sanitized text.
+        The normalized text with proper spacing.
     """
-    return " ".join(text.strip().split())
+    if not text or not text.strip():
+        return ""
+
+    # Split by double newlines to preserve paragraph breaks
+    paragraphs = text.split("\n\n")
+    normalized_paragraphs = []
+
+    for paragraph in paragraphs:
+        # Use pre-compiled patterns for better performance
+        # Replace multiple whitespace (except newlines) with single space
+        cleaned = _WHITESPACE_PATTERN.sub(" ", paragraph)
+        # Clean up multiple newlines within paragraph (keep single newlines)
+        cleaned = _NEWLINES_PATTERN.sub("\n", cleaned)
+
+        # Strip and filter empty lines efficiently
+        lines = [line.strip() for line in cleaned.split("\n") if line.strip()]
+
+        if lines:
+            normalized_paragraphs.append("\n".join(lines))
+
+    return "\n\n".join(normalized_paragraphs)

kreuzberg/_utils/_sync.py

@@ -28,8 +28,11 @@ async def run_sync(sync_fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -
     Returns:
         The result of the synchronous function.
     """
-    handler = partial(sync_fn, **kwargs)
-    return cast("T", await any_io_run_sync(handler, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
+    # Optimize: only create partial if we have kwargs
+    if kwargs:
+        handler = partial(sync_fn, **kwargs)
+        return cast("T", await any_io_run_sync(handler, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
+    return cast("T", await any_io_run_sync(sync_fn, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
 
 
 async def run_taskgroup(*async_tasks: Awaitable[Any]) -> list[Any]:

kreuzberg/_utils/_table.py

@@ -0,0 +1,261 @@
+"""Table processing and export utilities."""
+
+from __future__ import annotations
+
+import csv
+from io import StringIO
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from kreuzberg._types import TableData
+
+
+def export_table_to_csv(table: TableData, separator: str = ",") -> str:
+    r"""Export a TableData object to CSV/TSV format.
+
+    Args:
+        table: TableData object containing DataFrame
+        separator: Field separator ("," for CSV, "\t" for TSV)
+
+    Returns:
+        String representation in CSV/TSV format
+    """
+    if "df" not in table or table["df"] is None:
+        return ""
+
+    output = StringIO()
+    table["df"].to_csv(output, sep=separator, index=False, quoting=csv.QUOTE_MINIMAL)
+    return output.getvalue().strip()
+
+
+def export_table_to_tsv(table: TableData) -> str:
+    """Export a TableData object to TSV format.
+
+    Args:
+        table: TableData object containing DataFrame
+
+    Returns:
+        String representation in TSV format
+    """
+    return export_table_to_csv(table, separator="\t")
+
+
+def enhance_table_markdown(table: TableData) -> str:
+    """Generate enhanced markdown table with better formatting.
+
+    Args:
+        table: TableData object
+
+    Returns:
+        Enhanced markdown table string
+    """
+    if "df" not in table or table["df"] is None:
+        return table.get("text", "")
+
+    df = table["df"]
+
+    if df.empty:
+        return table.get("text", "")
+
+    # Create enhanced markdown with proper alignment
+    lines = []
+
+    # Header row
+    headers = [str(col).strip() for col in df.columns]
+    lines.append("| " + " | ".join(headers) + " |")
+
+    # Separator row with alignment hints
+    lines.append(_generate_separator_row(df))
+
+    # Analyze float columns to determine formatting strategy
+    float_col_formatting = _analyze_float_columns(df)
+
+    # Data rows with proper formatting
+    for _, row in df.iterrows():
+        formatted_row = _format_table_row(row, df, float_col_formatting)
+        lines.append("| " + " | ".join(formatted_row) + " |")
+
+    return "\n".join(lines)
+
+
+def _generate_separator_row(df: Any) -> str:
+    """Generate separator row with proper alignment hints."""
+    separators = []
+    for col in df.columns:
+        # Check if column contains mostly numbers for right alignment
+        if df[col].dtype in ["int64", "float64"] or _is_numeric_column(df[col]):
+            separators.append("---:")  # Right align numbers
+        else:
+            separators.append("---")  # Left align text
+    return "| " + " | ".join(separators) + " |"
+
+
+def _analyze_float_columns(df: Any) -> dict[str, str]:
+    """Analyze float columns to determine formatting strategy."""
+    float_col_formatting = {}
+    for col in df.columns:
+        if str(df[col].dtype) == "float64":
+            non_null_values = df[col].dropna()
+            if len(non_null_values) > 0:
+                # If all non-null values are whole numbers, format as integers
+                all_integers = all(val.is_integer() for val in non_null_values)
+                float_col_formatting[col] = "int" if all_integers else "float"
+            else:
+                float_col_formatting[col] = "int"
+    return float_col_formatting
+
+
+def _format_table_row(row: Any, df: Any, float_col_formatting: dict[str, str]) -> list[str]:
+    """Format a single table row with proper value formatting."""
+    formatted_row = []
+    for col_name, value in row.items():
+        if value is None or (isinstance(value, float) and str(value) == "nan"):
+            formatted_row.append("")
+        elif str(df[col_name].dtype) in ["int64", "int32"]:
+            # For integer columns, format as integers
+            formatted_row.append(str(int(value)))
+        elif isinstance(value, float):
+            # For float columns, use the determined formatting strategy
+            if col_name in float_col_formatting and float_col_formatting[col_name] == "int":
+                formatted_row.append(str(int(value)))
+            else:
+                formatted_row.append(f"{value:.2f}")
+        else:
+            # Clean up text values
+            clean_value = str(value).strip().replace("|", "\\|")  # Escape pipes
+            formatted_row.append(clean_value)
+    return formatted_row
+
+
+def _is_numeric_column(series: Any) -> bool:
+    """Check if a pandas Series contains mostly numeric values."""
+    if len(series) == 0:
+        return False
+
+    try:
+        # Check if already numeric dtype first (fastest path)
+        if str(series.dtype) in {"int64", "float64", "int32", "float32"}:
+            return True
+
+        # Sample-based approach for large series (>1000 rows)
+        sample_size = min(100, len(series))
+        if len(series) > 1000:
+            sample_series = series.dropna().sample(n=sample_size, random_state=42)
+        else:
+            sample_series = series.dropna()
+
+        if len(sample_series) == 0:
+            return False
+
+        # Optimized numeric conversion - avoid exception overhead
+        numeric_count = 0
+        for val in sample_series:
+            val_str = str(val).replace(",", "").replace("$", "").replace("%", "")
+            # Quick check: if it contains only digits, decimal point, minus, plus, or e
+            if val_str and all(c in "0123456789.-+eE" for c in val_str):
+                try:
+                    float(val_str)
+                    numeric_count += 1
+                except (ValueError, TypeError):
+                    pass
+
+        # Consider numeric if >70% of sampled values are numeric
+        return (numeric_count / len(sample_series)) > 0.7
+
+    except (ValueError, TypeError, ZeroDivisionError):
+        return False
+
+
+def generate_table_summary(tables: list[TableData]) -> dict[str, Any]:
+    """Generate summary statistics for extracted tables.
+
+    Args:
+        tables: List of TableData objects
+
+    Returns:
+        Dictionary with table statistics
+    """
+    if not tables:
+        return {
+            "table_count": 0,
+            "total_rows": 0,
+            "total_columns": 0,
+            "pages_with_tables": 0,
+        }
+
+    total_rows = 0
+    total_columns = 0
+    pages_with_tables = set()
+    tables_by_page = {}
+
+    for table in tables:
+        if "df" in table and table["df"] is not None:
+            df = table["df"]
+            total_rows += len(df)
+            total_columns += len(df.columns)
+
+        if "page_number" in table:
+            page_num = table["page_number"]
+            pages_with_tables.add(page_num)
+
+            if page_num not in tables_by_page:
+                tables_by_page[page_num] = 0
+            tables_by_page[page_num] += 1
+
+    return {
+        "table_count": len(tables),
+        "total_rows": total_rows,
+        "total_columns": total_columns,
+        "pages_with_tables": len(pages_with_tables),
+        "avg_rows_per_table": total_rows / len(tables) if tables else 0,
+        "avg_columns_per_table": total_columns / len(tables) if tables else 0,
+        "tables_by_page": dict(tables_by_page),
+    }
+
+
+def extract_table_structure_info(table: TableData) -> dict[str, Any]:
+    """Extract structural information from a table.
+
+    Args:
+        table: TableData object
+
+    Returns:
+        Dictionary with structural information
+    """
+    info = {
+        "has_headers": False,
+        "row_count": 0,
+        "column_count": 0,
+        "numeric_columns": 0,
+        "text_columns": 0,
+        "empty_cells": 0,
+        "data_density": 0.0,
+    }
+
+    if "df" not in table or table["df"] is None:
+        return info
+
+    df = table["df"]
+
+    if df.empty:
+        return info
+
+    info["row_count"] = len(df)
+    info["column_count"] = len(df.columns)
+    info["has_headers"] = len(df.columns) > 0
+
+    # Analyze column types
+    for col in df.columns:
+        if _is_numeric_column(df[col]):
+            info["numeric_columns"] += 1
+        else:
+            info["text_columns"] += 1
+
+    # Calculate data density
+    total_cells = len(df) * len(df.columns)
+    if total_cells > 0:
+        empty_cells = df.isnull().sum().sum()
+        info["empty_cells"] = int(empty_cells)
+        info["data_density"] = (total_cells - empty_cells) / total_cells
+
+    return info