autochunks 0.0.8__py3-none-any.whl

autochunk/chunkers/html_section.py

@@ -0,0 +1,225 @@
+
+from __future__ import annotations
+from typing import List, Optional, Dict, Any
+import re
+from .base import BaseChunker, Chunk
+from ..utils.text import count_tokens
+
+try:
+    from bs4 import BeautifulSoup, NavigableString, Tag
+    BS4_AVAILABLE = True
+except ImportError:
+    BS4_AVAILABLE = False
+
+class HTMLSectionChunker(BaseChunker):
+    """
+    DOM-Aware HTML Chunker for structural web content splitting.
+    
+    BEST-OF-BREED FEATURES:
+    1. DOM-Tree Navigation: Respects HTML hierarchy (doesn't blindly split tags).
+    2. Structural Metadata: Tracks DOM path IDs (e.g. body > main > article).
+    3. Semantic Grouping: Keeps tables, lists, and definition lists intact.
+    4. Header Hierarchy: Uses H1-H6 as natural boundaries.
+    """
+    name = "html_section"
+
+    # Tags that act as hard section boundaries
+    SECTION_TAGS = {'body', 'main', 'section', 'article', 'nav', 'aside', 'footer', 'header'}
+    
+    # Tags that are logical blocks (like paragraphs)
+    BLOCK_TAGS = {'p', 'div', 'blockquote', 'pre', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'figure', 'li', 'td', 'th'}
+    
+    # Tags that should be kept atomic if possible
+    ATOMIC_TAGS = {'table', 'ul', 'ol', 'dl', 'code', 'pre'}
+
+    def __init__(self, 
+                 base_token_size: int = 512, 
+                 max_token_size: int = 2048,
+                 respect_headers: bool = True):
+        self.base_token_size = base_token_size
+        self.max_token_size = max_token_size
+        self.respect_headers = respect_headers
+
+    def chunk(self, doc_id: str, text: str, **params) -> List[Chunk]:
+        """
+        Chunk HTML text using DOM analysis.
+        
+        Args:
+            doc_id: Document ID
+            text: HTML source string
+        """
+        if not BS4_AVAILABLE:
+            from ..utils.logger import logger
+            logger.warning("BeautifulSoup not installed, falling back to RecursiveCharacterChunker")
+            from .recursive_character import RecursiveCharacterChunker
+            return RecursiveCharacterChunker().chunk(doc_id, text, base_token_size=self.base_token_size)
+
+        # Parse HTML
+        soup = BeautifulSoup(text, 'lxml')
+        
+        # Remove noisy tags
+        for t in soup(['script', 'style', 'noscript', 'meta', 'link']):
+            t.decompose()
+
+        chunks = []
+        current_chunk_text = []
+        current_chunk_tokens = 0
+        current_meta = {}
+        
+        # Traverse DOM depth-first
+        elements = self._flatten_dom(soup.body if soup.body else soup)
+        
+        chunk_idx = 0
+        
+        for elem_text, dom_path, is_header in elements:
+            token_count = count_tokens(elem_text)
+            
+            # If single element is huge, need to split it (fallback)
+            if token_count > self.max_token_size:
+                # Flush current accumulator first
+                if current_chunk_text:
+                    self._flush_chunk(doc_id, chunk_idx, current_chunk_text, current_meta, chunks)
+                    chunk_idx += 1
+                    current_chunk_text = []
+                    current_chunk_tokens = 0
+                
+                # Split the huge element
+                from .recursive_character import RecursiveCharacterChunker
+                sub_chunks = RecursiveCharacterChunker().chunk(
+                    f"{doc_id}_sub", elem_text, base_token_size=self.base_token_size
+                )
+                for sc in sub_chunks:
+                    chunks.append(Chunk(
+                        id=f"{doc_id}#html#{chunk_idx}",
+                        doc_id=doc_id,
+                        text=sc.text,
+                        meta={**current_meta, "chunk_index": chunk_idx, "dom_path": dom_path, "subtype": "large_element_split"}
+                    ))
+                    chunk_idx += 1
+                continue
+            
+            # Check if we should split
+            # 1. Header detected (and we have content)
+            # 2. Size limit reached
+            is_new_section = is_header and self.respect_headers
+            is_full = (current_chunk_tokens + token_count) > self.base_token_size
+            
+            if (is_new_section or is_full) and current_chunk_text:
+                self._flush_chunk(doc_id, chunk_idx, current_chunk_text, current_meta, chunks)
+                chunk_idx += 1
+                current_chunk_text = []
+                current_chunk_tokens = 0
+                # Use metadata from new starting element (specifically header info)
+                current_meta = {"dom_path": dom_path, "is_header": is_header}
+            
+            if not current_chunk_text:
+                current_meta = {"dom_path": dom_path, "is_header": is_header}
+            
+            current_chunk_text.append(elem_text)
+            current_chunk_tokens += token_count
+        
+        # Final flush
+        if current_chunk_text:
+            self._flush_chunk(doc_id, chunk_idx, current_chunk_text, current_meta, chunks)
+        
+        return chunks
+
+    def _flush_chunk(self, doc_id, idx, text_parts, meta, chunks_list):
+        full_text = "\n\n".join(text_parts).strip()
+        if not full_text:
+            return
+        
+        chunks_list.append(Chunk(
+            id=f"{doc_id}#html#{idx}",
+            doc_id=doc_id,
+            text=full_text,
+            meta={
+                "chunk_index": idx,
+                "strategy": "html_section",
+                "token_count": count_tokens(full_text),
+                **meta
+            }
+        ))
+
+    def _flatten_dom(self, node) -> List[tuple[str, str, bool]]:
+        """
+        Flatten DOM into text blocks with metadata.
+        Returns list of (text, dom_path, is_header).
+        """
+        results = []
+        
+        # Helper to get path
+        def get_path(tag):
+            path = []
+            p = tag.parent
+            while p and p.name != '[document]':
+                path.insert(0, p.name)
+                p = p.parent
+            path.append(tag.name)
+            return " > ".join(path)
+
+        # Iterate only over block elements or meaningful leaf nodes
+        # This is a simplification: complex iteration logic needed for perfect reconstruction
+        # We'll walk and extract text from "safe" blocks
+        
+        for element in node.descendants:
+            if isinstance(element, Tag):
+                if element.name in self.BLOCK_TAGS or element.name in self.ATOMIC_TAGS:
+                    # Check if this element contains OTHER block tags. If so, don't extract yet (wait for children).
+                    # Exception: ATOMIC_TAGS (tables, etc.) - extract WHOLE text
+                    has_block_children = any(child.name in self.BLOCK_TAGS for child in element.find_all(recursive=False))
+                    
+                    if element.name in self.ATOMIC_TAGS or not has_block_children:
+                        # Extract text
+                        text = element.get_text(separator=" ", strip=True)
+                        if text:
+                            is_header = element.name in {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+                            path = get_path(element)
+                            results.append((text, path, is_header))
+                            
+                        # If atomic, skip processing detailed descendants to avoid dups
+                        # (BeautifulSoup yields same nodes if we don't handle this carefully)
+                        # Actually node.descendants is a flat generator. 
+                        # We need to manually control recursion to avoid duplication.
+                        # Since we can't easily skip in .descendants loop, we just rely on the fact 
+                        # that we only grab "leaf-like" blocks. 
+                        
+        # Better approach: recursive generator
+        return self._recursive_extract(node)
+
+    def _recursive_extract(self, node, path="") -> List[tuple[str, str, bool]]:
+        results = []
+        
+        if isinstance(node, NavigableString):
+            text = str(node).strip()
+            if text:
+                return [(text, path, False)]
+            return []
+            
+        if isinstance(node, Tag):
+            new_path = f"{path} > {node.name}" if path else node.name
+            is_header = node.name in {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+            
+            # Atomic tags: return all text as one block
+            if node.name in self.ATOMIC_TAGS:
+                text = node.get_text(separator="\n", strip=True)
+                if text:
+                    return [(text, new_path, is_header)]
+                return []
+            
+            # Block tags: process content
+            if node.name in self.BLOCK_TAGS:
+                # Process children
+                block_content = []
+                for child in node.children:
+                    block_content.extend(self._recursive_extract(child, new_path))
+                
+                # If we gathered content, maybe we should join it if it's small?
+                # For now, just return specific detailed blocks
+                return block_content
+                
+            # Inline tags (span, b, etc.): just continue
+            for child in node.children:
+                results.extend(self._recursive_extract(child, path)) # passing parent path for inline
+                
+        return results

autochunk/chunkers/hybrid_semantic_stat.py

@@ -0,0 +1,199 @@
+
+from __future__ import annotations
+import numpy as np
+from typing import List, Any, Callable, Dict
+from .base import BaseChunker, Chunk
+from ..utils.text import split_sentences, count_tokens
+
+class HybridSemanticStatChunker(BaseChunker):
+    """
+     Hybrid Chunker combining Semantic Similarity with Statistical Constraints.
+    
+    BEST-OF-BREED FEATURES:
+    1. Windowed Similarity: Uses window-averaged embeddings for noise suppression.
+    2. Percentile-Based Threshold: Adaptive boundary detection like SemanticLocal.
+    3. Statistical Forces: Token pressure, sentence length variance, and entropy.
+    4. Multi-Factor Scoring: Configurable weights for semantic vs statistical signals.
+    """
+    name = "hybrid_semantic_stat"
+
+    def __init__(self,
+                 alpha: float = 0.6,
+                 beta: float = 0.4,
+                 window_size: int = 3,
+                 threshold_percentile: float = 0.85):
+        """
+        Initialize the chunker.
+        
+        Args:
+            alpha: Weight for semantic similarity signal (0-1)
+            beta: Weight for statistical signal (0-1)
+            window_size: Number of sentences for windowed similarity
+            threshold_percentile: Percentile for adaptive threshold (0-1)
+        """
+        self.alpha = alpha
+        self.beta = beta
+        self.window_size = window_size
+        self.threshold_percentile = threshold_percentile
+
+    def chunk(self, 
+              doc_id: str, 
+              text: str, 
+              embedding_fn: Callable[[List[str]], List[List[float]]] = None,
+              alpha: float = None,
+              beta: float = None,
+              base_token_size: int = 512,
+              **params) -> List[Chunk]:
+        """
+        Split text using hybrid semantic-statistical analysis.
+        
+        Args:
+            doc_id: Document identifier  
+            text: Input text
+            embedding_fn: Function to generate embeddings for sentences
+            alpha: Override semantic weight
+            beta: Override statistical weight
+            base_token_size: Target chunk size
+        
+        Returns:
+            List of Chunk objects
+        """
+        alpha = alpha if alpha is not None else self.alpha
+        beta = beta if beta is not None else self.beta
+        
+        sentences = split_sentences(text)
+        if len(sentences) <= 1:
+            return [Chunk(id=f"{doc_id}#hss#0", doc_id=doc_id, text=text, meta={"chunk_index": 0})]
+
+        if embedding_fn is None:
+            from .sentence_aware import SentenceAwareChunker
+            return SentenceAwareChunker().chunk(doc_id, text, base_token_size=base_token_size)
+
+        # 1. Get embeddings
+        embeddings = np.array(embedding_fn(sentences))
+        
+        # 2. Calculate per-sentence metrics
+        sent_lengths = [count_tokens(s) for s in sentences]
+        avg_length = np.mean(sent_lengths)
+        std_length = np.std(sent_lengths) if len(sent_lengths) > 1 else 1.0
+        
+        # 3. Calculate windowed semantic distances (Vectorized)
+        n = len(embeddings)
+        semantic_distances = []
+        
+        # Pre-calculate norms for efficient similarity calculation
+        norms = np.linalg.norm(embeddings, axis=1)
+        norms[norms < 1e-9] = 1e-9
+        
+        for i in range(n - 1):
+            start_prev = max(0, i - self.window_size + 1)
+            end_prev = i + 1
+            start_next = i + 1  
+            end_next = min(n, i + 1 + self.window_size)
+            
+            vec_prev = np.mean(embeddings[start_prev:end_prev], axis=0)
+            vec_next = np.mean(embeddings[start_next:end_next], axis=0)
+            
+            norm_p = np.linalg.norm(vec_prev)
+            norm_n = np.linalg.norm(vec_next)
+            
+            if norm_p < 1e-9 or norm_n < 1e-9:
+                dist = 0.0
+            else:
+                sim = np.dot(vec_prev, vec_next) / (norm_p * norm_n)
+                dist = float(1 - sim)
+            semantic_distances.append(dist)
+        
+        # 4. Calculate boundary scores (Vectorized Signals)
+        semantic_signals = np.array(semantic_distances) if semantic_distances else np.zeros(n-1)
+        
+        # Vectorize cumulative token pressure
+        cumulative_tokens = np.cumsum(sent_lengths)[:-1]
+        token_pressures = np.minimum(1.0, (cumulative_tokens / base_token_size) ** 2)
+        
+        # Vectorize length anomaly signals
+        length_zs = np.abs(np.array(sent_lengths[:-1]) - avg_length) / (std_length + 1e-6)
+        length_signals = np.minimum(1.0, length_zs / 3)
+        
+        # Combined statistical signal
+        stat_signals = 0.7 * token_pressures + 0.3 * length_signals
+        
+        # Vectorized combined boundary scores
+        combined_scores = (alpha * semantic_signals) + (beta * stat_signals)
+        
+        # Prepare score_info for the split loop (legacy compatibility with split-logic)
+        boundary_scores = []
+        for i in range(n - 1):
+            boundary_scores.append({
+                "position": i,
+                "semantic": float(semantic_signals[i]),
+                "statistical": float(stat_signals[i]),
+                "combined": float(combined_scores[i]),
+                "running_tokens": int(cumulative_tokens[i])
+            })
+        
+        # 5. Determine adaptive threshold
+        if boundary_scores:
+            all_combined = [b["combined"] for b in boundary_scores]
+            threshold = np.percentile(all_combined, self.threshold_percentile * 100)
+        else:
+            threshold = 0.5
+        
+        # 6. Build chunks using detected boundaries
+        chunks = []
+        curr_sentences = [sentences[0]]
+        curr_tokens = sent_lengths[0]
+        
+        for i, score_info in enumerate(boundary_scores):
+            should_split = False
+            split_reason = "none"
+            
+            # Semantic+Statistical split
+            if score_info["combined"] >= threshold:
+                should_split = True
+                split_reason = "hybrid"
+            
+            # Safety split (hard token limit)
+            next_sent_tokens = sent_lengths[i + 1] if i + 1 < len(sent_lengths) else 0
+            if curr_tokens + next_sent_tokens > base_token_size * 1.3:
+                should_split = True
+                split_reason = "safety"
+            
+            if should_split and curr_sentences:
+                chunk_text = " ".join(curr_sentences)
+                chunks.append(Chunk(
+                    id=f"{doc_id}#hss#{len(chunks)}",
+                    doc_id=doc_id,
+                    text=chunk_text,
+                    meta={
+                        "chunk_index": len(chunks),
+                        "strategy": "hybrid_semantic_stat",
+                        "split_reason": split_reason,
+                        "boundary_score": score_info["combined"],
+                        "token_count": count_tokens(chunk_text)
+                    }
+                ))
+                curr_sentences = []
+                curr_tokens = 0
+            
+            # Add next sentence to buffer
+            if i + 1 < len(sentences):
+                curr_sentences.append(sentences[i + 1])
+                curr_tokens += sent_lengths[i + 1]
+        
+        # Final chunk
+        if curr_sentences:
+            chunk_text = " ".join(curr_sentences)
+            chunks.append(Chunk(
+                id=f"{doc_id}#hss#{len(chunks)}",
+                doc_id=doc_id,
+                text=chunk_text,
+                meta={
+                    "chunk_index": len(chunks),
+                    "strategy": "hybrid_semantic_stat",
+                    "split_reason": "final",
+                    "token_count": count_tokens(chunk_text)
+                }
+            ))
+        
+        return chunks

autochunk/chunkers/layout_aware.py

@@ -0,0 +1,192 @@
+
+from __future__ import annotations
+from typing import List, Dict, Any
+import re
+from .base import BaseChunker, Chunk
+from ..utils.text import count_tokens, extract_code_blocks
+
+class LayoutAwareChunker(BaseChunker):
+    """
+     Document Chunker with Structure Preservation.
+    
+    BEST-OF-BREED FEATURES:
+    1. Table Inheritance: Re-attaches table headers to split table fragments.
+    2. Header Lineage: Prepends [Section: X > Y] for retrieval context.
+    3. Code Block Integrity: Never splits inside fenced code blocks.
+    4. List Awareness: Keeps list items together when possible.
+    5. Multi-Format: Handles Markdown, HTML tables, and plain text.
+    """
+    name = "layout_aware"
+
+    def __init__(self, 
+                 prepend_lineage: bool = True,
+                 preserve_code_blocks: bool = True,
+                 preserve_tables: bool = True):
+        """
+        Initialize the chunker.
+        
+        Args:
+            prepend_lineage: If True, prepend section hierarchy to chunk text
+            preserve_code_blocks: If True, avoid splitting inside ``` blocks
+            preserve_tables: If True, re-attach table headers to fragments
+        """
+        self.prepend_lineage = prepend_lineage
+        self.preserve_code_blocks = preserve_code_blocks
+        self.preserve_tables = preserve_tables
+
+    def chunk(self, doc_id: str, text: str, base_token_size: int = 512, **params) -> List[Chunk]:
+        """
+        Split text while respecting document structure.
+        
+        Args:
+            doc_id: Document identifier
+            text: Input text (Markdown preferred)
+            base_token_size: Target chunk size in tokens
+        
+        Returns:
+            List of Chunk objects with structural metadata
+        """
+        # Extract structural elements
+        code_blocks = extract_code_blocks(text) if self.preserve_code_blocks else []
+        
+        lines = text.split("\n")
+        chunks = []
+        buffer = []
+        buffer_tokens = 0
+        
+        # State tracking
+        header_stack = []  # Current header hierarchy
+        table_header = None  # Current table header row
+        table_separator = None  # Table separator row |---|
+        in_code_block = False
+        code_block_buffer = []
+        
+        for line_idx, line in enumerate(lines):
+            stripped = line.strip()
+            line_tokens = count_tokens(line)
+            
+            # Track fenced code blocks
+            if stripped.startswith("```"):
+                if not in_code_block:
+                    # Starting a code block
+                    in_code_block = True
+                    code_block_buffer = [line]
+                    continue
+                else:
+                    # Ending a code block
+                    in_code_block = False
+                    code_block_buffer.append(line)
+                    code_block_text = "\n".join(code_block_buffer)
+                    code_block_tokens = count_tokens(code_block_text)
+                    
+                    # Flush buffer if adding code block would overflow
+                    if buffer_tokens + code_block_tokens > base_token_size and buffer:
+                        chunks.append(self._make_chunk(doc_id, buffer, len(chunks), header_stack))
+                        buffer = []
+                        buffer_tokens = 0
+                    
+                    buffer.append(code_block_text)
+                    buffer_tokens += code_block_tokens
+                    code_block_buffer = []
+                    continue
+            
+            if in_code_block:
+                code_block_buffer.append(line)
+                continue
+            
+            # Skip empty lines but preserve them in buffer
+            if not stripped:
+                if buffer:
+                    buffer.append("")
+                continue
+            
+            # Header detection - update lineage
+            if stripped.startswith("#"):
+                # Count header level
+                match = re.match(r'^(#+)\s+(.+)$', stripped)
+                if match:
+                    level = len(match.group(1))
+                    title = match.group(2).strip()
+                    
+                    # Trim stack to parent level and add new header
+                    header_stack = header_stack[:level-1]
+                    header_stack.append(title)
+                    
+                    # Hard break on new header
+                    if buffer:
+                        chunks.append(self._make_chunk(doc_id, buffer, len(chunks), header_stack[:-1]))
+                        buffer = []
+                        buffer_tokens = 0
+            
+            # Table detection
+            is_table_row = "|" in stripped and not stripped.startswith("```")
+            is_separator_row = is_table_row and re.match(r'^[\|\s\-:]+$', stripped)
+            
+            if is_table_row:
+                if table_header is None and not is_separator_row:
+                    table_header = line
+                elif is_separator_row:
+                    table_separator = line
+            elif table_header:
+                # Exiting table
+                table_header = None
+                table_separator = None
+            
+            # Assembly logic
+            if buffer_tokens + line_tokens > base_token_size and buffer:
+                chunks.append(self._make_chunk(doc_id, buffer, len(chunks), header_stack))
+                buffer = []
+                buffer_tokens = 0
+                
+                # Table inheritance: add header to new chunk
+                if is_table_row and self.preserve_tables and table_header and line != table_header:
+                    buffer.append(table_header)
+                    buffer_tokens += count_tokens(table_header)
+                    if table_separator:
+                        buffer.append(table_separator)
+                        buffer_tokens += count_tokens(table_separator)
+            
+            buffer.append(line)
+            buffer_tokens += line_tokens
+        
+        # Handle remaining code block if unclosed
+        if code_block_buffer:
+            code_block_text = "\n".join(code_block_buffer)
+            buffer.append(code_block_text)
+            buffer_tokens += count_tokens(code_block_text)
+        
+        # Final chunk
+        if buffer:
+            chunks.append(self._make_chunk(doc_id, buffer, len(chunks), header_stack))
+
+        return chunks
+
+    def _make_chunk(self, doc_id: str, lines: List[str], index: int, lineage: List[str]) -> Chunk:
+        """Create a chunk with proper formatting and metadata."""
+        # Clean up empty lines at start/end
+        while lines and not lines[0].strip():
+            lines.pop(0)
+        while lines and not lines[-1].strip():
+            lines.pop()
+        
+        body_text = "\n".join(lines)
+        
+        # Prepend lineage for improved retrieval
+        if self.prepend_lineage and lineage:
+            lineage_str = " > ".join(lineage)
+            final_text = f"[Section: {lineage_str}]\n{body_text}"
+        else:
+            final_text = body_text
+        
+        return Chunk(
+            id=f"{doc_id}#la#{index}",
+            doc_id=doc_id,
+            text=final_text,
+            meta={
+                "chunk_index": index,
+                "strategy": "layout_aware",
+                "lineage": lineage,
+                "lineage_str": " > ".join(lineage) if lineage else "",
+                "token_count": count_tokens(final_text)
+            }
+        )