autochunks 0.0.8__py3-none-any.whl

autochunk/chunkers/__init__.py

@@ -0,0 +1,100 @@
+"""
+AutoChunks - World-Class Text Chunking Library
+
+The definitive Swiss Army Knife of text splitting with extreme precision.
+"""
+
+from .base import BaseChunker, Chunk
+
+# ============================================================================
+# BASIC SPLITTERS
+# ============================================================================
+from .fixed_length import FixedLengthChunker
+from .recursive_character import RecursiveCharacterChunker
+from .sentence_aware import SentenceAwareChunker
+
+# ============================================================================
+# SEMANTIC SPLITTERS
+# ============================================================================
+from .semantic_local import SemanticLocalChunker
+from .hybrid_semantic_stat import HybridSemanticStatChunker
+from .proposition import PropositionChunker
+from .agentic import AgenticChunker
+
+# ============================================================================
+# STRUCTURE-AWARE SPLITTERS
+# ============================================================================
+from .layout_aware import LayoutAwareChunker
+from .parent_child import ParentChildChunker
+from .contextual_retrieval import ContextualRetrievalChunker
+from .html_section import HTMLSectionChunker
+
+# ============================================================================
+# CODE SPLITTERS
+# ============================================================================
+from .python_ast import PythonASTChunker
+
+# ============================================================================
+# CHUNKER REGISTRY
+# ============================================================================
+CHUNKER_REGISTRY = {
+    # Basic
+    'fixed_length': FixedLengthChunker,
+    'recursive_character': RecursiveCharacterChunker,
+    'sentence_aware': SentenceAwareChunker,
+    
+    # Semantic
+    'semantic_local': SemanticLocalChunker,
+    'hybrid_semantic_stat': HybridSemanticStatChunker,
+    'proposition': PropositionChunker,
+    'agentic': AgenticChunker,
+    
+    # Structure-Aware
+    'layout_aware': LayoutAwareChunker,
+    'parent_child': ParentChildChunker,
+    'contextual_retrieval': ContextualRetrievalChunker,
+    'html_section': HTMLSectionChunker,
+    
+    # Code
+    'python_ast': PythonASTChunker,
+}
+
+def get_chunker(name: str) -> BaseChunker:
+    """Get a chunker instance by name."""
+    if name not in CHUNKER_REGISTRY:
+        raise ValueError(f"Unknown chunker: {name}. Available: {list(CHUNKER_REGISTRY.keys())}")
+    return CHUNKER_REGISTRY[name]()
+
+def list_chunkers() -> list:
+    """List all available chunker names."""
+    return list(CHUNKER_REGISTRY.keys())
+
+__all__ = [
+    # Base
+    'BaseChunker',
+    'Chunk',
+    
+    # Basic
+    'FixedLengthChunker',
+    'RecursiveCharacterChunker',
+    'SentenceAwareChunker',
+    
+    # Semantic
+    'SemanticLocalChunker',
+    'HybridSemanticStatChunker',
+    'PropositionChunker',
+    'AgenticChunker',
+    
+    # Structure-Aware
+    'LayoutAwareChunker',
+    'ParentChildChunker',
+    'ContextualRetrievalChunker',
+    
+    # Code
+    'PythonASTChunker',
+    
+    # Utilities
+    'CHUNKER_REGISTRY',
+    'get_chunker',
+    'list_chunkers',
+]

autochunk/chunkers/agentic.py

@@ -0,0 +1,184 @@
+
+from __future__ import annotations
+from typing import List, Callable, Optional, Any
+from .base import BaseChunker, Chunk
+from ..utils.text import count_tokens, split_sentences
+
+class AgenticChunker(BaseChunker):
+    """
+    LLM-Powered Agentic Chunker for Intelligent Boundary Detection.
+    
+    Uses a language model to decide optimal chunk boundaries based on
+    semantic coherence, topic shifts, and content structure.
+    
+    BEST-OF-BREED FEATURES:
+    1. LLM-Decided Boundaries: Model determines where to split based on meaning.
+    2. Configurable Prompts: Custom instructions for domain-specific chunking.
+    3. Fallback Safety: Reverts to sentence-aware if LLM unavailable.
+    4. Batch Processing: Efficient API usage with batched boundary decisions.
+    
+    Reference: Greg Kamradt's "Agentic Chunker" concept.
+    """
+    name = "agentic"
+
+    DEFAULT_SYSTEM_PROMPT = """You are a text segmentation expert. Your task is to identify natural boundaries in text where the topic, theme, or focus shifts significantly.
+
+For each boundary you identify, respond with the sentence number (1-indexed) where a NEW section should begin.
+
+Guidelines:
+- A new section should start when there's a clear topic shift
+- Keep related information together
+- Aim for chunks of roughly 3-10 sentences
+- Don't split in the middle of a logical argument or explanation
+- Consider paragraph breaks as potential (but not mandatory) boundaries"""
+
+    DEFAULT_USER_TEMPLATE = """Analyze the following text and identify where natural section boundaries should occur.
+
+TEXT:
+{text}
+
+SENTENCES (numbered):
+{numbered_sentences}
+
+Respond with a JSON array of sentence numbers where new sections should BEGIN.
+Example: [1, 5, 12, 18] means sections start at sentences 1, 5, 12, and 18.
+
+Only output the JSON array, nothing else."""
+
+    def __init__(self,
+                 llm_fn: Callable[[str, str], str] = None,
+                 system_prompt: str = None,
+                 user_template: str = None,
+                 max_sentences_per_call: int = 50):
+        """
+        Initialize the agentic chunker.
+        
+        Args:
+            llm_fn: Function that takes (system_prompt, user_message) and returns LLM response.
+                    If None, uses a mock that falls back to sentence-aware chunking.
+            system_prompt: Custom system prompt for the LLM
+            user_template: Custom user message template (must include {text} and {numbered_sentences})
+            max_sentences_per_call: Max sentences to process in one LLM call
+        """
+        self.llm_fn = llm_fn
+        self.system_prompt = system_prompt or self.DEFAULT_SYSTEM_PROMPT
+        self.user_template = user_template or self.DEFAULT_USER_TEMPLATE
+        self.max_sentences_per_call = max_sentences_per_call
+
+    def chunk(self,
+              doc_id: str,
+              text: str,
+              base_token_size: int = 512,
+              **params) -> List[Chunk]:
+        """
+        Use LLM to determine optimal chunk boundaries.
+        
+        Args:
+            doc_id: Document identifier
+            text: Input text
+            base_token_size: Target chunk size (used as guidance for LLM)
+        
+        Returns:
+            List of Chunk objects
+        """
+        sentences = split_sentences(text)
+        
+        if len(sentences) <= 1:
+            return [Chunk(id=f"{doc_id}#ag#0", doc_id=doc_id, text=text, 
+                         meta={"chunk_index": 0, "strategy": "agentic"})]
+        
+        # Fallback if no LLM function provided
+        if self.llm_fn is None:
+            from .sentence_aware import SentenceAwareChunker
+            return SentenceAwareChunker().chunk(doc_id, text, base_token_size=base_token_size)
+        
+        # Get boundary decisions from LLM
+        boundaries = self._get_boundaries(sentences)
+        
+        # Build chunks from boundaries
+        chunks = []
+        current_start = 0
+        
+        for boundary in sorted(set(boundaries)):
+            if boundary > current_start and boundary <= len(sentences):
+                chunk_sentences = sentences[current_start:boundary]
+                chunk_text = " ".join(chunk_sentences)
+                
+                chunks.append(Chunk(
+                    id=f"{doc_id}#ag#{len(chunks)}",
+                    doc_id=doc_id,
+                    text=chunk_text,
+                    meta={
+                        "chunk_index": len(chunks),
+                        "strategy": "agentic",
+                        "sentence_range": [current_start, boundary],
+                        "token_count": count_tokens(chunk_text)
+                    }
+                ))
+                current_start = boundary
+        
+        # Add final chunk
+        if current_start < len(sentences):
+            chunk_sentences = sentences[current_start:]
+            chunk_text = " ".join(chunk_sentences)
+            chunks.append(Chunk(
+                id=f"{doc_id}#ag#{len(chunks)}",
+                doc_id=doc_id,
+                text=chunk_text,
+                meta={
+                    "chunk_index": len(chunks),
+                    "strategy": "agentic",
+                    "sentence_range": [current_start, len(sentences)],
+                    "token_count": count_tokens(chunk_text)
+                }
+            ))
+        
+        return chunks if chunks else [Chunk(id=f"{doc_id}#ag#0", doc_id=doc_id, text=text,
+                                            meta={"chunk_index": 0, "strategy": "agentic"})]
+
+    def _get_boundaries(self, sentences: List[str]) -> List[int]:
+        """Get boundary positions from LLM."""
+        import json
+        
+        # Always include position 0 as first boundary
+        all_boundaries = [0]
+        
+        # Process in batches if needed
+        for batch_start in range(0, len(sentences), self.max_sentences_per_call):
+            batch_end = min(batch_start + self.max_sentences_per_call, len(sentences))
+            batch_sentences = sentences[batch_start:batch_end]
+            
+            # Create numbered list
+            numbered = "\n".join([f"{i+1}. {s}" for i, s in enumerate(batch_sentences)])
+            batch_text = " ".join(batch_sentences)
+            
+            user_message = self.user_template.format(
+                text=batch_text,
+                numbered_sentences=numbered
+            )
+            
+            try:
+                response = self.llm_fn(self.system_prompt, user_message)
+                
+                # Parse JSON response
+                # Handle various response formats
+                response = response.strip()
+                if response.startswith("```"):
+                    response = response.split("```")[1]
+                    if response.startswith("json"):
+                        response = response[4:]
+                
+                boundaries = json.loads(response)
+                
+                # Adjust for batch offset and add to all boundaries
+                for b in boundaries:
+                    if isinstance(b, int) and b > 0:
+                        all_boundaries.append(batch_start + b - 1)  # Convert 1-indexed to 0-indexed
+                        
+            except Exception as e:
+                # On parse failure, use sentence-aware heuristic for this batch
+                # Split roughly every 5 sentences
+                for i in range(5, len(batch_sentences), 5):
+                    all_boundaries.append(batch_start + i)
+        
+        return sorted(set(all_boundaries))

autochunk/chunkers/base.py

@@ -0,0 +1,16 @@
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import List, Dict, Any
+
+@dataclass
+class Chunk:
+    id: str
+    doc_id: str
+    text: str
+    meta: Dict[str, Any]
+
+class BaseChunker:
+    name = "base"
+    def chunk(self, doc_id: str, text: str, **params) -> List[Chunk]:
+        raise NotImplementedError

autochunk/chunkers/contextual_retrieval.py

@@ -0,0 +1,151 @@
+
+from __future__ import annotations
+from typing import List, Callable, Optional
+from .base import BaseChunker, Chunk
+from ..utils.text import count_tokens
+
+class ContextualRetrievalChunker(BaseChunker):
+    """
+    Contextual Retrieval Chunker (Anthropic's Approach).
+    
+    Each chunk is prepended with LLM-generated context that situates it
+    within the broader document. This dramatically improves retrieval accuracy.
+    
+    BEST-OF-BREED FEATURES:
+    1. Context Prepending: Each chunk starts with situating context.
+    2. Document-Aware: Context is generated with full document visibility.
+    3. Retrieval-Optimized: Context is designed to improve semantic search.
+    4. Caching: Efficient context reuse for repeated chunks.
+    
+    Reference: Anthropic's "Contextual Retrieval" blog post.
+    """
+    name = "contextual_retrieval"
+
+    DEFAULT_CONTEXT_PROMPT = """<document>
+{document}
+</document>
+
+Here is the chunk we want to situate within the whole document:
+<chunk>
+{chunk}
+</chunk>
+
+Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else."""
+
+    def __init__(self,
+                 llm_fn: Callable[[str, str], str] = None,
+                 base_chunker: BaseChunker = None,
+                 context_template: str = None,
+                 max_document_tokens: int = 8000,
+                 context_prefix: str = "Context: "):
+        """
+        Initialize the contextual retrieval chunker.
+        
+        Args:
+            llm_fn: Function that takes (system_prompt, user_message) and returns LLM response.
+            base_chunker: Chunker to use for initial splitting. Defaults to RecursiveCharacter.
+            context_template: Custom template for context generation.
+            max_document_tokens: Max tokens of document to include in context generation.
+            context_prefix: Prefix before the generated context.
+        """
+        self.llm_fn = llm_fn
+        self.base_chunker = base_chunker
+        self.context_template = context_template or self.DEFAULT_CONTEXT_PROMPT
+        self.max_document_tokens = max_document_tokens
+        self.context_prefix = context_prefix
+
+    def chunk(self,
+              doc_id: str,
+              text: str,
+              base_token_size: int = 512,
+              generate_context: bool = True,
+              **params) -> List[Chunk]:
+        """
+        Create chunks with contextual headers.
+        
+        Args:
+            doc_id: Document identifier
+            text: Input text
+            base_token_size: Target chunk size
+            generate_context: If False, skip context generation (for testing)
+        
+        Returns:
+            List of Chunk objects with context prepended
+        """
+        # Get base chunker
+        if self.base_chunker is None:
+            from .recursive_character import RecursiveCharacterChunker
+            base_chunker = RecursiveCharacterChunker()
+        else:
+            base_chunker = self.base_chunker
+        
+        # Create initial chunks
+        base_chunks = base_chunker.chunk(doc_id, text, base_token_size=base_token_size, **params)
+        
+        if not generate_context or self.llm_fn is None:
+            # Return chunks with simple document context from lineage if available
+            for chunk in base_chunks:
+                chunk.meta["strategy"] = "contextual_retrieval_base"
+            return base_chunks
+        
+        # Truncate document for context generation
+        from ..utils.text import get_tokens, decode_tokens
+        doc_tokens = get_tokens(text)
+        if len(doc_tokens) > self.max_document_tokens:
+            truncated_doc = decode_tokens(doc_tokens[:self.max_document_tokens]) + "\n... [truncated]"
+        else:
+            truncated_doc = text
+        
+        # Generate context for each chunk
+        contextualized_chunks = []
+        
+        for chunk in base_chunks:
+            context = self._generate_context(truncated_doc, chunk.text)
+            
+            # Prepend context to chunk text
+            if context:
+                contextualized_text = f"{self.context_prefix}{context}\n\n{chunk.text}"
+            else:
+                contextualized_text = chunk.text
+            
+            contextualized_chunks.append(Chunk(
+                id=f"{doc_id}#cr#{len(contextualized_chunks)}",
+                doc_id=doc_id,
+                text=contextualized_text,
+                meta={
+                    "chunk_index": len(contextualized_chunks),
+                    "strategy": "contextual_retrieval",
+                    "original_text": chunk.text,
+                    "generated_context": context,
+                    "token_count": count_tokens(contextualized_text),
+                    "original_token_count": count_tokens(chunk.text)
+                }
+            ))
+        
+        return contextualized_chunks
+
+    def _generate_context(self, document: str, chunk: str) -> str:
+        """Generate situating context for a chunk."""
+        try:
+            # Use the context template
+            prompt = self.context_template.format(
+                document=document,
+                chunk=chunk
+            )
+            
+            # Call LLM with empty system prompt (all in user message)
+            response = self.llm_fn("", prompt)
+            
+            # Clean response
+            context = response.strip()
+            
+            # Limit context length
+            if count_tokens(context) > 100:
+                from ..utils.text import get_tokens, decode_tokens
+                tokens = get_tokens(context)[:100]
+                context = decode_tokens(tokens)
+            
+            return context
+            
+        except Exception as e:
+            return ""

autochunk/chunkers/fixed_length.py

@@ -0,0 +1,110 @@
+
+from __future__ import annotations
+from typing import List, Callable, Optional
+from .base import BaseChunker, Chunk
+
+class FixedLengthChunker(BaseChunker):
+    """
+     Fixed-Length Chunker with Sliding Window (Overlap).
+    
+    BEST-OF-BREED FEATURES:
+    1. Pluggable Length Function: Supports tiktoken, char, word, or custom functions.
+    2. Start Index Tracking: Records character offset for citation purposes.
+    3. Accurate Token Counting: Uses tiktoken by default for GPT-model accuracy.
+    """
+    name = "fixed_length"
+
+    def __init__(self, 
+                 length_function: Callable[[str], int] = None,
+                 tokenizer: str = "auto"):
+        """
+        Initialize the chunker.
+        
+        Args:
+            length_function: Custom function to measure text length. 
+                             If None, uses token counting.
+            tokenizer: Backend for tokenization ("auto", "tiktoken", "whitespace", "character")
+        """
+        self.tokenizer = tokenizer
+        self._length_function = length_function
+
+    def _get_length(self, text: str) -> int:
+        """Get length using configured method."""
+        if self._length_function:
+            return self._length_function(text)
+        from ..utils.text import count_tokens
+        return count_tokens(text, tokenizer=self.tokenizer)
+
+    def chunk(self, 
+              doc_id: str, 
+              text: str, 
+              base_token_size: int = 512, 
+              overlap: int = 64,
+              add_start_index: bool = False,
+              **params) -> List[Chunk]:
+        """
+        Split text into fixed-size chunks with overlap.
+        
+        Args:
+            doc_id: Document identifier
+            text: Input text
+            base_token_size: Target chunk size in tokens
+            overlap: Number of tokens to overlap between chunks
+            add_start_index: If True, record character start position in metadata
+        
+        Returns:
+            List of Chunk objects
+        """
+        from ..utils.text import get_tokens, decode_tokens, count_tokens
+        
+        if not text:
+            return []
+        
+        # Get tokens for splitting
+        all_tokens = get_tokens(text, tokenizer=self.tokenizer)
+        if not all_tokens:
+            return []
+
+        chunks = []
+        idx = 0
+        token_pos = 0
+        char_pos = 0  # Track character position for start_index
+        
+        while token_pos < len(all_tokens):
+            # Take a window of tokens
+            window_tokens = all_tokens[token_pos : token_pos + base_token_size]
+            chunk_text = decode_tokens(window_tokens)
+            
+            # Build metadata
+            meta = {
+                "chunk_index": idx, 
+                "strategy": "fixed_length",
+                "token_count": len(window_tokens)
+            }
+            
+            if add_start_index:
+                meta["start_index"] = char_pos
+            
+            chunks.append(Chunk(
+                id=f"{doc_id}#fl#{idx}", 
+                doc_id=doc_id, 
+                text=chunk_text, 
+                meta=meta
+            ))
+            
+            idx += 1
+            
+            # Step size = window - overlap (move forward)
+            step = max(1, base_token_size - overlap)
+            
+            # Update character position (for start_index tracking)
+            stepped_tokens = all_tokens[token_pos : token_pos + step]
+            char_pos += len(decode_tokens(stepped_tokens))
+            
+            token_pos += step
+            
+            # Boundary safety
+            if token_pos >= len(all_tokens):
+                break
+            
+        return chunks