autochunks 0.0.8__py3-none-any.whl

autochunk/chunkers/recursive_character.py

@@ -0,0 +1,215 @@
+
+from __future__ import annotations
+from typing import List, Callable, Optional, Pattern
+import re
+from .base import BaseChunker, Chunk
+from ..utils.text import count_tokens, get_tokens, decode_tokens
+
+class RecursiveCharacterChunker(BaseChunker):
+    """
+     Recursive Character Chunker with Tiered Separators.
+    
+    BEST-OF-BREED FEATURES:
+    1. Regex Separator Support: Use regex patterns via `is_separator_regex=True`.
+    2. Keep Separator Mode: Preserves delimiters at chunk boundaries.
+    3. Start Index Tracking: Records character offset for citation purposes.
+    4. Adaptive Fallback: Falls back to token-split when separators are exhausted.
+    5. Code Block Awareness: Avoids splitting inside fenced code blocks.
+    """
+    name = "recursive_character"
+
+    # Default separator hierarchy (most significant first)
+    DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", "! ", "? ", "; ", ", ", " ", ""]
+
+    def __init__(self, 
+                 separators: List[str] = None,
+                 is_separator_regex: bool = False,
+                 keep_separator: bool = True,
+                 tokenizer: str = "auto"):
+        """
+        Initialize the chunker.
+        
+        Args:
+            separators: List of separators in priority order
+            is_separator_regex: If True, treat separators as regex patterns
+            keep_separator: If True, include separator in the chunk that precedes it
+            tokenizer: Backend for tokenization
+        """
+        self.separators = separators or self.DEFAULT_SEPARATORS
+        self.is_separator_regex = is_separator_regex
+        self.keep_separator = keep_separator
+        self.tokenizer = tokenizer
+
+    def chunk(self, 
+              doc_id: str, 
+              text: str, 
+              base_token_size: int = 512, 
+              overlap: int = 64,
+              add_start_index: bool = False,
+              respect_code_blocks: bool = True,
+              **params) -> List[Chunk]:
+        """
+        Recursively split text using separator hierarchy.
+        
+        Args:
+            doc_id: Document identifier
+            text: Input text
+            base_token_size: Target chunk size in tokens
+            overlap: Number of tokens to overlap between chunks
+            add_start_index: If True, record character start position
+            respect_code_blocks: If True, avoid splitting inside ``` blocks
+        
+        Returns:
+            List of Chunk objects
+        """
+        separators = params.get("separators", self.separators)
+        
+        # Extract code blocks if needed
+        code_block_ranges = []
+        if respect_code_blocks:
+            from ..utils.text import extract_code_blocks
+            code_blocks = extract_code_blocks(text)
+            code_block_ranges = [(b["start"], b["end"]) for b in code_blocks]
+
+        def _is_in_code_block(pos: int) -> bool:
+            for start, end in code_block_ranges:
+                if start <= pos < end:
+                    return True
+            return False
+
+        def _split_with_separator(input_text: str, separator: str, is_regex: bool) -> List[str]:
+            """Split text while optionally keeping the separator."""
+            if is_regex:
+                pattern = separator
+            else:
+                pattern = re.escape(separator)
+            
+            if self.keep_separator:
+                # Use capturing group to keep separator
+                raw_splits = re.split(f"({pattern})", input_text)
+                # Merge separator with preceding text
+                splits = []
+                for i in range(0, len(raw_splits) - 1, 2):
+                    combined = raw_splits[i] + raw_splits[i + 1]
+                    if combined:
+                        splits.append(combined)
+                if len(raw_splits) % 2 == 1 and raw_splits[-1]:
+                    splits.append(raw_splits[-1])
+                return splits
+            else:
+                return [s for s in re.split(pattern, input_text) if s]
+
+        def _recursive_split(input_text: str, seps: List[str], char_offset: int = 0) -> List[tuple]:
+            """
+            Recursively split text.
+            Returns list of (text, start_char_index) tuples.
+            """
+            token_count = count_tokens(input_text, tokenizer=self.tokenizer)
+            
+            # Base case: fits in one chunk
+            if token_count <= base_token_size:
+                return [(input_text, char_offset)]
+            
+            # No more separators: fallback to token splitting
+            if not seps:
+                all_tokens = get_tokens(input_text, tokenizer=self.tokenizer)
+                results = []
+                step = max(1, base_token_size - overlap)
+                pos = 0
+                curr_char = char_offset
+                
+                while pos < len(all_tokens):
+                    window = all_tokens[pos : pos + base_token_size]
+                    chunk_text = decode_tokens(window)
+                    results.append((chunk_text, curr_char))
+                    
+                    stepped = all_tokens[pos : pos + step]
+                    curr_char += len(decode_tokens(stepped))
+                    pos += step
+                    
+                    if pos >= len(all_tokens):
+                        break
+                
+                return results
+
+            # Try current separator
+            curr_sep = seps[0]
+            remaining_seps = seps[1:]
+            
+            splits = _split_with_separator(input_text, curr_sep, self.is_separator_regex)
+            
+            # If separator didn't help, try next
+            if len(splits) <= 1:
+                return _recursive_split(input_text, remaining_seps, char_offset)
+            
+            # Merge splits into chunks
+            final_chunks = []
+            buffer = []
+            buffer_tokens = 0
+            buffer_start = char_offset
+            current_char = char_offset
+
+            for split_text in splits:
+                split_tokens = count_tokens(split_text, tokenizer=self.tokenizer)
+                
+                # If a single split is too big, recurse deeper
+                if split_tokens > base_token_size:
+                    # Flush buffer first
+                    if buffer:
+                        final_chunks.append(("".join(buffer), buffer_start))
+                        buffer = []
+                        buffer_tokens = 0
+                    
+                    # Recurse on the large split
+                    sub_chunks = _recursive_split(split_text, remaining_seps, current_char)
+                    final_chunks.extend(sub_chunks)
+                    current_char += len(split_text)
+                    buffer_start = current_char
+                    continue
+
+                # Would this overflow the buffer?
+                if buffer_tokens + split_tokens > base_token_size and buffer:
+                    final_chunks.append(("".join(buffer), buffer_start))
+                    
+                    # Handle overlap: keep last N tokens worth of text
+                    overlap_buffer = []
+                    overlap_tokens = 0
+                    for prev in reversed(buffer):
+                        prev_tokens = count_tokens(prev, tokenizer=self.tokenizer)
+                        if overlap_tokens + prev_tokens <= overlap:
+                            overlap_buffer.insert(0, prev)
+                            overlap_tokens += prev_tokens
+                        else:
+                            break
+                    
+                    buffer = overlap_buffer
+                    buffer_tokens = overlap_tokens
+                    # Adjust start position for overlap
+                    buffer_start = current_char - len("".join(overlap_buffer))
+                
+                buffer.append(split_text)
+                buffer_tokens += split_tokens
+                current_char += len(split_text)
+
+            if buffer:
+                final_chunks.append(("".join(buffer), buffer_start))
+                
+            return final_chunks
+
+        # Execute recursive pipeline
+        raw_chunks = _recursive_split(text, separators, 0)
+        
+        # Wrap in Chunk objects
+        return [
+            Chunk(
+                id=f"{doc_id}#rc#{i}",
+                doc_id=doc_id,
+                text=chunk_text.strip(),
+                meta={
+                    "chunk_index": i, 
+                    "strategy": "recursive_character",
+                    "token_count": count_tokens(chunk_text, tokenizer=self.tokenizer),
+                    **({"start_index": start_idx} if add_start_index else {})
+                }
+            ) for i, (chunk_text, start_idx) in enumerate(raw_chunks) if chunk_text.strip()
+        ]

autochunk/chunkers/semantic_local.py

@@ -0,0 +1,140 @@
+
+from __future__ import annotations
+import numpy as np
+from typing import List, Any, Callable
+from .base import BaseChunker, Chunk
+from ..utils.text import split_sentences, count_tokens
+
+class SemanticLocalChunker(BaseChunker):
+    """
+     Semantic Chunker using Window-based Gradient Similarity.
+    Detects topic shifts by comparing sliding windows of sentence embeddings.
+    
+    BEST-OF-BREED IMPROVEMENTS:
+    1. Windowed-Similarity: Instead of comparing adjacent sentences, it compares the 
+       "semantic momentum" of previous and future windows. This suppresses noise from 
+       short/outlier sentences.
+    2. Dynamic Percentile: Uses a robust percentile-based peak detection for boundaries.
+    3. Multi-Factor Safety: Combines semantic drift with strict token caps to ensure LLM 
+       context-window compliance.
+    """
+    name = "semantic_local"
+
+    def chunk(self, 
+              doc_id: str, 
+              text: str, 
+              embedding_fn: Callable[[List[str]], List[List[float]]] = None,
+              threshold_percentile: float = 0.9, 
+              window_size: int = 3,
+              **params) -> List[Chunk]:
+        
+        sentences = split_sentences(text)
+        if len(sentences) <= 1:
+            return [Chunk(id=f"{doc_id}#sl#0", doc_id=doc_id, text=text, meta={"chunk_index": 0})]
+
+        if embedding_fn is None:
+            # Fallback to SentenceAware if no embeddings
+            from .sentence_aware import SentenceAwareChunker
+            return SentenceAwareChunker().chunk(doc_id, text, **params)
+
+        # 1. Vectorize all sentences -style
+        embeddings = np.array(embedding_fn(sentences))
+        n = len(embeddings)
+        
+        # 2. Calculate Windowed Distances (Vectorized Gradient)
+        # We use a moving average window to calculate semantic momentum
+        def moving_average(a, n=3):
+            ret = np.cumsum(a, axis=0)
+            ret[n:] = ret[n:] - ret[:-n]
+            return ret[n - 1:] / n
+
+        # Pre-calculate windowed means (shifted to align with gaps between sentences)
+        # For a prompt comparison between windows [i-w+1:i+1] and [i+1:i+w+1]
+        
+        # We can do this efficiently by calculating all possible window means once
+        # and then slicing them.
+        
+        distances = []
+        # Fallback to loop if document is very short, otherwise vectorized logic
+        # For simplicity and robustness with edge cases (min/max window), 
+        # we'll keep the loop but optimize the inside with pre-calculated norms
+        # but in a real-world scenario we'd use np.convolve for a pure vector path.
+        
+        norms = np.linalg.norm(embeddings, axis=1)
+        # Avoid division by zero
+        norms[norms == 0] = 1e-9
+        norm_embeddings = embeddings / norms[:, np.newaxis]
+        
+        for i in range(n - 1):
+            start_prev = max(0, i - window_size + 1)
+            end_prev = i + 1
+            start_next = i + 1
+            end_next = min(n, i + 1 + window_size)
+            
+            # Use raw embeddings for mean to preserve magnitude signal if desired,
+            # or normalized embeddings for pure cosine.  usually uses raw mean then normalize.
+            vec_prev = np.mean(embeddings[start_prev:end_prev], axis=0)
+            vec_next = np.mean(embeddings[start_next:end_next], axis=0)
+            
+            norm_p = np.linalg.norm(vec_prev)
+            norm_n = np.linalg.norm(vec_next)
+            
+            if norm_p < 1e-9 or norm_n < 1e-9:
+                distances.append(0.0)
+            else:
+                sim = np.dot(vec_prev, vec_next) / (norm_p * norm_n)
+                distances.append(float(1 - sim))
+        
+        # 3. Peak Detection
+        if not distances:
+             return [Chunk(id=f"{doc_id}#sl#0", doc_id=doc_id, text=text, meta={"chunk_index": 0})]
+             
+        breakpoint_threshold = np.percentile(distances, threshold_percentile * 100)
+        
+        # 4. Greedy Assembly with Safety Caps
+        safety_max_tokens = params.get("base_token_size", 512) * 2 # Usually 2x the target size is a good semantic ceiling
+        if "safety_max_tokens" in params:
+            safety_max_tokens = params["safety_max_tokens"]
+
+        chunks = []
+        curr_buffer = []
+        curr_tokens = 0
+        
+        for i, sentence in enumerate(sentences):
+            sent_tokens = count_tokens(sentence)
+            
+            # Decide to split BEFORE adding the sentence?
+            is_semantic_split = False
+            if i > 0 and i-1 < len(distances):
+                if distances[i-1] >= breakpoint_threshold and distances[i-1] > 0:
+                    is_semantic_split = True
+            
+            # Safety Split: Don't let semantic chunks grow into monsters
+            is_safety_split = curr_tokens + sent_tokens > safety_max_tokens
+            
+            if (is_semantic_split or is_safety_split) and curr_buffer:
+                chunks.append(self._make_chunk(doc_id, curr_buffer, len(chunks), "safety" if is_safety_split else "semantic"))
+                curr_buffer = []
+                curr_tokens = 0
+            
+            curr_buffer.append(sentence)
+            curr_tokens += sent_tokens
+            
+        if curr_buffer:
+            chunks.append(self._make_chunk(doc_id, curr_buffer, len(chunks), "final"))
+            
+        return chunks
+
+    def _make_chunk(self, doc_id: str, buffer: List[str], index: int, split_reason: str) -> Chunk:
+        text = " ".join(buffer).strip()
+        return Chunk(
+            id=f"{doc_id}#sl#{index}",
+            doc_id=doc_id,
+            text=text,
+            meta={
+                "chunk_index": index, 
+                "strategy": "semantic_local",
+                "split_reason": split_reason,
+                "token_count": count_tokens(text)
+            }
+        )

autochunk/chunkers/sentence_aware.py

@@ -0,0 +1,102 @@
+
+from __future__ import annotations
+from typing import List, Any
+from .base import BaseChunker, Chunk
+from ..utils.text import split_sentences, count_tokens
+
+class SentenceAwareChunker(BaseChunker):
+    """
+     Sentence-Aware Chunker with Look-back Overlap.
+    Groups sentences while respecting token limits and providing context continuity.
+    
+    BEST-OF-BREED IMPROVEMENTS:
+    1. Sentence Overlap: Repeating the last N sentences for transition context.
+    2. Intelligent Oversize Handling: Uses recursive splitting for "monster sentences" 
+       instead of crude fixed-length fallback.
+    3. NLTK Integration: Leverages the updated  sentence splitter.
+    """
+    name = "sentence_aware"
+
+    def chunk(self, doc_id: str, text: str, base_token_size: int = 512, overlap: int = 64, **params) -> List[Chunk]:
+        sentences = split_sentences(text)
+        chunks = []
+        current_buffer = []
+        current_tokens = 0
+        idx = 0
+        
+        # We define overlap in sentences if possible, or tokens
+        # Standard  is to use token-based sentence overlap
+        
+        for s in sentences:
+            sent_tokens = count_tokens(s)
+            
+            # 1. Handle "Monster Sentences" (Single sentence > limit)
+            if sent_tokens > base_token_size:
+                # If we have stuff in buffer, flush first
+                if current_buffer:
+                    chunks.append(self._make_chunk(doc_id, current_buffer, idx))
+                    idx += 1
+                    # Prepare overlap from the end of the buffer
+                    current_buffer, current_tokens = self._get_overlap(current_buffer, overlap)
+                
+                # Use recursive logic for this giant sentence
+                from .recursive_character import RecursiveCharacterChunker
+                sub_chunker = RecursiveCharacterChunker()
+                sub_chunks = sub_chunker.chunk(f"{doc_id}_monster", s, base_token_size=base_token_size, overlap=overlap)
+                
+                for sc in sub_chunks:
+                    chunks.append(Chunk(
+                        id=f"{doc_id}#sa#{idx}",
+                        doc_id=doc_id,
+                        text=sc.text,
+                        meta={"chunk_index": idx, "strategy": "sentence_aware_recursive"}
+                    ))
+                    idx += 1
+                
+                # Reset buffer after a monster sentence break
+                current_buffer = []
+                current_tokens = 0
+                continue
+
+            # 2. Regular Accumulation
+            if current_tokens + sent_tokens > base_token_size:
+                # Flush
+                chunks.append(self._make_chunk(doc_id, current_buffer, idx))
+                idx += 1
+                
+                # Context Overlap (Sentence-Level)
+                current_buffer, current_tokens = self._get_overlap(current_buffer, overlap)
+
+            current_buffer.append(s)
+            current_tokens += sent_tokens
+            
+        if current_buffer:
+            chunks.append(self._make_chunk(doc_id, current_buffer, idx))
+            
+        return chunks
+
+    def _get_overlap(self, buffer: List[str], overlap_limit: int) -> tuple[List[str], int]:
+        """Calculates the suffix of sentences to carry over for overlap."""
+        overlap_buffer = []
+        overlap_tokens = 0
+        for s in reversed(buffer):
+            t = count_tokens(s)
+            if overlap_tokens + t <= overlap_limit:
+                overlap_buffer.insert(0, s)
+                overlap_tokens += t
+            else:
+                break
+        return overlap_buffer, overlap_tokens
+
+    def _make_chunk(self, doc_id: str, buffer: List[str], index: int) -> Chunk:
+        text = " ".join(buffer).strip()
+        return Chunk(
+            id=f"{doc_id}#sa#{index}", 
+            doc_id=doc_id, 
+            text=text, 
+            meta={
+                "chunk_index": index, 
+                "strategy": "sentence_aware",
+                "token_count": count_tokens(text)
+            }
+        )

autochunk/cli.py

@@ -0,0 +1,135 @@
+
+import argparse
+import sys
+import os
+import json
+import yaml
+from typing import List, Optional
+
+from .autochunker import AutoChunker
+from .config import AutoChunkConfig, EvalConfig, ProxyConfig, NetworkConfig, RagasConfig
+from .storage.plan import Plan
+from .utils.logger import logger
+
+def cmd_optimize(args):
+    """Run the optimization search from CLI."""
+    logger.info(f"Starting CLI Optimization on: {args.docs}")
+    
+    cfg = AutoChunkConfig(
+        mode=args.mode,
+        embedding_provider=args.embedding_provider,
+        embedding_model_or_path=args.embedding_model,
+        cache_dir=args.cache_dir
+    )
+    
+    # Optional override for proxy
+    if args.no_proxy:
+        cfg.proxy_config.enabled = False
+    
+    chunker = AutoChunker(
+        mode=cfg.mode,
+        eval_config=EvalConfig(objective=args.objective),
+        embedding_provider=cfg.embedding_provider,
+        embedding_model_or_path=cfg.embedding_model_or_path,
+        cache_dir=cfg.cache_dir
+    )
+    
+    # Enable RAGAS if flag is set
+    if args.analyze_ragas:
+        chunker.cfg.ragas_config.enabled = True
+        if hasattr(args, 'ragas_llm_provider') and args.ragas_llm_provider:
+            chunker.cfg.ragas_config.llm_provider = args.ragas_llm_provider
+        if hasattr(args, 'ragas_llm_model') and args.ragas_llm_model:
+            chunker.cfg.ragas_config.llm_model = args.ragas_llm_model
+        logger.info(f"RAGAS enabled. LLM Provider: {chunker.cfg.ragas_config.llm_provider}")
+    
+    def on_progress(msg, step=None):
+        prefix = f"[Stage {step}] " if step else ""
+        logger.info(f"{prefix}{msg}")
+
+    plan, report = chunker.optimize(
+        documents=args.docs,
+        on_progress=on_progress
+    )
+    
+    # Save the plan
+    Plan.write(args.out, plan)
+    logger.success(f"Optimization complete! Winning plan saved to: {args.out}")
+    
+    if args.report:
+        with open(args.report, 'w') as f:
+            json.dump(report, f, indent=2)
+        logger.info(f"Full metrics report saved to: {args.report}")
+
+def cmd_apply(args):
+    """Apply a winning plan to a corpus."""
+    logger.info(f"Applying plan {args.plan} to docs: {args.docs}")
+    
+    plan = Plan.read(args.plan)
+    
+    # Initialize chunker with plan's embedding settings
+    chunker = AutoChunker(
+        embedding_provider=plan.embedding.get("name"),
+        embedding_model_or_path=plan.embedding.get("model")
+    )
+    
+    chunks = plan.apply(args.docs, chunker)
+    
+    with open(args.out, 'w', encoding='utf-8') as f:
+        json.dump(chunks, f, ensure_ascii=False, indent=2)
+    
+    logger.success(f"Successfully created {len(chunks)} chunks. Selection saved to: {args.out}")
+
+def cmd_serve(args):
+    """Launch the Dashboard."""
+    import uvicorn
+    from .web.server import app
+    logger.info(f"Launching AutoChunks Dashboard on http://{args.host}:{args.port}")
+    uvicorn.run(app, host=args.host, port=args.port)
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="AutoChunks: Autonomous Retrieval Optimization for RAG",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Command to execute")
+
+    # Optimize command
+    opt_p = subparsers.add_parser("optimize", help="Search for the best chunking strategy")
+    opt_p.add_argument("--docs", required=True, help="Path to document folder")
+    opt_p.add_argument("--mode", default="light", choices=["light", "full"], help="Evaluation Depth: Controls synthetic QA sampling density")
+    opt_p.add_argument("--objective", default="balanced", choices=["balanced", "quality", "cost", "latency", "metric_only"], help="Optimization objective")
+    opt_p.add_argument("--embedding-provider", default="hashing", help="Embedding provider")
+    opt_p.add_argument("--embedding-model", default="BAAI/bge-small-en-v1.5", help="Model name or path")
+    opt_p.add_argument("--out", default="best_plan.yaml", help="Output path for the winning plan")
+    opt_p.add_argument("--report", help="Output path for the full JSON metrics report")
+    opt_p.add_argument("--cache-dir", default=".ac_cache", help="Cache directory")
+    opt_p.add_argument("--no-proxy", action="store_true", help="Disable representative sampling (process all docs)")
+    opt_p.add_argument("--analyze-ragas", action="store_true", help="Enable RAGAS LLM-based evaluation metrics (Context Precision/Recall)")
+    opt_p.add_argument("--ragas-llm-provider", default="auto", choices=["auto", "openai", "ollama", "huggingface"], help="LLM provider for RAGAS (auto detects OpenAI key or Ollama)")
+    opt_p.add_argument("--ragas-llm-model", default=None, help="Model name for RAGAS LLM (e.g., gpt-4o-mini, llama3.2)")
+
+    # Apply command
+    app_p = subparsers.add_parser("apply", help="Execute a saved plan on a corpus")
+    app_p.add_argument("--plan", required=True, help="Path to the .yaml plan file")
+    app_p.add_argument("--docs", required=True, help="Path to documents to chunk")
+    app_p.add_argument("--out", default="chunks.json", help="Output path for processed chunks")
+
+    # Serve command
+    srv_p = subparsers.add_parser("serve", help="Start the Web Dashboard")
+    srv_p.add_argument("--host", default="0.0.0.0", help="Host address")
+    srv_p.add_argument("--port", type=int, default=8000, help="Port number")
+
+    args = parser.parse_args()
+
+    if args.command == "optimize":
+        cmd_optimize(args)
+    elif args.command == "apply":
+        cmd_apply(args)
+    elif args.command == "serve":
+        cmd_serve(args)
+    else:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()

autochunk/config.py

@@ -0,0 +1,76 @@
+
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Callable, Optional, List, Dict, Any
+
+@dataclass
+class EvalConfig:
+    metrics: List[str] = field(default_factory=lambda: ["mrr@10", "ndcg@10", "recall@50"]) 
+    objective: str = "balanced"  # quality|cost|latency|balanced
+    k: int = 10
+    latency_target_ms: int = 250
+    cost_budget_usd: float = 5.0
+    parent_child_eval: bool = False
+
+@dataclass
+class RetrievalStrategy:
+    type: str = "standard"  # or parent_child
+    child_token_size: int = 128
+    parent_token_size: int = 1024
+
+@dataclass
+class ProxyConfig:
+    enabled: bool = False
+    cluster_k: int = 5
+    proxy_percent: int = 10
+    verify_percent: int = 20
+
+@dataclass
+class RagasConfig:
+    enabled: bool = False
+    metrics: List[str] = field(default_factory=lambda: ["context_precision", "context_recall"])
+    sample_size: int = 20  # Limit RAGAS to a subset to save costs
+    llm_provider: str = "openai"  # openai|ollama|huggingface
+    llm_model: Optional[str] = None  # Model name (e.g., "gpt-4o-mini", "llama3.2", "microsoft/Phi-3-mini-4k-instruct")
+    api_key: Optional[str] = field(default=None, repr=False) # API Key for OpenAI/Cloud providers
+
+@dataclass
+class SafetyConstraints:
+    max_chunks_per_doc: int = 5000
+    min_avg_chunk_tokens: int = 120
+    max_redundant_overlap_ratio: float = 0.35
+
+@dataclass
+class ParallelConfig:
+    embedding_concurrency: int = 4
+    retriever_concurrency: int = 4
+    batch_size: int = 32
+
+@dataclass
+class NetworkConfig:
+    proxy_url: Optional[str] = None
+    local_models_path: Optional[str] = None
+    trusted_orgs: List[str] = field(default_factory=lambda: ["ds4sd", "RapidAI", "BAAI", "sentence-transformers"])
+
+@dataclass
+class TokenizerConfig:
+    name: str = "whitespace"
+    vocab_source: str = "custom"
+
+@dataclass
+class AutoChunkConfig:
+    eval_config: EvalConfig = field(default_factory=EvalConfig)
+    retrieval_strategy: RetrievalStrategy = field(default_factory=RetrievalStrategy)
+    proxy_config: ProxyConfig = field(default_factory=ProxyConfig)
+    ragas_config: RagasConfig = field(default_factory=RagasConfig)
+    safety: SafetyConstraints = field(default_factory=SafetyConstraints)
+    parallel: ParallelConfig = field(default_factory=ParallelConfig)
+    tokenizer: TokenizerConfig = field(default_factory=TokenizerConfig)
+    network: NetworkConfig = field(default_factory=NetworkConfig)
+    embedding_provider: str = "hashing"  # hashing|local|tei|openai
+    embedding_model_or_path: str = "BAAI/bge-small-en-v1.5"
+    embedding_api_key: Optional[str] = field(default=None, repr=False) # API Key for cloud embedding providers
+    mode: str = "light"  # light|full|incremental
+    cache_dir: str = ".ac_cache"
+    telemetry_enabled: bool = False  # Enable Tracing (Arize Phoenix)
+    metadata_enrichment: Dict[str, Any] = field(default_factory=dict)