autochunks 0.0.8__py3-none-any.whl

autochunk/utils/io.py

@@ -0,0 +1,176 @@
+import os, hashlib
+from typing import List, Dict, Optional, Callable
+from .logger import logger
+from .hashing import sha256_hex
+
+SUPPORTED_EXTS = {".txt", ".md", ".pdf", ".html", ".htm"}
+
+def read_text_file(path: str) -> str:
+    with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+        return f.read()
+
+def _load_pdf_markdown(path: str, on_progress: Optional[Callable[[str], None]] = None) -> str:
+    """Progressive  PDF to Markdown extraction for AutoChunks native chunkers."""
+    import fitz
+    doc = fitz.open(path)
+    total_pages = len(doc)
+    
+    if on_progress: 
+        on_progress(f"Analyzing structure of {total_pages} pages...")
+        logger.info(f"Deep Analysis started for {os.path.basename(path)} ({total_pages} pages)")
+
+    full_text = []
+    
+    try:
+        import pymupdf4llm
+        # Process in chunks of 5 pages for high-frequency real-time progress
+        chunk_size = 5
+        for i in range(0, total_pages, chunk_size):
+            end_page = min(i + chunk_size, total_pages)
+            if on_progress: 
+                on_progress(f"Extracting layout (Pages {i+1}-{end_page} of {total_pages})...")
+            
+            # Extract specific page range
+            page_text = pymupdf4llm.to_markdown(path, pages=list(range(i, end_page)))
+            full_text.append(page_text)
+            
+        return "\n".join(full_text)
+        
+    except Exception as e:
+        logger.warning(f" extraction failed, falling back to basic text: {e}")
+        return _load_pdf_raw(path, on_progress)
+
+def _load_pdf_raw(path: str, on_progress: Optional[Callable[[str], None]] = None) -> str:
+    """Raw text extraction from PDF for fair bridge comparison."""
+    import fitz
+    doc = fitz.open(path)
+    total_pages = len(doc)
+    if on_progress: on_progress(f"Fast extraction: {total_pages} pages...")
+    return "\n".join([page.get_text() for page in doc])
+
+def _load_html_processed(path: str) -> str:
+    """Clean HTML to plain text using BeautifulSoup - for AutoChunks native chunkers."""
+    try:
+        from bs4 import BeautifulSoup
+        content = read_text_file(path)
+        soup = BeautifulSoup(content, "lxml")
+        
+        # Remove script and style elements
+        for script in soup(["script", "style"]):
+            script.decompose()
+            
+        # Get text
+        text = soup.get_text(separator=' ')
+        
+        # Break into lines and remove leading and trailing whitespace
+        lines = (line.strip() for line in text.splitlines())
+        # Break multi-headlines into a line each
+        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+        # Drop blank lines
+        text = '\n'.join(chunk for chunk in chunks if chunk)
+        return text
+    except Exception as e:
+        logger.warning(f"HTML parsing failed for {path}: {e}")
+        return read_text_file(path)
+
+def _load_html_raw(path: str) -> str:
+    """Raw HTML text for LangChain HTML splitter."""
+    return read_text_file(path)
+
+def load_documents(root: str, on_progress: Optional[Callable[[str], None]] = None, high_fidelity: bool = True) -> List[Dict]:
+    """
+    Load documents with BOTH raw and processed versions for fair evaluation.
+    
+    Returns documents with:
+    - text: Processed version (for AutoChunks native chunkers)
+    - raw_text: Original/raw version (for LangChain bridges)
+    """
+    if on_progress: on_progress(f"Scanning directory: {root}")
+    docs = []
+    
+    # Text cache directory
+    cache_root = os.path.join(os.getcwd(), ".autochunk", "text_cache")
+    os.makedirs(cache_root, exist_ok=True)
+    
+    if not os.path.exists(root):
+        raise FileNotFoundError(f"Path not found: {root}")
+        
+    for dirpath, _, filenames in os.walk(root):
+        for fn in filenames:
+            ext = os.path.splitext(fn)[1].lower()
+            if ext in SUPPORTED_EXTS:
+                p = os.path.abspath(os.path.normpath(os.path.join(dirpath, fn)))
+                try:
+                    # 1. Quick Binary Hash
+                    with open(p, "rb") as bf:
+                        binary_data = bf.read()
+                        b_hash = sha256_hex(binary_data)
+                    
+                    cache_file_proc = os.path.join(cache_root, f"{b_hash}_processed.txt")
+                    cache_file_raw = os.path.join(cache_root, f"{b_hash}_raw.txt")
+                    
+                    text_processed = ""
+                    text_raw = ""
+                    
+                    # Check cache for processed text
+                    if os.path.exists(cache_file_proc) and os.path.exists(cache_file_raw):
+                        # Cache Hit!
+                        if on_progress: on_progress(f"⚡ Cache Hit: Loading {fn}...")
+                        logger.info(f"CACHE HIT: Reusing text for {fn}")
+                        with open(cache_file_proc, "r", encoding="utf-8") as tf:
+                            text_processed = tf.read()
+                        with open(cache_file_raw, "r", encoding="utf-8") as tf:
+                            text_raw = tf.read()
+                    else:
+                        # Cache Miss: Load both versions
+                        if on_progress: on_progress(f"Processing ({len(docs)+1}): {fn}...")
+                        logger.info(f"Loading document: {fn}")
+                        
+                        if ext == ".pdf":
+                            # Processed: Markdown extraction (AutoChunks advantage)
+                            if high_fidelity:
+                                text_processed = _load_pdf_markdown(p, on_progress)
+                            else:
+                                text_processed = _load_pdf_raw(p, on_progress)
+                            # Raw: Basic text extraction (fair for bridges)
+                            text_raw = _load_pdf_raw(p, None)
+                            
+                        elif ext in [".html", ".htm"]:
+                            # Processed: BeautifulSoup cleaned (for text-based chunkers)
+                            text_processed = _load_html_processed(p)
+                            # Raw: Original HTML (for LangChain HTML splitter)
+                            text_raw = _load_html_raw(p)
+                            
+                        else:
+                            # Plain text files: same for both
+                            text_processed = read_text_file(p)
+                            text_raw = text_processed
+                        
+                        # Save to cache
+                        if text_processed.strip():
+                            with open(cache_file_proc, "w", encoding="utf-8") as tf:
+                                tf.write(text_processed)
+                        if text_raw.strip():
+                            with open(cache_file_raw, "w", encoding="utf-8") as tf:
+                                tf.write(text_raw)
+                    
+                    if text_processed.strip() or text_raw.strip():
+                        docs.append({
+                            "id": p, 
+                            "path": p, 
+                            "text": text_processed,      # For AutoChunks native chunkers
+                            "raw_text": text_raw,        # For LangChain bridges
+                            "ext": ext,
+                            "hash": b_hash
+                        })
+                        logger.success(f"Loaded {fn} (processed: {len(text_processed)} chars, raw: {len(text_raw)} chars)")
+                    else:
+                        logger.warning(f"Skipping empty document: {p}")
+                except Exception as e:
+                    logger.error(f"Failed to load {p}: {e}")
+                    
+    if not docs:
+        raise RuntimeError(f"No supported documents found in {root} (supported: {SUPPORTED_EXTS})")
+        
+    logger.info(f"Loaded {len(docs)} documents from {root}")
+    return docs

autochunk/utils/logger.py

@@ -0,0 +1,64 @@
+import sys, time, logging
+from loguru import logger
+from contextvars import ContextVar
+from typing import List, Dict, Any, Optional
+
+# Global storage for jobs
+jobs_data = {}
+# ContextVar to track the job_id in the current task/thread
+current_job_id: ContextVar[Optional[str]] = ContextVar("current_job_id", default=None)
+
+# Add a Pretty Console Handler (Keep it)
+logger.remove()
+logger.add(
+    sys.stderr,
+    format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+    level="INFO"
+)
+
+# Custom sink to relay logs to the UI jobs_data
+def ui_log_sink(message):
+    record = message.record
+    # Try extra first (bind), then contextvar
+    job_id = record["extra"].get("job_id") or current_job_id.get()
+    
+    if job_id and job_id in jobs_data:
+        msg_text = record["message"]
+        level = record["level"].name
+        success = level not in ["ERROR", "CRITICAL"]
+        
+        # Format the message for the UI with its level
+        formatted_msg = f"[{level}] {msg_text}"
+        
+        entry = {
+            "time": time.strftime("%H:%M:%S"),
+            "msg": formatted_msg,
+            "success": success
+        }
+        jobs_data[job_id]["logs"].append(entry)
+
+# Redirect standard library logging to Loguru
+class PropagateHandler(logging.Handler):
+    def emit(self, record):
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        frame, depth = logging.currentframe(), 2
+        while frame and frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+def setup_library_interception():
+    logging.basicConfig(handlers=[PropagateHandler()], level=0, force=True)
+    # Target specific verbose libraries if needed
+    for name in ["pymupdf", "fitz", "pymupdf4llm"]:
+        l = logging.getLogger(name)
+        l.handlers = [PropagateHandler()]
+        l.propagate = False
+
+setup_library_interception()
+logger.add(ui_log_sink, level="INFO")

autochunk/utils/telemetry.py

@@ -0,0 +1,44 @@
+
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from .logger import logger
+
+_tracer_initialized = False
+
+def init_telemetry(endpoint: str = "http://localhost:6006/v1/traces", enabled: bool = True):
+    """
+    Initialize OpenTelemetry tracing.
+    By default, it points to a local Arize Phoenix instance.
+    """
+    global _tracer_initialized
+    if _tracer_initialized or not enabled:
+        return
+
+    try:
+        # Quick check: Is the telemetry server even there?
+        import urllib.request
+        try:
+            # We just check the base URL, not the full /v1/traces path necessarily, 
+            # but a 200 or even a 404/405 from the host is better than a connection refused.
+            base_url = endpoint.split("/v1/")[0]
+            with urllib.request.urlopen(base_url, timeout=1):
+                pass
+        except Exception:
+            # If we can't connect in 1 second, it's likely not running.
+            # We don't want to spam the user with ConnectionRefusedErrors in background threads.
+            logger.info("Telemetry server not found at localhost:6006. Skipping tracing.")
+            return
+
+        provider = TracerProvider()
+        processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))
+        provider.add_span_processor(processor)
+        trace.set_tracer_provider(provider)
+        _tracer_initialized = True
+        logger.info(f"Telemetry initialized, exporting to {endpoint}")
+    except Exception as e:
+        logger.debug(f"Failed to initialize telemetry: {e}. Running without tracing.")
+
+def get_tracer(name: str = "autochunk"):
+    return trace.get_tracer(name)

autochunk/utils/text.py

@@ -0,0 +1,199 @@
+
+import re
+from typing import List, Callable, Optional
+from functools import lru_cache
+
+# ============================================================================
+# TOKENIZATION LAYER -  Multi-Backend Support
+# ============================================================================
+
+# Global tokenizer cache for performance
+_tokenizer_cache = {}
+
+def _get_tiktoken_encoder(model: str = "cl100k_base"):
+    """Get or create a tiktoken encoder (cached)."""
+    if model not in _tokenizer_cache:
+        try:
+            import tiktoken
+            _tokenizer_cache[model] = tiktoken.get_encoding(model)
+        except ImportError:
+            return None
+    return _tokenizer_cache.get(model)
+
+def get_tokens(text: str, tokenizer: str = "auto") -> List[str]:
+    """
+    Unified tokenization across all chunkers.
+    
+    Args:
+        text: Input text to tokenize
+        tokenizer: One of "auto", "tiktoken", "whitespace", "character"
+                   "auto" tries tiktoken first, falls back to whitespace
+    
+    Returns:
+        List of token strings
+    """
+    if tokenizer == "auto":
+        # Try tiktoken first (most accurate for GPT models)
+        enc = _get_tiktoken_encoder()
+        if enc:
+            token_ids = enc.encode(text)
+            # Return token strings for compatibility
+            return [enc.decode([t]) for t in token_ids]
+        # Fallback to whitespace
+        tokenizer = "whitespace"
+    
+    if tokenizer == "tiktoken":
+        enc = _get_tiktoken_encoder()
+        if enc:
+            token_ids = enc.encode(text)
+            return [enc.decode([t]) for t in token_ids]
+        raise ImportError("tiktoken not installed. Run: pip install tiktoken")
+    
+    if tokenizer == "character":
+        return list(text)
+    
+    # Default: whitespace + punctuation splitting
+    return [t for t in re.split(r"(\s+|[.,!?;:'\"\(\)\[\]\{\}])", text) if t]
+
+def decode_tokens(tokens: List[str]) -> str:
+    """Unified detokenization."""
+    return "".join(tokens)
+
+def count_tokens(text: str, tokenizer: str = "auto") -> int:
+    """
+    Count tokens using the specified tokenizer.
+    
+    For production RAG with GPT models, use tokenizer="tiktoken".
+    """
+    if not text:
+        return 0
+    
+    if tokenizer == "auto" or tokenizer == "tiktoken":
+        enc = _get_tiktoken_encoder()
+        if enc:
+            return len(enc.encode(text))
+    
+    if tokenizer == "character":
+        return len(text)
+    
+    # Fallback to whitespace tokenization
+    return len([t for t in re.split(r"(\s+|[.,!?;:'\"\(\)\[\]\{\}])", text) if t.strip()])
+
+def create_length_function(method: str = "token") -> Callable[[str], int]:
+    """
+    Factory for length functions (LangChain compatibility).
+    
+    Args:
+        method: "token" (tiktoken), "char" (character count), "word" (word count)
+    
+    Returns:
+        A function that takes text and returns length
+    """
+    if method == "token":
+        return lambda text: count_tokens(text, tokenizer="auto")
+    elif method == "char":
+        return len
+    elif method == "word":
+        return lambda text: len(text.split())
+    else:
+        return lambda text: count_tokens(text, tokenizer="auto")
+
+# ============================================================================
+# SENTENCE SPLITTING -  Multi-Backend Support
+# ============================================================================
+
+def split_sentences(text: str, backend: str = "auto") -> List[str]:
+    """
+     Sentence Splitting with multiple backends.
+    
+    Args:
+        text: Input text
+        backend: "auto", "nltk", "spacy", "regex"
+    
+    Returns:
+        List of sentences
+    """
+    if not text or not text.strip():
+        return []
+    
+    if backend == "auto" or backend == "nltk":
+        try:
+            import nltk
+            try:
+                nltk.data.find('tokenizers/punkt_tab')
+            except LookupError:
+                try:
+                    nltk.data.find('tokenizers/punkt')
+                except LookupError:
+                    nltk.download('punkt_tab', quiet=True)
+            return nltk.sent_tokenize(text)
+        except Exception:
+            if backend == "nltk":
+                raise
+            # Fall through to regex
+    
+    if backend == "spacy":
+        try:
+            import spacy
+            nlp = spacy.blank("en")
+            nlp.add_pipe("sentencizer")
+            doc = nlp(text)
+            return [sent.text.strip() for sent in doc.sents]
+        except ImportError:
+            raise ImportError("spacy not installed. Run: pip install spacy")
+    
+    # Regex fallback - handles abbreviations better than naive split
+    # Negative lookbehinds for common abbreviations
+    pattern = r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<![A-Z]\.)(?<=[.!?])\s+'
+    parts = re.split(pattern, text.strip())
+    return [p.strip() for p in parts if p.strip()]
+
+def whitespace_tokens(text: str) -> List[str]:
+    """Keep whitespace tokens for reconstruction."""
+    return [t for t in re.split(r"(\s+)", text) if t]
+
+# ============================================================================
+# TEXT UTILITIES
+# ============================================================================
+
+def get_char_to_token_map(text: str) -> List[int]:
+    """
+    Create a mapping from character index to token index.
+    Useful for tracking start indices in chunks.
+    """
+    tokens = get_tokens(text)
+    char_map = []
+    token_idx = 0
+    char_pos = 0
+    
+    for i, token in enumerate(tokens):
+        token_len = len(token)
+        for _ in range(token_len):
+            char_map.append(i)
+        char_pos += token_len
+    
+    return char_map
+
+def extract_code_blocks(text: str) -> List[dict]:
+    """
+    Extract fenced code blocks from markdown text.
+    Returns list of {"start": int, "end": int, "content": str, "language": str}
+    """
+    pattern = r'```(\w*)\n(.*?)```'
+    blocks = []
+    for match in re.finditer(pattern, text, re.DOTALL):
+        blocks.append({
+            "start": match.start(),
+            "end": match.end(),
+            "content": match.group(2),
+            "language": match.group(1) or "text"
+        })
+    return blocks
+
+def is_inside_code_block(text: str, position: int) -> bool:
+    """Check if a character position is inside a fenced code block."""
+    blocks = extract_code_blocks(text)
+    for block in blocks:
+        if block["start"] <= position < block["end"]:
+            return True
+    return False

autochunks-0.0.8.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: autochunks
+Version: 0.0.8
+Summary: Autonomous Retrieval Optimization for RAG
+Author: Sumit Joshi
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/s8ilabs/AutoChunks
+Project-URL: Documentation, https://autochunks.readthedocs.io/
+Project-URL: Repository, https://github.com/s8ilabs/AutoChunks
+Project-URL: Issues, https://github.com/s8ilabs/AutoChunks/issues
+Keywords: rag,chunking,retrieval,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: loguru
+Requires-Dist: arize-phoenix>=4.3.0
+Requires-Dist: opentelemetry-api
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: opentelemetry-exporter-otlp
+Requires-Dist: nltk
+Requires-Dist: pymupdf4llm
+Requires-Dist: sentence-transformers
+Requires-Dist: torch
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: python-multipart
+Dynamic: license-file
+
+# AutoChunks
+### The Intelligent Data Optimization Layer for RAG Engineering
+
+[![Version](https://img.shields.io/badge/version-0.08--alpha-blue)](https://github.com/s8ilabs/AutoChunks)
+[![Documentation](https://img.shields.io/badge/docs-read--the--docs-teal)](https://autochunks.readthedocs.io/)
+[![License](https://img.shields.io/badge/license-Apache--2.0-green)](LICENSE)
+
+![AutoChunks Hero](docs/assets/hero_banner.png)
+
+AutoChunks is a specialized engine designed to eliminate the guesswork from Retrieval-Augmented Generation (RAG). By treating chunking as an optimization problem rather than a set of heuristics, it empirically discovers the most performant data structures for your specific documents and retrieval models.
+
+---
+
+## From Heuristics to Evidence
+
+Most RAG pipelines today rely on arbitrary settings—like a 512-token chunk size with a 10% overlap. These values are often chosen without validation, leading to:
+
+*   **Fragmented Context**: Related information is split across multiple retrieval units.
+*   **Semantic Noise**: Poorly defined boundaries dilute the signal-to-noise ratio in LLM prompts.
+*   **Retrieval Gaps**: Critical information hidden in "dead zones" between chunks results in recall failure.
+
+**AutoChunks replaces trial-and-error with a data-driven tournament.** It generates adversarial synthetic ground truth from your documents and pits over 15+ chunking strategies against each other to find the mathematical optimum for your corpus.
+
+---
+
+## Core Pillars
+
+### The Vectorized Tournament
+AutoChunks runs an exhaustive parallel search across multiple strategy families—Recursive, Semantic, Layout-Aware, and Hybrid. Every candidate is evaluated in a high-speed NumPy-accelerated retrieval simulation, measuring performance across hundreds of queries in seconds.
+
+### Adversarial Synthetic QA
+The system performs a structural audit of your documents to generate "needle-in-a-haystack" question-answer pairs. This ensures that your chunking strategy is optimized against real-world search intent, not just random text splits.
+
+### Optimization Goals
+Align your data engineering with your business objectives. Choose from intent-based presets that guide the engine toward specific outcomes:
+*   **Balanced Ranking**: Optimizes for general-purpose retrieval quality.
+*   **Speed and Precision**: Minimizes LLM reading time by prioritizing Rank #1 hits.
+*   **Comprehensive Retrieval**: Prioritizes recall for compliance or legal use cases.
+*   **Cost Efficiency**: Minimizes vector storage and inference costs for massive datasets.
+
+---
+
+## Advanced Feature Set
+
+*   **Hybrid Semantic-Statistical Chunker**: Uses real-time embedding distance analysis to detect topic shifts while maintaining strict token limits.
+*   **Framework Bridges**: Native adapters for LangChain, LlamaIndex, and Haystack, allowing you to benchmark and optimize your existing framework code directly.
+*   **Layout-Aware Processing**: High-fidelity extraction that respects the nested structures of PDFs, HTML sections, and Markdown hierarchies.
+*   **Fidelity Inspector**: A visual debugging dashboard to qualitatively verify how different strategies fragment complex documents.
+*   **Enterprise Security**: Air-gap compatible. Supports local model deployment, SHA-256 binary fingerprinting for data privacy, and SecretStr protection for all cloud credentials.
+
+---
+
+## Quick Start
+
+### Installation
+```bash
+pip install -r requirements.txt
+```
+*Note: For GPU acceleration with Local Embeddings or Ragas, please refer to the [Getting Started guide](docs/getting_started.md).*
+
+### Launch the Dashboard
+The most effective way to optimize your data is through the visual interactive dashboard.
+```bash
+python -m autochunk.web.server
+```
+Navigate to `http://localhost:8000` to begin your first optimization run.
+
+### Python API
+```python
+from autochunk import AutoChunker
+
+# Initialize in Light Mode for rapid iteration
+optimizer = AutoChunker(mode="light")
+
+# Discover the optimal plan for your dataset
+plan, report = optimizer.optimize(
+    documents_path="./my_data_folder",
+    objective="balanced"
+)
+
+# Apply the winning strategy
+chunks = plan.apply("./new_documents", optimizer)
+```
+
+---
+
+## Documentation and Resources
+
+*   [Getting Started](docs/getting_started.md)
+*   [The Optimization Lifecycle](docs/core_concepts/eval_flow.md)
+*   [Metric Definitions and Scoring](docs/core_concepts/evaluation.md)
+*   [RAGAS Semantic Evaluation](docs/guides/ragas_evaluation.md)
+
+---
+
+Developed for the RAG and LLM Community.
+AutoChunks is released under the Apache License 2.0.

autochunks-0.0.8.dist-info/RECORD

@@ -0,0 +1,61 @@
+autochunk/__init__.py,sha256=tu8KBFUMu7eymVuURj7np8T_3dXTvv7JQnRcJ7BoSbM,572
+autochunk/__main__.py,sha256=e_fn8GhFWu_dqslEnOKv13ueXdYWekoBFsslltAP4ZI,67
+autochunk/autochunker.py,sha256=CNwCxqym25xeHAU17T2ZJM0psXpa0MrfzpP19mhX8VQ,33759
+autochunk/cli.py,sha256=LQDf_D94vdtwpOsKYtGA_lF6FwFvfFK6jqU--PSVbQs,5632
+autochunk/config.py,sha256=2O_lPanpp3_uLDpbvsRbrbuHy4p9O3Z2CStfzezgTsw,2891
+autochunk/adapters/__init__.py,sha256=K49nSsmEy7vhlf28SzCEQQIXn6NxX2ir8M4D7XlxOGo,150
+autochunk/adapters/haystack.py,sha256=pJuFOGkdpeD6GY7QafeX39lzPOgIP4McZDoL071aUgw,2230
+autochunk/adapters/langchain.py,sha256=HQwqEeuUw64rl6Nh_UkjlB1vjF72LR4UzpULDBVThfk,3061
+autochunk/adapters/llamaindex.py,sha256=62Yfb1VylhF9Ws86h6l70Bpe3JpHqJlSnH2THRztbEg,3213
+autochunk/chunkers/__init__.py,sha256=Asb2U-lEjZyW5AbLCCPhg1xCpBMw4IXJXs6nDbus_gI,3262
+autochunk/chunkers/agentic.py,sha256=rR2Og5c3zmyNciVw_aU_AXcue4lrEtonzFy46uQEU2Y,7687
+autochunk/chunkers/base.py,sha256=MBjwgZyywHb1e8Oe5J5e1gF8_XltxnV2oKyKuZvPSpc,339
+autochunk/chunkers/contextual_retrieval.py,sha256=bxfqBmM0x2yngnZdj1Qg9oIaNSAHtRHj5RwVjEi1soY,5993
+autochunk/chunkers/fixed_length.py,sha256=PI6qcEg3XXUHnhbBP9MZHjD73meyHquJv-ljmClVgQc,3686
+autochunk/chunkers/html_section.py,sha256=rNXTCjcy3emg2mGLuICqxbGUC1Uhfq6v73WvvyUs16M,9598
+autochunk/chunkers/hybrid_semantic_stat.py,sha256=sZinkrtj08DSP9uFXjYZlJ1H2SUyaqmkYsL7LUnxgik,7779
+autochunk/chunkers/layout_aware.py,sha256=H5lhvfgsYU9-fnost-WoRFLD6O2ByMo26ZD62iN44s8,7704
+autochunk/chunkers/parent_child.py,sha256=pFDFj-b_3g77o9A4JyJ-N69ObkWUOSSVk8So-1aA8cQ,6570
+autochunk/chunkers/proposition.py,sha256=glJsZ_ysd6I6ijtuBDCRv16oWBGu1K7boGmB7Tpx368,6317
+autochunk/chunkers/python_ast.py,sha256=ekPjxhxjgAFoLSlmHqi1sdeeCMjSYnkjWgDkfjQcuFw,9563
+autochunk/chunkers/recursive_character.py,sha256=ayRrjYsTDZvaLZY5V6cwViAIqlZ_TbPrsgBDfoaMidk,8975
+autochunk/chunkers/semantic_local.py,sha256=IpYbs7UDKa-bNM5UkWzRGboSxkfGnPEujS-qdluapBY,5988
+autochunk/chunkers/sentence_aware.py,sha256=EPnhNBS_OY8kN6fCTv3Yx_oFLfHFZaw2X2boSwapjoo,4064
+autochunk/embedding/__init__.py,sha256=Ip8EGbBkbFUbjfhlcZy5jXBaI5uwq8glEuYLmk0nDN4,822
+autochunk/embedding/adapter.py,sha256=jj0v-Oc3LJSWHBgVJ-A-_4jo8Bh03ymDjURPTAwUNJw,320
+autochunk/embedding/base.py,sha256=xmcNLZeiyAYMfsddq1qh_YmmTQAoDnZ-V5phH0kZjJc,789
+autochunk/embedding/hashing.py,sha256=uSB8tYiGcd-2hlayx3NbdL9mQITMXPYW4lO6-apkMfs,1174
+autochunk/embedding/local.py,sha256=U1gj2EwSn0ycEDt1WhcSVc951pJSZehlEzds1h93WYc,7100
+autochunk/embedding/ollama.py,sha256=-POiPUUx6qvCxkXuL2hXLzXoUcs5kZMOtF1jzw-vlLQ,2421
+autochunk/embedding/openai.py,sha256=d-Dlgk1lqfikx7yb9Aig1bTuLhOCI0_eY1EDyp2aImE,2080
+autochunk/embedding/tokenizer.py,sha256=gcdS4rtoW54Y8ANhY00Ctd6wkdQ535eF8MrXQ0qF8Fs,188
+autochunk/enrichment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autochunk/enrichment/contextual.py,sha256=S6pewHKr8Z6pGsqv8qbQcOsMnb9qpAUA7_fOFxLpu8g,1143
+autochunk/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autochunk/eval/harness.py,sha256=XP2EldIbNEEeIMcHAKwbSPtjuFrm4Qn4xizFpIqp-e4,7952
+autochunk/eval/metrics.py,sha256=u2MblvFpoLZbAXC0YDOP7mO8r3eJJ9YxDJgMVBXRd0o,731
+autochunk/eval/ragas_eval.py,sha256=dBq-TRPtnpKaUC3BBaiTFMPZodmH7XCzHhCqAv7FPho,10573
+autochunk/eval/synthetic.py,sha256=ecOSh60gPyXYJ8mC_6W2CTePUNdh91zQpYQ3Qfp34-g,4036
+autochunk/quality/__init__.py,sha256=Bsfch7Icm8syUa-g-VR0z7WlUIT4MPw8bddRSDNj93U,823
+autochunk/quality/deduplicator.py,sha256=zYFQE2CJdNN8foZ5SpDJQRtgetHrIJ6D0H33_Vj7qjY,14046
+autochunk/quality/overlap_optimizer.py,sha256=jDX4w4gz57eL-z1tj8cThbT4z6tfXUq0BxprDJTFUuI,15719
+autochunk/quality/post_processor.py,sha256=NSvZ13pqIlR0erY0zs8dZxm2X1luP1njzH-x2N5LLtk,9882
+autochunk/quality/scorer.py,sha256=yK7GkT_ZO1yN6ApMpcH0QMu6vkXteZm5fxiK41rWe-E,18781
+autochunk/retrieval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autochunk/retrieval/in_memory.py,sha256=RGKGc-_C06j4PzKSFj92DhaUmU315P9K1rCGq40eYms,1682
+autochunk/retrieval/parent_child.py,sha256=SVeuVvIqiHdXk78EAqKGvKAiCRIOT0wf84TC1dEWl5w,147
+autochunk/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autochunk/storage/cache.py,sha256=O9nOHkV9C8Lsl3bUmNqqRmPLTDQus7B5IlVY1l-0EAc,1047
+autochunk/storage/plan.py,sha256=J3vf3Lqi6Zc-HLNHOINuPoYv-A3bSxbpzUCXaamdRtQ,1434
+autochunk/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autochunk/utils/hashing.py,sha256=lz4I4EPyulfc3ZUIS4X-vhZNsAwC9Ehx32NSsw5Ko6Y,178
+autochunk/utils/io.py,sha256=di2UPiplB1LWpiNpbVCC250K66w2NGPpU_M2rC3kDa4,7943
+autochunk/utils/logger.py,sha256=kzNkxWfTF6ivEE5WQJKa77RNvY2HpYCybzxgH5yan0E,2202
+autochunk/utils/telemetry.py,sha256=EdYylC-rOTF1beL84AQRLN11nn0QC4X_qwNLYVmlud8,1854
+autochunk/utils/text.py,sha256=prp0oeclNPZMHMAROorGIlthwAVjZZ8Ei0O9uywDQ1I,6474
+autochunks-0.0.8.dist-info/licenses/LICENSE,sha256=QWYXD3-1Anc9MANMS_BgZjU3rZpfeSKGwulkQVdUsn4,572
+autochunks-0.0.8.dist-info/METADATA,sha256=RsNKhIJifOI_0SmlkqITX3WAG85nw168uWbGEDKiFns,5865
+autochunks-0.0.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+autochunks-0.0.8.dist-info/entry_points.txt,sha256=h1jlX_VnUy2h8FV0Xn2lAjOaofA_ogRMWCOTYORN0eo,50
+autochunks-0.0.8.dist-info/top_level.txt,sha256=KHX0veKaI0ukg1x8Yv_5Pmjy8VI0LUH6KT8e-6KakHU,10
+autochunks-0.0.8.dist-info/RECORD,,