autochunks 0.0.8__py3-none-any.whl

autochunk/__init__.py

@@ -0,0 +1,9 @@
+import warnings
+# Suppress Pydantic v2 namespace conflicts common in docling models
+warnings.filterwarnings("ignore", message='.*conflict with protected namespace "model_".*', category=UserWarning)
+
+from .autochunker import AutoChunker
+from .embedding.adapter import EmbeddingFn
+from .config import AutoChunkConfig, EvalConfig, ProxyConfig, RetrievalStrategy, SafetyConstraints, ParallelConfig, TokenizerConfig, NetworkConfig, RagasConfig
+from .adapters import AutoChunkLangChainAdapter, AutoChunkLlamaIndexAdapter, AutoChunkHaystackAdapter
+from .storage.plan import Plan

autochunk/__main__.py

@@ -0,0 +1,5 @@
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

autochunk/adapters/__init__.py

@@ -0,0 +1,3 @@
+from .langchain import AutoChunkLangChainAdapter
+from .llamaindex import AutoChunkLlamaIndexAdapter
+from .haystack import AutoChunkHaystackAdapter

autochunk/adapters/haystack.py

@@ -0,0 +1,68 @@
+
+from __future__ import annotations
+from typing import List, Dict, Any, Optional, Union
+from ..storage.plan import Plan
+from ..autochunker import AutoChunker
+
+try:
+    from haystack import component, Document
+    HAYSTACK_AVAILABLE = True
+except ImportError:
+    # Robust fallback for environment without Haystack
+    def component(cls): return cls
+    def output_types(**kwargs):
+        def decorator(func): return func
+        return decorator
+    component.output_types = output_types
+    class Document: pass
+    HAYSTACK_AVAILABLE = False
+
+@component
+class AutoChunkHaystackAdapter:
+    """
+    Official AutoChunks Adapter for Haystack 2.0.
+    Acts as a Pipeline Component for optimized document splitting.
+    """
+    def __init__(self, plan: Union[Plan, str]):
+        if isinstance(plan, str):
+            self.plan = Plan.read(plan)
+        else:
+            self.plan = plan
+        
+        # Initialize internal engine
+        self.chunker = AutoChunker(
+            embedding_provider=self.plan.embedding.get("name"),
+            embedding_model_or_path=self.plan.embedding.get("model")
+        )
+
+    @component.output_types(documents=List[Document])
+    def run(self, documents: List[Document]):
+        """
+        Implementation of the Haystack Component interface.
+        """
+        if not HAYSTACK_AVAILABLE:
+            raise ImportError("Please install haystack-ai: pip install haystack-ai")
+
+        # Convert Haystack docs to AutoChunks format
+        ac_docs = []
+        for d in documents:
+            ac_docs.append({
+                "id": str(getattr(d, "id", hash(d.content))),
+                "text": d.content,
+                "metadata": d.meta
+            })
+
+        # Process via  pipeline
+        gen_name = self.plan.generator_pipeline.get("name")
+        params = self.plan.generator_pipeline.get("params", {})
+        ac_chunks = self.chunker.apply_with_generator(ac_docs, gen_name, params)
+
+        # Re-wrap as Haystack Documents
+        return {
+            "documents": [
+                Document(
+                    content=ch["text"],
+                    meta={**ch.get("meta", {}), "autochunk_plan_id": self.plan.id}
+                ) for ch in ac_chunks
+            ]
+        }

autochunk/adapters/langchain.py

@@ -0,0 +1,81 @@
+
+from __future__ import annotations
+from typing import List, Dict, Any, TYPE_CHECKING, Union
+from ..storage.plan import Plan
+from ..autochunker import AutoChunker, AutoChunkConfig
+
+if TYPE_CHECKING:
+    from langchain_core.documents import Document
+
+try:
+    from langchain_core.documents import BaseDocumentTransformer, Document
+    LANGCHAIN_AVAILABLE = True
+except ImportError:
+    class BaseDocumentTransformer: pass
+    LANGCHAIN_AVAILABLE = False
+
+class AutoChunkLangChainAdapter(BaseDocumentTransformer):
+    """
+    Official AutoChunks Adapter for LangChain.
+    Inherits from BaseDocumentTransformer for seamless integration 
+    into LangChain Indexing and LCEL pipelines.
+    """
+    def __init__(self, plan: Union[Plan, str], config: AutoChunkConfig = None):
+        if isinstance(plan, str):
+            self.plan = Plan.read(plan)
+        else:
+            self.plan = plan
+
+        # We use a configured AutoChunker to execute the plan
+        self.chunker = AutoChunker(
+            embedding_provider=self.plan.embedding.get("name"),
+            embedding_model_or_path=self.plan.embedding.get("model")
+        )
+
+    def transform_documents(self, documents: List[Document], **kwargs: Any) -> List[Document]:
+        """
+        Apply the optimized AutoChunks plan to a list of LangChain documents.
+        This processes ALL documents provided.
+        """
+        try:
+            from langchain_core.documents import Document
+        except ImportError:
+            raise ImportError("Please install langchain-core: pip install langchain-core")
+
+        # Convert LangChain docs to AutoChunks format
+        ac_docs = []
+        for d in documents:
+            # We use metadata.get('source', id(d)) as a unique doc_id
+            doc_id = str(d.metadata.get("source", id(d)))
+            ac_docs.append({
+                "id": doc_id,
+                "text": d.page_content,
+                "metadata": d.metadata
+            })
+
+        # Run the execution pipeline
+        gen_name = self.plan.generator_pipeline.get("name")
+        params = self.plan.generator_pipeline.get("params", {})
+        
+        ac_chunks = self.chunker.apply_with_generator(ac_docs, gen_name, params)
+
+        # Convert back to LangChain docs
+        lc_docs = []
+        for ch in ac_chunks:
+            # Preserve original metadata and add chunking metadata
+            meta = ch.get("meta", {}).copy()
+            # If original metadata was passed through, it might be nested or direct
+            # For now, we assume simple merger
+            lc_docs.append(Document(
+                page_content=ch["text"],
+                metadata={**meta, "autochunk_plan_id": self.plan.id}
+            ))
+            
+        return lc_docs
+
+    def split_documents(self, documents: List[Document]) -> List[Document]:
+        """Alias for transform_documents to match TextSplitter interface."""
+        return self.transform_documents(documents)
+
+    def __call__(self, documents: List[Document]) -> List[Document]:
+        return self.transform_documents(documents)

autochunk/adapters/llamaindex.py

@@ -0,0 +1,94 @@
+
+from __future__ import annotations
+from typing import List, Dict, Any, TYPE_CHECKING, Union
+from ..storage.plan import Plan
+from ..autochunker import AutoChunker
+
+if TYPE_CHECKING:
+    from llama_index.core.schema import BaseNode, Document
+
+try:
+    from llama_index.core.node_parser import NodeParser, BaseNodeParser
+    from llama_index.core.schema import TextNode, BaseNode, Document
+    LLAMA_INDEX_AVAILABLE = True
+except ImportError:
+    class BaseNodeParser: pass
+    LLAMA_INDEX_AVAILABLE = False
+
+class AutoChunkLlamaIndexAdapter(BaseNodeParser):
+    """
+    Official AutoChunks Adapter for LlamaIndex.
+    Acts as a native NodeParser for seamless integration into IngestionPipelines.
+    """
+    def __init__(self, plan: Union[Plan, str]):
+        if isinstance(plan, str):
+            self.plan = Plan.read(plan)
+        else:
+            self.plan = plan
+
+        self.chunker = AutoChunker(
+            embedding_provider=self.plan.embedding.get("name"),
+            embedding_model_or_path=self.plan.embedding.get("model")
+        )
+
+    def _parse_nodes(self, nodes: List[BaseNode], show_progress: bool = False, **kwargs: Any) -> List[BaseNode]:
+        """
+        Internal implementation for LlamaIndex BaseNodeParser.
+        """
+        # Convert Nodes to AutoChunks format
+        ac_docs = []
+        for n in nodes:
+            ac_docs.append({
+                "id": n.node_id,
+                "text": n.get_content(),
+                "metadata": n.metadata
+            })
+        
+        # Run the execution pipeline
+        gen_name = self.plan.generator_pipeline.get("name")
+        params = self.plan.generator_pipeline.get("params", {})
+        
+        ac_chunks = self.chunker.apply_with_generator(ac_docs, gen_name, params)
+
+        # Convert back to LlamaIndex Nodes
+        final_nodes = []
+        for ch in ac_chunks:
+            node = TextNode(
+                text=ch["text"],
+                metadata={**ch.get("meta", {}), "autochunk_plan_id": self.plan.id}
+            )
+            final_nodes.append(node)
+            
+        return final_nodes
+
+    def get_nodes_from_documents(self, documents: List[Document], **kwargs: Any) -> List[BaseNode]:
+        try:
+            from llama_index.core.schema import TextNode
+        except ImportError:
+            raise ImportError("Please install llama-index-core: pip install llama-index-core")
+
+        # Convert LlamaIndex docs to AutoChunks format
+        ac_docs = []
+        for d in documents:
+            ac_docs.append({
+                "id": d.doc_id,
+                "text": d.get_content(),
+                "metadata": d.metadata
+            })
+
+        # Run the execution pipeline
+        gen_name = self.plan.generator_pipeline.get("name")
+        params = self.plan.generator_pipeline.get("params", {})
+        
+        ac_chunks = self.chunker.apply_with_generator(ac_docs, gen_name, params)
+
+        # Convert back to LlamaIndex Nodes
+        nodes = []
+        for ch in ac_chunks:
+            node = TextNode(
+                text=ch["text"],
+                metadata={**ch.get("meta", {}), "autochunk_plan_id": self.plan.id}
+            )
+            nodes.append(node)
+            
+        return nodes