haiku.rag 0.4.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

haiku/rag/chunker.py

@@ -1,6 +1,11 @@
+from io import BytesIO
 from typing import ClassVar
 
 import tiktoken
+from docling.chunking import HybridChunker  # type: ignore
+from docling.document_converter import DocumentConverter
+from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
+from docling_core.types.io import DocumentStream
 
 from haiku.rag.config import Config
 
@@ -8,9 +13,11 @@ from haiku.rag.config import Config
 class Chunker:
     """A class that chunks text into smaller pieces for embedding and retrieval.
 
+    Uses docling's structure-aware chunking to create semantically meaningful chunks
+    that respect document boundaries.
+
     Args:
         chunk_size: The maximum size of a chunk in tokens.
-        chunk_overlap: The number of tokens of overlap between chunks.
     """
 
     encoder: ClassVar[tiktoken.Encoding] = tiktoken.encoding_for_model("gpt-4o")
@@ -18,50 +25,36 @@ class Chunker:
     def __init__(
         self,
         chunk_size: int = Config.CHUNK_SIZE,
-        chunk_overlap: int = Config.CHUNK_OVERLAP,
     ):
         self.chunk_size = chunk_size
-        self.chunk_overlap = chunk_overlap
+        tokenizer = OpenAITokenizer(
+            tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size
+        )
+
+        self.chunker = HybridChunker(tokenizer=tokenizer)  # type: ignore
 
     async def chunk(self, text: str) -> list[str]:
-        """Split the text into chunks based on token boundaries.
+        """Split the text into chunks using docling's structure-aware chunking.
 
         Args:
             text: The text to be split into chunks.
 
         Returns:
-            A list of text chunks with token-based boundaries and overlap.
+            A list of text chunks with semantic boundaries.
         """
         if not text:
             return []
 
-        encoded_tokens = self.encoder.encode(text, disallowed_special=())
-
-        if self.chunk_size > len(encoded_tokens):
-            return [text]
-
-        chunks = []
-        i = 0
-        split_id_counter = 0
-        while i < len(encoded_tokens):
-            # Overlap
-            start_i = i
-            end_i = min(i + self.chunk_size, len(encoded_tokens))
-
-            chunk_tokens = encoded_tokens[start_i:end_i]
-            chunk_text = self.encoder.decode(chunk_tokens)
-
-            chunks.append(chunk_text)
-            split_id_counter += 1
-
-            # Exit loop if this was the last possible chunk
-            if end_i == len(encoded_tokens):
-                break
+        # Convert to docling document
+        bytes_io = BytesIO(text.encode("utf-8"))
+        doc_stream = DocumentStream(name="text.md", stream=bytes_io)
+        converter = DocumentConverter()
+        result = converter.convert(doc_stream)
+        doc = result.document
 
-            i += (
-                self.chunk_size - self.chunk_overlap
-            )  # Step forward, considering overlap
-        return chunks
+        # Chunk using docling's hybrid chunker
+        chunks = list(self.chunker.chunk(doc))
+        return [self.chunker.contextualize(chunk) for chunk in chunks]
 
 
 chunker = Chunker()

haiku/rag/config.py

@@ -27,7 +27,6 @@ class AppConfig(BaseModel):
     QA_MODEL: str = "qwen3"
 
     CHUNK_SIZE: int = 256
-    CHUNK_OVERLAP: int = 32
 
     OLLAMA_BASE_URL: str = "http://localhost:11434"
 

haiku/rag/reader.py

@@ -16,7 +16,8 @@ class FileReader:
         ".jpeg",
         ".jpg",
         ".md",
-        ".pdf.png",
+        ".pdf",
+        ".png",
         ".pptx",
         ".tiff",
         ".xlsx",

haiku/rag/store/repositories/settings.py

@@ -63,7 +63,6 @@ class SettingsRepository:
             "EMBEDDINGS_MODEL",
             "EMBEDDINGS_VECTOR_DIM",
             "CHUNK_SIZE",
-            "CHUNK_OVERLAP",
         ]
 
         errors = []

{haiku_rag-0.4.3.dist-info → haiku_rag-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.4.3
+Version: 0.5.0
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT

{haiku_rag-0.4.3.dist-info → haiku_rag-0.5.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 haiku/rag/app.py,sha256=FpLVyP1-zAq_XPmU8CPVLkuIAeuhBOGvMqhYS8RbN40,7649
-haiku/rag/chunker.py,sha256=MbCtP66OfTFoIBvqmVT9T9c87fozsYYzAQzJJEfPBVI,1812
+haiku/rag/chunker.py,sha256=P2slbmoABygYRlqjOGzPBEOYsBZNTnNpE6bnW_dkVOE,1850
 haiku/rag/cli.py,sha256=k7EhLkvTncxsdh5TYrg8BHLYh_lfyzupsWGj1dEEdqY,5992
 haiku/rag/client.py,sha256=MZNIpMm6MS3P6vjLqiCztT2dBOM7-bZOosX5IpbHJbI,12724
-haiku/rag/config.py,sha256=_Ss54kmfxVAJupExLKaYjYUlFxJgb7hEEdbG4-isapY,1662
+haiku/rag/config.py,sha256=GXTWC3vasBMaWju-yh8Es3CidBz1ftqRH6E5PHpgsSQ,1634
 haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
 haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
 haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
-haiku/rag/reader.py,sha256=dLz3yyc5r8dzdqCc2VViC3fADpScw4lxXueKiu-cI7c,2915
+haiku/rag/reader.py,sha256=s5dinZ-WffioiRH7OWZtE2v7FHRPd1PkqpPYsXtwqtc,2927
 haiku/rag/utils.py,sha256=Ez_tvNlRO_D8c2CBZ83Hs9Gmzcqdq4cmw_V5GBdKy_8,2214
 haiku/rag/embeddings/__init__.py,sha256=yFBlxS0jBiVHl_rWz5kb43t6Ha132U1ZGdlIPfhzPdg,1491
 haiku/rag/embeddings/base.py,sha256=NTQvuzbZPu0LBo5wAu3qGyJ4xXUaRAt1fjBO0ygWn_Y,465
@@ -33,11 +33,11 @@ haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPv
 haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
 haiku/rag/store/repositories/chunk.py,sha256=UyvHhKb1ESZePoTp2GneAARdfKoocEdfPOwgWPPQ0v8,16878
 haiku/rag/store/repositories/document.py,sha256=fXIWevJaOe6x2cK4u9cQxiEGD0ntKQb9y3VRqklQypE,7920
-haiku/rag/store/repositories/settings.py,sha256=dme3_ulQdQvyF9daavSjAd-SjZ5hh0MJoxP7iXgap-A,2492
+haiku/rag/store/repositories/settings.py,sha256=qZLXvLsErnCWL0nBQQNfRnatHzCKhtUDLvUK9k-W_fU,2463
 haiku/rag/store/upgrades/__init__.py,sha256=kKS1YWT_P-CYKhKtokOLTIFNKf9jlfjFFr8lyIMeogM,100
 haiku/rag/store/upgrades/v0_3_4.py,sha256=GLogKZdZ40NX1vBHKdOJju7fFzNUCHoEnjSZg17Hm2U,663
-haiku_rag-0.4.3.dist-info/METADATA,sha256=T2ZHdGL_zd1eSfEjFolh3R_zJpuWmUhKsnNkYLKtT7E,4198
-haiku_rag-0.4.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.4.3.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.4.3.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.4.3.dist-info/RECORD,,
+haiku_rag-0.5.0.dist-info/METADATA,sha256=Z29lOzGgaD2PJ6OxZc53QuMzFdosEZCdm7HZYOUNN3M,4198
+haiku_rag-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.5.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.5.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.5.0.dist-info/RECORD,,