haiku.rag 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

haiku/rag/cli.py

@@ -1,4 +1,5 @@
 import asyncio
+from importlib.metadata import version
 from pathlib import Path
 
 import typer
@@ -26,8 +27,23 @@ async def check_version():
         console.print("[yellow]Please update.[/yellow]")
 
 
+def version_callback(value: bool):
+    if value:
+        v = version("haiku.rag")
+        console.print(f"haiku.rag version {v}")
+        raise typer.Exit()
+
+
 @cli.callback()
-def main():
+def main(
+    _version: bool = typer.Option(
+        False,
+        "-v",
+        "--version",
+        callback=version_callback,
+        help="Show version and exit",
+    ),
+):
     """haiku.rag CLI - SQLite-based RAG system"""
     # Run version check before any command
     event_loop.run_until_complete(check_version())

haiku/rag/reader.py

@@ -1,32 +1,45 @@
 from pathlib import Path
 from typing import ClassVar
 
-from markitdown import MarkItDown
+from docling.document_converter import DocumentConverter
 
 
 class FileReader:
-    extensions: ClassVar[list[str]] = [
+    # Extensions supported by docling
+    docling_extensions: ClassVar[list[str]] = [
+        ".asciidoc",
+        ".bmp",
+        ".csv",
+        ".docx",
+        ".html",
+        ".xhtml",
+        ".jpeg",
+        ".jpg",
+        ".md",
+        ".pdf.png",
+        ".pptx",
+        ".tiff",
+        ".xlsx",
+        ".xml",
+        ".webp",
+    ]
+
+    # Plain text extensions that we'll read directly
+    text_extensions: ClassVar[list[str]] = [
         ".astro",
         ".c",
         ".cpp",
         ".css",
-        ".csv",
-        ".docx",
         ".go",
         ".h",
         ".hpp",
-        ".html",
         ".java",
         ".js",
         ".json",
         ".kt",
-        ".md",
         ".mdx",
         ".mjs",
-        ".mp3",
-        ".pdf",
         ".php",
-        ".pptx",
         ".py",
         ".rb",
         ".rs",
@@ -36,17 +49,61 @@ class FileReader:
         ".tsx",
         ".txt",
         ".vue",
-        ".wav",
-        ".xml",
-        ".xlsx",
         ".yaml",
         ".yml",
     ]
 
+    # Code file extensions with their markdown language identifiers for syntax highlighting
+    code_markdown_identifier: ClassVar[dict[str, str]] = {
+        ".astro": "astro",
+        ".c": "c",
+        ".cpp": "cpp",
+        ".css": "css",
+        ".go": "go",
+        ".h": "c",
+        ".hpp": "cpp",
+        ".java": "java",
+        ".js": "javascript",
+        ".json": "json",
+        ".kt": "kotlin",
+        ".mjs": "javascript",
+        ".php": "php",
+        ".py": "python",
+        ".rb": "ruby",
+        ".rs": "rust",
+        ".svelte": "svelte",
+        ".swift": "swift",
+        ".ts": "typescript",
+        ".tsx": "tsx",
+        ".vue": "vue",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+    }
+
+    extensions: ClassVar[list[str]] = docling_extensions + text_extensions
+
     @staticmethod
     def parse_file(path: Path) -> str:
         try:
-            reader = MarkItDown()
-            return reader.convert(path).text_content
+            file_extension = path.suffix.lower()
+
+            if file_extension in FileReader.docling_extensions:
+                # Use docling for complex document formats
+                converter = DocumentConverter()
+                result = converter.convert(path)
+                return result.document.export_to_markdown()
+            elif file_extension in FileReader.text_extensions:
+                # Read plain text files directly
+                content = path.read_text(encoding="utf-8")
+
+                # Wrap code files (but not plain txt) in markdown code blocks for better presentation
+                if file_extension in FileReader.code_markdown_identifier:
+                    language = FileReader.code_markdown_identifier[file_extension]
+                    return f"```{language}\n{content}\n```"
+
+                return content
+            else:
+                # Fallback: try to read as text
+                return path.read_text(encoding="utf-8")
         except Exception:
             raise ValueError(f"Failed to parse file: {path}")

haiku/rag/store/engine.py

@@ -37,6 +37,11 @@ class Store:
         db = sqlite3.connect(self.db_path)
         db.enable_load_extension(True)
         sqlite_vec.load(db)
+
+        # Enable WAL mode for better concurrency (skip for in-memory databases)
+        if self.db_path != ":memory:":
+            db.execute("PRAGMA journal_mode=WAL")
+
         self._connection = db
         existing_tables = [
             row[0]

{haiku_rag-0.4.2.dist-info → haiku_rag-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.4.2
+Version: 0.4.3
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -18,9 +18,9 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
+Requires-Dist: docling>=2.15.0
 Requires-Dist: fastmcp>=2.8.1
 Requires-Dist: httpx>=0.28.1
-Requires-Dist: markitdown[audio-transcription,docx,pdf,pptx,xlsx]>=0.1.2
 Requires-Dist: mxbai-rerank>=0.1.6
 Requires-Dist: ollama>=0.5.1
 Requires-Dist: pydantic>=2.11.7
@@ -55,7 +55,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 - **Reranking**: Default search result reranking with MixedBread AI or Cohere
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
-- **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
+- **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs
 - **MCP server**: Expose as tools for AI assistants
 - **CLI & Python API**: Use from command line or Python
 

{haiku_rag-0.4.2.dist-info → haiku_rag-0.4.3.dist-info}/RECORD

@@ -1,13 +1,13 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 haiku/rag/app.py,sha256=FpLVyP1-zAq_XPmU8CPVLkuIAeuhBOGvMqhYS8RbN40,7649
 haiku/rag/chunker.py,sha256=MbCtP66OfTFoIBvqmVT9T9c87fozsYYzAQzJJEfPBVI,1812
-haiku/rag/cli.py,sha256=oCj65JcV2MEhzA2okbLHAK1I0FrClIKuYZx2jtbjbqE,5628
+haiku/rag/cli.py,sha256=k7EhLkvTncxsdh5TYrg8BHLYh_lfyzupsWGj1dEEdqY,5992
 haiku/rag/client.py,sha256=MZNIpMm6MS3P6vjLqiCztT2dBOM7-bZOosX5IpbHJbI,12724
 haiku/rag/config.py,sha256=_Ss54kmfxVAJupExLKaYjYUlFxJgb7hEEdbG4-isapY,1662
 haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
 haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
 haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
-haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
+haiku/rag/reader.py,sha256=dLz3yyc5r8dzdqCc2VViC3fADpScw4lxXueKiu-cI7c,2915
 haiku/rag/utils.py,sha256=Ez_tvNlRO_D8c2CBZ83Hs9Gmzcqdq4cmw_V5GBdKy_8,2214
 haiku/rag/embeddings/__init__.py,sha256=yFBlxS0jBiVHl_rWz5kb43t6Ha132U1ZGdlIPfhzPdg,1491
 haiku/rag/embeddings/base.py,sha256=NTQvuzbZPu0LBo5wAu3qGyJ4xXUaRAt1fjBO0ygWn_Y,465
@@ -25,7 +25,7 @@ haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,4
 haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
 haiku/rag/reranking/mxbai.py,sha256=46sVTsTIkzIX9THgM3u8HaEmgY7evvEyB-N54JTHvK8,867
 haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
-haiku/rag/store/engine.py,sha256=4ouAD0s-TFwEoEHjVVw_KnV6aaw5nwhe9fdT8PRXfok,6061
+haiku/rag/store/engine.py,sha256=cOMBToLilI1Di1qQrFzGLqtRMsuvtiX0Q5RNIEzQy9w,6232
 haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
 haiku/rag/store/models/chunk.py,sha256=9-vIxW75-kMTelIhgVIMd_WhP-Drc1q65vjaWMP8w1E,364
 haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
@@ -36,8 +36,8 @@ haiku/rag/store/repositories/document.py,sha256=fXIWevJaOe6x2cK4u9cQxiEGD0ntKQb9
 haiku/rag/store/repositories/settings.py,sha256=dme3_ulQdQvyF9daavSjAd-SjZ5hh0MJoxP7iXgap-A,2492
 haiku/rag/store/upgrades/__init__.py,sha256=kKS1YWT_P-CYKhKtokOLTIFNKf9jlfjFFr8lyIMeogM,100
 haiku/rag/store/upgrades/v0_3_4.py,sha256=GLogKZdZ40NX1vBHKdOJju7fFzNUCHoEnjSZg17Hm2U,663
-haiku_rag-0.4.2.dist-info/METADATA,sha256=0ctTSGB6uqGl2INUcNxnCphxwrLTlK7KVdKKXXB35mg,4235
-haiku_rag-0.4.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.4.2.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.4.2.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.4.2.dist-info/RECORD,,
+haiku_rag-0.4.3.dist-info/METADATA,sha256=T2ZHdGL_zd1eSfEjFolh3R_zJpuWmUhKsnNkYLKtT7E,4198
+haiku_rag-0.4.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.4.3.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.4.3.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.4.3.dist-info/RECORD,,