haiku.rag 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

haiku/rag/app.py

@@ -1,9 +1,13 @@
+import asyncio
 from pathlib import Path
 
 from rich.console import Console
 from rich.markdown import Markdown
 
 from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.mcp import create_mcp_server
+from haiku.rag.monitor import FileWatcher
 from haiku.rag.store.models.chunk import Chunk
 from haiku.rag.store.models.document import Document
 
@@ -88,20 +92,25 @@ class HaikuRAGApp:
         self.console.print(content)
         self.console.rule()
 
-    def serve(self, transport: str | None = None):
+    async def serve(self, transport: str | None = None):
         """Start the MCP server."""
-        from haiku.rag.mcp import create_mcp_server
+        async with HaikuRAG(self.db_path) as client:
+            monitor = FileWatcher(paths=Config.MONITOR_DIRECTORIES, client=client)
+            monitor_task = asyncio.create_task(monitor.observe())
+            server = create_mcp_server(self.db_path)
 
-        server = create_mcp_server(self.db_path)
-
-        if transport == "stdio":
-            self.console.print("[green]Starting MCP server on stdio...[/green]")
-            server.run("stdio")
-        elif transport == "sse":
-            self.console.print(
-                "[green]Starting MCP server with streamable HTTP...[/green]"
-            )
-            server.run("sse")
-        else:
-            self.console.print("[green]Starting MCP server with HTTP...[/green]")
-            server.run("streamable-http")
+            try:
+                if transport == "stdio":
+                    await server.run_stdio_async()
+                elif transport == "sse":
+                    await server.run_sse_async("sse")
+                else:
+                    await server.run_http_async("streamable-http")
+            except KeyboardInterrupt:
+                pass
+            finally:
+                monitor_task.cancel()
+                try:
+                    await monitor_task
+                except asyncio.CancelledError:
+                    pass

haiku/rag/cli.py

@@ -146,7 +146,7 @@ def serve(
     elif sse:
         transport = "sse"
 
-    app.serve(transport=transport)
+    event_loop.run_until_complete(app.serve(transport=transport))
 
 
 if __name__ == "__main__":

haiku/rag/client.py

@@ -88,7 +88,7 @@ class HaikuRAG:
         if not source_path.exists():
             raise ValueError(f"File does not exist: {source_path}")
 
-        uri = str(source_path.resolve())
+        uri = source_path.as_uri()
         md5_hash = hashlib.md5(source_path.read_bytes()).hexdigest()
 
         # Check if document already exists

haiku/rag/config.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 
 from dotenv import load_dotenv
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from haiku.rag.utils import get_default_data_dir
 
@@ -13,16 +13,28 @@ class AppConfig(BaseModel):
     ENV: str = "development"
 
     DEFAULT_DATA_DIR: Path = get_default_data_dir()
+    MONITOR_DIRECTORIES: list[Path] = []
 
-    EMBEDDING_PROVIDER: str = "ollama"
-    EMBEDDING_MODEL: str = "mxbai-embed-large"
-    EMBEDDING_VECTOR_DIM: int = 1024
+    EMBEDDINGS_PROVIDER: str = "ollama"
+    EMBEDDINGS_MODEL: str = "mxbai-embed-large"
+    EMBEDDINGS_VECTOR_DIM: int = 1024
 
     CHUNK_SIZE: int = 256
     CHUNK_OVERLAP: int = 32
 
     OLLAMA_BASE_URL: str = "http://localhost:11434"
 
+    @field_validator("MONITOR_DIRECTORIES", mode="before")
+    @classmethod
+    def parse_monitor_directories(cls, v):
+        if isinstance(v, str):
+            if not v.strip():
+                return []
+            return [
+                Path(path.strip()).absolute() for path in v.split(",") if path.strip()
+            ]
+        return v
+
 
 # Expose Config object for app to import
 Config = AppConfig.model_validate(os.environ)

haiku/rag/embeddings/__init__.py

@@ -8,10 +8,10 @@ def get_embedder() -> EmbedderBase:
     Factory function to get the appropriate embedder based on the configuration.
     """
 
-    if Config.EMBEDDING_PROVIDER == "ollama":
-        return OllamaEmbedder(Config.EMBEDDING_MODEL, Config.EMBEDDING_VECTOR_DIM)
+    if Config.EMBEDDINGS_PROVIDER == "ollama":
+        return OllamaEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
 
-    if Config.EMBEDDING_PROVIDER == "voyageai":
+    if Config.EMBEDDINGS_PROVIDER == "voyageai":
         try:
             from haiku.rag.embeddings.voyageai import Embedder as VoyageAIEmbedder
         except ImportError:
@@ -20,5 +20,17 @@ def get_embedder() -> EmbedderBase:
                 "Please install haiku.rag with the 'voyageai' extra:"
                 "uv pip install haiku.rag --extra voyageai"
             )
-        return VoyageAIEmbedder(Config.EMBEDDING_MODEL, Config.EMBEDDING_VECTOR_DIM)
-    raise ValueError(f"Unsupported embedding provider: {Config.EMBEDDING_PROVIDER}")
+        return VoyageAIEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
+
+    if Config.EMBEDDINGS_PROVIDER == "openai":
+        try:
+            from haiku.rag.embeddings.openai import Embedder as OpenAIEmbedder
+        except ImportError:
+            raise ImportError(
+                "OpenAI embedder requires the 'openai' package. "
+                "Please install haiku.rag with the 'openai' extra:"
+                "uv pip install haiku.rag --extra openai"
+            )
+        return OpenAIEmbedder(Config.EMBEDDINGS_MODEL, Config.EMBEDDINGS_VECTOR_DIM)
+
+    raise ValueError(f"Unsupported embedding provider: {Config.EMBEDDINGS_PROVIDER}")

haiku/rag/embeddings/ollama.py

@@ -5,7 +5,7 @@ from haiku.rag.embeddings.base import EmbedderBase
 
 
 class Embedder(EmbedderBase):
-    _model: str = Config.EMBEDDING_MODEL
+    _model: str = Config.EMBEDDINGS_MODEL
     _vector_dim: int = 1024
 
     async def embed(self, text: str) -> list[float]:

haiku/rag/embeddings/openai.py

@@ -0,0 +1,20 @@
+try:
+    from openai import AsyncOpenAI
+
+    from haiku.rag.config import Config
+    from haiku.rag.embeddings.base import EmbedderBase
+
+    class Embedder(EmbedderBase):
+        _model: str = Config.EMBEDDINGS_MODEL
+        _vector_dim: int = 1536
+
+        async def embed(self, text: str) -> list[float]:
+            client = AsyncOpenAI()
+            response = await client.embeddings.create(
+                model=self._model,
+                input=text,
+            )
+            return response.data[0].embedding
+
+except ImportError:
+    pass

haiku/rag/embeddings/voyageai.py

@@ -5,7 +5,7 @@ try:
     from haiku.rag.embeddings.base import EmbedderBase
 
     class Embedder(EmbedderBase):
-        _model: str = Config.EMBEDDING_MODEL
+        _model: str = Config.EMBEDDINGS_MODEL
         _vector_dim: int = 1024
 
         async def embed(self, text: str) -> list[float]:

haiku/rag/logging.py

@@ -0,0 +1,24 @@
+import logging
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+
+def get_logger() -> logging.Logger:
+    logger = logging.getLogger("haiku.rag")
+
+    handler = RichHandler(
+        console=Console(stderr=True),
+        rich_tracebacks=True,
+    )
+    formatter = logging.Formatter("%(message)s")
+    handler.setFormatter(formatter)
+
+    logger.setLevel("INFO")
+
+    # Remove any existing handlers to avoid duplicates on reconfiguration
+    for hdlr in logger.handlers[:]:
+        logger.removeHandler(hdlr)
+
+    logger.addHandler(handler)
+    return logger

haiku/rag/monitor.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+from watchfiles import Change, DefaultFilter, awatch
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.logging import get_logger
+from haiku.rag.reader import FileReader
+from haiku.rag.store.models.document import Document
+
+logger = get_logger()
+
+
+class FileFilter(DefaultFilter):
+    def __init__(self, *, ignore_paths: list[Path] | None = None) -> None:
+        self.extensions = tuple(FileReader.extensions)
+        super().__init__(ignore_paths=ignore_paths)
+
+    def __call__(self, change: "Change", path: str) -> bool:
+        return path.endswith(self.extensions) and super().__call__(change, path)
+
+
+class FileWatcher:
+    def __init__(self, paths: list[Path], client: HaikuRAG):
+        self.paths = paths
+        self.client = client
+
+    async def observe(self):
+        logger.info(f"Watching files in {self.paths}")
+        filter = FileFilter()
+        await self.refresh()
+
+        async for changes in awatch(*self.paths, watch_filter=filter):
+            await self.handler(changes)
+
+    async def handler(self, changes: set[tuple[Change, str]]):
+        for change, path in changes:
+            if change == Change.added or change == Change.modified:
+                await self._upsert_document(Path(path))
+            elif change == Change.deleted:
+                await self._delete_document(Path(path))
+
+    async def refresh(self):
+        for path in self.paths:
+            for f in Path(path).rglob("**/*"):
+                if f.is_file() and f.suffix in FileReader.extensions:
+                    await self._upsert_document(f)
+
+    async def _upsert_document(self, file: Path) -> Document | None:
+        try:
+            uri = file.as_uri()
+            existing_doc = await self.client.get_document_by_uri(uri)
+            print(uri)
+            if existing_doc:
+                doc = await self.client.create_document_from_source(str(file))
+                logger.info(f"Updated document {existing_doc.id} from {file}")
+                return doc
+            else:
+                doc = await self.client.create_document_from_source(str(file))
+                logger.info(f"Created new document {doc.id} from {file}")
+                return doc
+        except Exception as e:
+            logger.error(f"Failed to upsert document from {file}: {e}")
+            return None
+
+    async def _delete_document(self, file: Path):
+        try:
+            uri = file.as_uri()
+            existing_doc = await self.client.get_document_by_uri(uri)
+
+            if existing_doc and existing_doc.id:
+                await self.client.delete_document(existing_doc.id)
+                logger.info(f"Deleted document {existing_doc.id} for {file}")
+        except Exception as e:
+            logger.error(f"Failed to delete document for {file}: {e}")

{haiku_rag-0.1.0.dist-info → haiku_rag-0.2.0.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.1.0
+Version: 0.2.0
 Summary: Retrieval Augmented Generation (RAG) with SQLite
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
 License-File: LICENSE
+Keywords: RAG,mcp,ml,sqlite,sqlite-vec
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -28,6 +29,8 @@ Requires-Dist: sqlite-vec>=0.1.6
 Requires-Dist: tiktoken>=0.9.0
 Requires-Dist: typer>=0.16.0
 Requires-Dist: watchfiles>=1.1.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
 Provides-Extra: voyageai
 Requires-Dist: voyageai>=0.3.2; extra == 'voyageai'
 Description-Content-Type: text/markdown
@@ -38,9 +41,12 @@ A SQLite-based Retrieval-Augmented Generation (RAG) system built for efficient d
 
 ## Features
 - **Local SQLite**: No need to run additional servers
-- **Support for various embedding providers**: You can use Ollama, VoyageAI or add your own
+- **Support for various embedding providers**: You can use Ollama, VoyageAI, OpenAI or add your own
 - **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
 - **Multi-format Support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a url!
+- **File monitoring** when run as a server automatically indexing your files
+- **MCP server** Exposes functionality as MCP tools.
+- **Python client** Call `haiku.rag` from your own python applications.
 
 ## Installation
 
@@ -52,24 +58,41 @@ By default Ollama (with the `mxbai-embed-large` model) is used for the embedding
 For other providers use:
 
 - **VoyageAI**: `uv pip install haiku.rag --extra voyageai`
+- **OpenAI**: `uv pip install haiku.rag --extra openai`
 
 ## Configuration
 
+You can set the directories to monitor using the `MONITOR_DIRECTORIES` environment variable (as comma separated values) :
+
+```bash
+# Monitor single directory
+export MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
+```
+
 If you want to use an alternative embeddings provider (Ollama being the default) you will need to set the provider details through environment variables:
 
 By default:
 
 ```bash
-EMBEDDING_PROVIDER="ollama"
-EMBEDDING_MODEL="mxbai-embed-large" # or any other model
-EMBEDDING_VECTOR_DIM=1024
+EMBEDDINGS_PROVIDER="ollama"
+EMBEDDINGS_MODEL="mxbai-embed-large" # or any other model
+EMBEDDINGS_VECTOR_DIM=1024
 ```
 
 For VoyageAI:
 ```bash
-EMBEDDING_PROVIDER="voyageai"
-EMBEDDING_MODEL="voyage-3.5" # or any other model
-EMBEDDING_VECTOR_DIM=1024
+EMBEDDINGS_PROVIDER="voyageai"
+EMBEDDINGS_MODEL="voyage-3.5" # or any other model
+EMBEDDINGS_VECTOR_DIM=1024
+VOYAGE_API_KEY="your-api-key"
+```
+
+For OpenAI:
+```bash
+EMBEDDINGS_PROVIDER="openai"
+EMBEDDINGS_MODEL="text-embedding-3-small" # or text-embedding-3-large
+EMBEDDINGS_VECTOR_DIM=1536
+OPENAI_API_KEY="your-api-key"
 ```
 
 ## Command Line Interface
@@ -101,7 +124,7 @@ haiku-rag search "machine learning"
 # Search with custom options
 haiku-rag search "python programming" --limit 10 --k 100
 
-# Start MCP server (default HTTP transport)
+# Start file monitoring & MCP server (default HTTP transport)
 haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
 ```
 
@@ -111,7 +134,26 @@ haiku-rag command -h
 ```
 to see additional parameters for a command.
 
-## MCP Server
+## File Monitoring & MCP server
+
+You can start the server (using Streamble HTTP, stdio or SSE transports) with:
+
+```bash
+# Start with default HTTP transport
+haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
+```
+
+You need to have set the `MONITOR_DIRECTORIES` environment variable for monitoring to take place.
+
+### File monitoring
+
+`haiku.rag` can watch directories for changes and automatically update the document store:
+
+- **Startup**: Scan all monitored directories and add any new files
+- **File Added/Modified**: Automatically parse and add/update the document in the database
+- **File Deleted**: Remove the corresponding document from the database
+
+### MCP Server
 
 `haiku.rag` includes a Model Context Protocol (MCP) server that exposes RAG functionality as tools for AI assistants like Claude Desktop. The MCP server provides the following tools:
 
@@ -123,13 +165,6 @@ to see additional parameters for a command.
 - `list_documents` - List all documents with pagination
 - `delete_document` - Delete documents by ID
 
-You can start the server (using Streamble HTTP, stdio or SSE transports) with:
-
-```bash
-# Start with default HTTP transport
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
 ## Using `haiku.rag` from python
 
 ### Managing documents

{haiku_rag-0.1.0.dist-info → haiku_rag-0.2.0.dist-info}/RECORD

@@ -1,16 +1,19 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=jJb5THgH3nbh2K8uiYsMVlkqVbSkIGEyxPMISM3epMA,4546
+haiku/rag/app.py,sha256=FedUvIxPXCi7SmxUi9zJcxmoZBTQZJO00XIkoD-k87s,4915
 haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
-haiku/rag/cli.py,sha256=XOxl7H86La7fB4DvsEJxtNuSfZgOtwqQDmECSaxv4sY,4020
-haiku/rag/client.py,sha256=H5zE-HO8Asxo-_vEcnxFqvQixdiTFTqvNH8EkH7Xo4E,9713
-haiku/rag/config.py,sha256=GxpfUwsQmfzQcknIAPEET_Qu-0WFYtPkHrV3arvNdxM,596
+haiku/rag/cli.py,sha256=SvDPYHHdjPu8bEF8PgE4agUo-5j3Kuq_rS9Cav6xch0,4051
+haiku/rag/client.py,sha256=uWqcowc8J2Yv-liGBGSJnuQkFw4CDlf_ivOxp6E5C1M,9707
+haiku/rag/config.py,sha256=b346EQ7HCFy-OU3K-SzSSoOLMuQseHFoiysYZMO1zCc,1003
+haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
 haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
+haiku/rag/monitor.py,sha256=aFJb5cnesEBIGyVzt8IXYrlTujiC1QSPczPuAam2yXw,2793
 haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
 haiku/rag/utils.py,sha256=6xVM6z2OmhzB4FEDlPbMsr_ZBBmCbMQb83nP6E2UdxY,629
-haiku/rag/embeddings/__init__.py,sha256=jOamqhoeFX9J-ThwvVyHGd2s8jqJzA8B6J4sxHGZ39o,1007
+haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
 haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,379
-haiku/rag/embeddings/ollama.py,sha256=i_w7hbh-_ukysco274fLkQuFRgaFq0zIwIs8CNmRcLE,440
-haiku/rag/embeddings/voyageai.py,sha256=MPioqQ0duzjglqvnN_8ftVq11fvBrcpV03p9MMLwflM,533
+haiku/rag/embeddings/ollama.py,sha256=hWdrTiuJwNSRYCqP0WP-z6XXA3RBGkAiknZMsPLH0qU,441
+haiku/rag/embeddings/openai.py,sha256=reh8AykG2f9f5hhRDmqSsjiuCPi9SsXfe2YEZFlxXk8,550
+haiku/rag/embeddings/voyageai.py,sha256=jc0JywdLJD3Ee1MUv1m8MhWCEo0enNnVcrIBtUvD-Ss,534
 haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
 haiku/rag/store/engine.py,sha256=BeYZRZ08zaYeeu375ysnAL3tGz4roA3GzP7WRNwznCo,2603
 haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
@@ -20,8 +23,8 @@ haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPv
 haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
 haiku/rag/store/repositories/chunk.py,sha256=6zABVlb5zbMQ4s50z9qb53ieHYaiv4CjgxpbsXxs814,14639
 haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
-haiku_rag-0.1.0.dist-info/METADATA,sha256=kDmX6IcmvyL8ss4Go30_UDaSBA4TTzpkp6unzcDOgnM,6141
-haiku_rag-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.1.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.1.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.1.0.dist-info/RECORD,,
+haiku_rag-0.2.0.dist-info/METADATA,sha256=o9PPoiXU7VIRAuQVwFvfQg4w-8ufz5aLo9PuG0ykWuI,7468
+haiku_rag-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.2.0.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.2.0.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.2.0.dist-info/RECORD,,