haiku.rag 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

haiku/rag/utils.py

@@ -176,19 +176,19 @@ def prefetch_models():
 
     # Collect Ollama models from config
     required_models: set[str] = set()
-    if Config.EMBEDDINGS_PROVIDER == "ollama":
-        required_models.add(Config.EMBEDDINGS_MODEL)
-    if Config.QA_PROVIDER == "ollama":
-        required_models.add(Config.QA_MODEL)
-    if Config.RESEARCH_PROVIDER == "ollama":
-        required_models.add(Config.RESEARCH_MODEL)
-    if Config.RERANK_PROVIDER == "ollama":
-        required_models.add(Config.RERANK_MODEL)
+    if Config.embeddings.provider == "ollama":
+        required_models.add(Config.embeddings.model)
+    if Config.qa.provider == "ollama":
+        required_models.add(Config.qa.model)
+    if Config.research.provider == "ollama":
+        required_models.add(Config.research.model)
+    if Config.reranking.provider == "ollama":
+        required_models.add(Config.reranking.model)
 
     if not required_models:
         return
 
-    base_url = Config.OLLAMA_BASE_URL
+    base_url = Config.providers.ollama.base_url
 
     with httpx.Client(timeout=None) as client:
         for model in sorted(required_models):

{haiku_rag-0.12.1.dist-info → haiku_rag-0.13.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiku.rag
-Version: 0.12.1
+Version: 0.13.1
 Summary: Agentic Retrieval Augmented Generation (RAG) with LanceDB
 Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
 License: MIT
@@ -13,9 +13,8 @@ Classifier: Operating System :: MacOS
 Classifier: Operating System :: Microsoft :: Windows :: Windows 10
 Classifier: Operating System :: Microsoft :: Windows :: Windows 11
 Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
 Requires-Python: >=3.12
 Requires-Dist: docling>=2.56.1
@@ -26,6 +25,7 @@ Requires-Dist: pydantic-ai>=1.0.18
 Requires-Dist: pydantic-graph>=1.0.18
 Requires-Dist: pydantic>=2.12.2
 Requires-Dist: python-dotenv>=1.1.1
+Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: rich>=14.2.0
 Requires-Dist: tiktoken>=0.12.0
 Requires-Dist: typer>=0.19.2
@@ -40,11 +40,13 @@ Description-Content-Type: text/markdown
 
 # Haiku RAG
 
+mcp-name: io.github.ggozad/haiku-rag
+
 Retrieval-Augmented Generation (RAG) library built on LanceDB.
 
 `haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
 
-> **Note**: Starting with version 0.7.0, haiku.rag uses LanceDB instead of SQLite. If you have an existing SQLite database, use `haiku-rag migrate old_database.sqlite` to migrate your data safely.
+> **Note**: Configuration now uses YAML files instead of environment variables. If you're upgrading from an older version, run `haiku-rag init-config --from-env` to migrate your `.env` file to `haiku.rag.yaml`. See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for details.
 
 ## Features
 
@@ -65,6 +67,7 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB.
 
 ```bash
 # Install
+# Python 3.12 or newer required
 uv pip install haiku.rag
 
 # Add documents
@@ -98,14 +101,12 @@ haiku-rag research \
 # Rebuild database (re-chunk and re-embed all documents)
 haiku-rag rebuild
 
-# Migrate from SQLite to LanceDB
-haiku-rag migrate old_database.sqlite
-
 # Start server with file monitoring
-export MONITOR_DIRECTORIES="/path/to/docs"
-haiku-rag serve
+haiku-rag serve --monitor
 ```
 
+To customize settings, create a `haiku.rag.yaml` config file (see [Configuration](https://ggozad.github.io/haiku.rag/configuration/)).
+
 ## Python Usage
 
 ```python
@@ -197,17 +198,26 @@ haiku-rag a2aclient
 ```
 
 The A2A agent provides:
+
 - Multi-turn dialogue with context
 - Intelligent multi-search for complex questions
 - Source citations with titles and URIs
 - Full document retrieval on request
 
+## Examples
+
+See the [examples directory](examples/) for working examples:
+
+- **[Interactive Research Assistant](examples/ag-ui-research/)** - Full-stack research assistant with Pydantic AI and AG-UI featuring human-in-the-loop approval and real-time state synchronization
+- **[Docker Setup](examples/docker/)** - Complete Docker deployment with file monitoring, MCP server, and A2A agent
+- **[A2A Security](examples/a2a-security/)** - Authentication examples (API key, OAuth2, GitHub)
+
 ## Documentation
 
 Full documentation at: https://ggozad.github.io/haiku.rag/
 
 - [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup
-- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
+- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - YAML configuration
 - [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
 - [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
 - [Agents](https://ggozad.github.io/haiku.rag/agents/) - QA agent and multi-agent research

{haiku_rag-0.12.1.dist-info → haiku_rag-0.13.1.dist-info}/RECORD

@@ -1,16 +1,14 @@
 haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-haiku/rag/app.py,sha256=nrfg3iGWP_HJBfwUFPv60_1Do8YK3WQYWZhq22r001s,21548
-haiku/rag/chunker.py,sha256=PVe6ysv8UlacUd4Zb3_8RFWIaWDXnzBAy2VDJ4TaUsE,1555
-haiku/rag/cli.py,sha256=ghmfvCmoitgySZsF6t5UQjsm3_rul0KUh0L774BzeuI,13196
-haiku/rag/client.py,sha256=GVXHq9weIaFdcZvO9a4YO1WnrroJJUXVVriDGdMxpH4,25855
-haiku/rag/config.py,sha256=FBsMMijl5PxIfPGifk_AJVRjL4omb03jfoZm0P_VqxI,2743
+haiku/rag/app.py,sha256=lNPYYWQLOPlOLYiwIR4yQuwzl-LrRNQn7n2nacgdq_k,21594
+haiku/rag/chunker.py,sha256=pA0S0fFKAuvzGm2dGyp7FAkeFZA0YTCm_ata83Pnflw,1566
+haiku/rag/cli.py,sha256=Y42tnlVFGvCZVjBcLWrIVgM0A7KjNYX9MAuk9-zQvvE,14523
+haiku/rag/client.py,sha256=cG6DAhzJJ4vdo8QFn9p8iA6YTa0arMrTtIswoZc7sY0,26816
 haiku/rag/logging.py,sha256=dm65AwADpcQsH5OAPtRA-4hsw0w5DK-sGOvzYkj6jzw,1720
-haiku/rag/mcp.py,sha256=DZk-IJgVjAesu-vvqVd5BYnfDWKWNR6TQugKgdoFrvg,8976
-haiku/rag/migration.py,sha256=XldX0CTHPXNGrkdQ-gocr4kQGBsz-316WcE0ZDRfb48,11076
-haiku/rag/monitor.py,sha256=VP3bqY0mEodOP60eN4RMldgrL1ti5gMjuDuQ-_vBvFc,2759
+haiku/rag/mcp.py,sha256=txuEnrUMWvs_shQBk15gEkJD7xNdSYzp3z75UUWaHFM,9328
+haiku/rag/monitor.py,sha256=d92oRufhI8oYXH7oF6oYVf1_AcpFUafjM6tl4VhAupI,3322
 haiku/rag/reader.py,sha256=aW8LG0X31kVWS7kU2tKVpe8RqP3Ne_oIidd_X3UDLH0,3307
-haiku/rag/utils.py,sha256=dBzhKaOHI9KRiJqHErcXUnqtnXY2AgOK8PCLA3rhO0A,6115
-haiku/rag/a2a/__init__.py,sha256=4SlJBr9GUVZ0879o5VI6-qpcBKpieP2hW4hmNbm8NGg,5933
+haiku/rag/utils.py,sha256=47ehVYJlLz6Of_Ua89qj94JclO5ZPBFU9eyonifvnVg,6131
+haiku/rag/a2a/__init__.py,sha256=tY_jLSUM0zKzyBctMkjpqmDWpxWc9QVEK1qAsb-plGs,5933
 haiku/rag/a2a/client.py,sha256=awuiHXgVHn1uzaEXE98RIqqKHj1JjszOvn9WI3Jtth8,8760
 haiku/rag/a2a/context.py,sha256=SofkFUZcGonoJcgZh-RGqHTh0UWT4J7Zl4Mz6WDkMl4,2053
 haiku/rag/a2a/models.py,sha256=XhGYj2g3rgVM4JoCDXlll0YjaysqdalybJrBqFXSwl4,689
@@ -18,15 +16,18 @@ haiku/rag/a2a/prompts.py,sha256=yCla8x0hbOhKrkuaqVrF1upn-YjQM3-2NsE2TSnet0M,3030
 haiku/rag/a2a/skills.py,sha256=dwyD2Bn493eL3Vf4uQzmyxj_9IUSb66kQ-085FBAuCs,2701
 haiku/rag/a2a/storage.py,sha256=c8vmGCiZ3nuV9wUuTnwpoRD2HVVvK2JPySQOc5PVMvg,2759
 haiku/rag/a2a/worker.py,sha256=S9hiA1ncpJPdtN0eEmMjsvr5LQ4wMVN5R8CjYkTeohU,12367
-haiku/rag/embeddings/__init__.py,sha256=44IfDITGIFTflGT6UEmiYOwpWFVbYv5smLY59D0YeCs,1419
-haiku/rag/embeddings/base.py,sha256=Aw4kjfVn2can0R17pdiAgpPRyk5BpdBgMXuor5mstDY,682
-haiku/rag/embeddings/ollama.py,sha256=KXq-eJ58co5rwYchIO3kpvIv0OBwMJkwMXq1xDsETz0,823
+haiku/rag/config/__init__.py,sha256=PSHsc7gXjvRxpzN4rxR083-WYU-pocqm0hf2uhkr9Vw,1019
+haiku/rag/config/loader.py,sha256=eWkD8uVTa19nf7d7yyZImk7t5k0-SagYH4RSBqfkPxQ,4848
+haiku/rag/config/models.py,sha256=vkq2WyJfuY1cm8YEFlox0Cd8sVyXb4l1XX2fkBjI6I4,2169
+haiku/rag/embeddings/__init__.py,sha256=zwWRU9S5YGEJxlgPv5haHBgj3LUJMe-dEwr3LKLa9RY,1731
+haiku/rag/embeddings/base.py,sha256=kzca54e2HGzS_0YKt9OLESM9lrFKpBm_97V07jx0aas,783
+haiku/rag/embeddings/ollama.py,sha256=_uIIObbZX9QVU1lcgWQFooA3b-AeZRNncM7yQ2TxlEU,825
 haiku/rag/embeddings/openai.py,sha256=BfmPni567DH8KqwLCPiOmr3q-dpzpOJkvFFoUuTR5as,731
-haiku/rag/embeddings/vllm.py,sha256=wgul0nMWTn6Q1aKA4DJe03EktsRoBxEgtB7gfpWVOyQ,854
+haiku/rag/embeddings/vllm.py,sha256=IZFS3pbvLXkhvdT7pFVi2csFlNTSza5bpybwz7ud3Po,847
 haiku/rag/embeddings/voyageai.py,sha256=6vEuk6q510AJv-K2lL93P2dVrziAjELTOe_w_Zp5YT4,917
 haiku/rag/graph/__init__.py,sha256=BHfMchuUO_UhHKpjjGHjd6xPxNkrIwJzHn4YJiLqG1g,62
 haiku/rag/graph/base.py,sha256=DepZqLF9E64YCCkjmbqmgyp28oNp69WfJCXp614xzh0,819
-haiku/rag/graph/common.py,sha256=xTejucXei3x9tqbal3ZS_64lZAC6Bw3-QfXPniZcZEw,986
+haiku/rag/graph/common.py,sha256=-Pdao6ZiTgv4ppNctrRpwLG0U6za-66aScQWZ0uCUjc,1016
 haiku/rag/graph/models.py,sha256=sgL5_wSbQJrNOITH615jryPBhTE8J3ZiZWVxO9Ty-JI,755
 haiku/rag/graph/prompts.py,sha256=xJgAHjUVczxCgk7YLPyy6DdQFi0lwj42vJqIFnPqcYw,2221
 haiku/rag/graph/nodes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,8 +35,8 @@ haiku/rag/graph/nodes/analysis.py,sha256=g-Aw3nPuCHWo0CXM96Ixa4vQI4TpI6tg6ooHT_J
 haiku/rag/graph/nodes/plan.py,sha256=Bb6Fva9vwArCU-5xBr24N4pM3wfLP-Vwufgss8HfXMQ,2622
 haiku/rag/graph/nodes/search.py,sha256=DdHhEY7fmWUqis6Nk0bj-di56-ML262B51N9zytzKYk,3699
 haiku/rag/graph/nodes/synthesize.py,sha256=WF0D44SwLP1OK8C6ViOAhFOtGQ0mj3aO54z5bemJb4E,1828
-haiku/rag/qa/__init__.py,sha256=eFRV5GFwe1UsqniEqOLdzAMT2J6QhSiHq5_Li7c6Fs4,520
-haiku/rag/qa/agent.py,sha256=sN2SVpaQAxg5Hm47LhrHpbo3ELVi1ev9DxKu_ec1c-Y,3123
+haiku/rag/qa/__init__.py,sha256=Q18B5cjgYSuOdzwsJkXDeqcclAI2pu3tBIcWLcMTT5M,949
+haiku/rag/qa/agent.py,sha256=ReuvluxVzaH82PhrFLNAAM3rVrSj-sKHkhki266SsGI,3181
 haiku/rag/qa/prompts.py,sha256=Lqwn3m4zCsu_CJiC4s9cLsuPNbb9nq6j2PqEF3lw1eA,3380
 haiku/rag/qa/deep/__init__.py,sha256=SnCpWxWip-TaFzVKlFyrOgYeXEqT_gpIlaSItEEJ6r0,50
 haiku/rag/qa/deep/dependencies.py,sha256=AKFqcC1D3N1VPudnFmLH29K5eJWEC5wtwUGkO4FM4jc,998
@@ -44,11 +45,11 @@ haiku/rag/qa/deep/models.py,sha256=siZMQXD21_3nk8kaLCv0BCuD9TydLYo-yC4-9CxQy3E,6
 haiku/rag/qa/deep/nodes.py,sha256=XbDujD_xg-NsJS2oYm3LqkfxeHZCzT2f9wBNhVh0wns,10442
 haiku/rag/qa/deep/prompts.py,sha256=t1fEvoD5Rdab92eAOvefv2wBVmkPFuR0BQ8Kh4X0-mY,2565
 haiku/rag/qa/deep/state.py,sha256=ICFIX0oRtBs6Sdq9YnmnP04BkGiQYwucfS3Mf8XLcCU,570
-haiku/rag/reranking/__init__.py,sha256=95ApqN51rcog9MLkTh_uNE69qOVozO1Z6KMbZZj8nH0,963
-haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
-haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
-haiku/rag/reranking/mxbai.py,sha256=uveGFIdmNmepd2EQsvYr64wv0ra2_wB845hdSZXy5Cw,908
-haiku/rag/reranking/vllm.py,sha256=xVGH9ss-ISWdJ5SKUUHUbTqBo7PIEmA_SQv0ScdJ6XA,1479
+haiku/rag/reranking/__init__.py,sha256=34xH2YZ7OCC3H8Yb-zyPuehTRQdijMSY9TC8AmZDOGQ,1296
+haiku/rag/reranking/base.py,sha256=Yji15nAR8LyIJGqZvEZifTWmortNQ4k_7ZHst_5mRYk,408
+haiku/rag/reranking/cohere.py,sha256=BhBPPnaSnDoVlkL_MHF74kegXQBrsZGKnWqC40ztiAk,1050
+haiku/rag/reranking/mxbai.py,sha256=qR55dmpaBz15lSN_wXD3-Z6Kqr_bmNKU9q4Pwef_wB8,911
+haiku/rag/reranking/vllm.py,sha256=Ip83qzV2RM7qXTj0mE2St66hvXykovoNW8Hu3AUebDc,1489
 haiku/rag/research/__init__.py,sha256=xfVzYkt8QETjZaP8v4LdK8MC2R_JmxKDD34LefvrJbo,201
 haiku/rag/research/common.py,sha256=mrKXolTnDxcONjodmSPVAtXYWqbU0Bie2W-4lOTclGY,2988
 haiku/rag/research/dependencies.py,sha256=hiGImk7HyJF4LRYnTmsLFGzY1QrGjAyPW5vb_2JQRDo,8148
@@ -58,19 +59,19 @@ haiku/rag/research/prompts.py,sha256=opz4MXjoDHH1wjG6bPyiqT0LVzk3pBA6y_a9zpBW8yM
 haiku/rag/research/state.py,sha256=P8RXJMi3wA3l1j6yo8dsAyso6S27FgqS7fvZUUY447A,917
 haiku/rag/research/stream.py,sha256=amyGDimkNp_FHYUXCqtpbeDOx7sC1jQ-7DwoxuNOL1g,5576
 haiku/rag/store/__init__.py,sha256=R2IRcxtkFDxqa2sgMirqLq3l2-FPdWr6ydYStaqm5OQ,104
-haiku/rag/store/engine.py,sha256=n2IxztyN2UpLLSUVXurjL-e_ANthKUpWyB1gdHfgBMM,11468
+haiku/rag/store/engine.py,sha256=FP1-9LOxoEvQBswYcM2GS_E2RpvSZct49vVktL-oPlo,11697
 haiku/rag/store/models/__init__.py,sha256=kc7Ctf53Jr483tk4QTIrcgqBbXDz4ZoeYSkFXfPnpks,89
 haiku/rag/store/models/chunk.py,sha256=3EuZav4QekJIeHBCub48EM8SjNX8HEJ6wVDXGot4PEQ,421
 haiku/rag/store/models/document.py,sha256=cZXy_jEti-hnhq7FKhuhCfd99ccY9fIHMLovB_Thbb8,425
 haiku/rag/store/repositories/__init__.py,sha256=Olv5dLfBQINRV3HrsfUpjzkZ7Qm7goEYyMNykgo_DaY,291
-haiku/rag/store/repositories/chunk.py,sha256=B0CowrBNy0fd8GLnVJVfqDaLoWxEPPJK3SODya0I0OI,14093
-haiku/rag/store/repositories/document.py,sha256=EtgD5pDjghXf6dloBOOEVJp8DI9O_celc_FbYzOywAE,8125
-haiku/rag/store/repositories/settings.py,sha256=ObrDrzxHn-yA1WcbgIoJoVmAbVvQHAFvEdRyJFt5Opc,5685
+haiku/rag/store/repositories/chunk.py,sha256=bXa-NBfLdLKJuFLGEKQhFlsLi-XNbojhQYVyBjwUxz8,14205
+haiku/rag/store/repositories/document.py,sha256=UOC_5QEUl-3RnGPJzn92KjrCnhmh5TmWln4yd5cJ4Ss,8133
+haiku/rag/store/repositories/settings.py,sha256=15gS7Xj7cG4qetv_ioxZO_r31by7GuSqtpowOsMkHmc,6129
 haiku/rag/store/upgrades/__init__.py,sha256=RQ8A6rEXBASLb5PD9vdDnEas_m_GgRzzdVu4B88Snqc,1975
 haiku/rag/store/upgrades/v0_10_1.py,sha256=qNGnxj6hoHaHJ1rKTiALfw0c9NQOi0KAK-VZCD_073A,1959
 haiku/rag/store/upgrades/v0_9_3.py,sha256=NrjNilQSgDtFWRbL3ZUtzQzJ8tf9u0dDRJtnDFwwbdw,3322
-haiku_rag-0.12.1.dist-info/METADATA,sha256=POFHzbGYiVj7UkX_1VSA8zUByIiQEG1dPePWO55T7nU,7477
-haiku_rag-0.12.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiku_rag-0.12.1.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
-haiku_rag-0.12.1.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
-haiku_rag-0.12.1.dist-info/RECORD,,
+haiku_rag-0.13.1.dist-info/METADATA,sha256=xoojNWUahlMw6gWdujYr_VNti4ss4We0mL0rkTOkxgo,8139
+haiku_rag-0.13.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiku_rag-0.13.1.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
+haiku_rag-0.13.1.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
+haiku_rag-0.13.1.dist-info/RECORD,,

haiku/rag/config.py

@@ -1,90 +0,0 @@
-import os
-from pathlib import Path
-
-from dotenv import load_dotenv
-from pydantic import BaseModel, field_validator
-
-from haiku.rag.utils import get_default_data_dir
-
-load_dotenv()
-
-
-class AppConfig(BaseModel):
-    ENV: str = "production"
-
-    LANCEDB_API_KEY: str = ""
-    LANCEDB_URI: str = ""
-    LANCEDB_REGION: str = ""
-
-    DEFAULT_DATA_DIR: Path = get_default_data_dir()
-    MONITOR_DIRECTORIES: list[Path] = []
-
-    EMBEDDINGS_PROVIDER: str = "ollama"
-    EMBEDDINGS_MODEL: str = "qwen3-embedding"
-    EMBEDDINGS_VECTOR_DIM: int = 4096
-
-    RERANK_PROVIDER: str = ""
-    RERANK_MODEL: str = ""
-
-    QA_PROVIDER: str = "ollama"
-    QA_MODEL: str = "gpt-oss"
-
-    # Research defaults (fallback to QA if not provided via env)
-    RESEARCH_PROVIDER: str = "ollama"
-    RESEARCH_MODEL: str = "gpt-oss"
-
-    CHUNK_SIZE: int = 256
-    CONTEXT_CHUNK_RADIUS: int = 0
-
-    # Optional dotted path or file path to a callable that preprocesses
-    # markdown content before chunking. Examples:
-    MARKDOWN_PREPROCESSOR: str = ""
-
-    OLLAMA_BASE_URL: str = "http://localhost:11434"
-
-    VLLM_EMBEDDINGS_BASE_URL: str = ""
-    VLLM_RERANK_BASE_URL: str = ""
-    VLLM_QA_BASE_URL: str = ""
-    VLLM_RESEARCH_BASE_URL: str = ""
-
-    # Provider keys
-    VOYAGE_API_KEY: str = ""
-    OPENAI_API_KEY: str = ""
-    ANTHROPIC_API_KEY: str = ""
-    COHERE_API_KEY: str = ""
-
-    # If true, refuse to auto-create a new LanceDB database or tables
-    # and error out when the database does not already exist.
-    DISABLE_DB_AUTOCREATE: bool = False
-
-    # Vacuum retention threshold in seconds. Only versions older than this
-    # threshold will be removed during vacuum operations. Default is 60 seconds
-    # to allow concurrent connections to safely use recent versions.
-    VACUUM_RETENTION_SECONDS: int = 60
-
-    # Maximum number of A2A contexts to keep in memory. When exceeded, least
-    # recently used contexts will be evicted. Default is 1000.
-    A2A_MAX_CONTEXTS: int = 1000
-
-    @field_validator("MONITOR_DIRECTORIES", mode="before")
-    @classmethod
-    def parse_monitor_directories(cls, v):
-        if isinstance(v, str):
-            if not v.strip():
-                return []
-            return [
-                Path(path.strip()).absolute() for path in v.split(",") if path.strip()
-            ]
-        return v
-
-
-# Expose Config object for app to import
-Config = AppConfig.model_validate(os.environ)
-if Config.OPENAI_API_KEY:
-    os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
-if Config.VOYAGE_API_KEY:
-    os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
-if Config.ANTHROPIC_API_KEY:
-    os.environ["ANTHROPIC_API_KEY"] = Config.ANTHROPIC_API_KEY
-if Config.COHERE_API_KEY:
-    os.environ["CO_API_KEY"] = Config.COHERE_API_KEY

haiku/rag/migration.py

@@ -1,316 +0,0 @@
-import json
-import sqlite3
-import struct
-from pathlib import Path
-from uuid import uuid4
-
-from rich.console import Console
-from rich.progress import Progress, TaskID
-
-from haiku.rag.store.engine import Store
-
-
-def deserialize_sqlite_embedding(data: bytes) -> list[float]:
-    """Deserialize sqlite-vec embedding from bytes."""
-    if not data:
-        return []
-    # sqlite-vec stores embeddings as float32 arrays
-    num_floats = len(data) // 4
-    return list(struct.unpack(f"{num_floats}f", data))
-
-
-class SQLiteToLanceDBMigrator:
-    """Migrates data from SQLite to LanceDB."""
-
-    def __init__(self, sqlite_path: Path, lancedb_path: Path):
-        self.sqlite_path = sqlite_path
-        self.lancedb_path = lancedb_path
-        self.console = Console()
-
-    async def migrate(self) -> bool:
-        """Perform the migration."""
-        try:
-            self.console.print(
-                f"[blue]Starting migration from {self.sqlite_path} to {self.lancedb_path}[/blue]"
-            )
-
-            # Check if SQLite database exists
-            if not self.sqlite_path.exists():
-                self.console.print(
-                    f"[red]SQLite database not found: {self.sqlite_path}[/red]"
-                )
-                return False
-
-            # Connect to SQLite database
-            sqlite_conn = sqlite3.connect(self.sqlite_path)
-            sqlite_conn.row_factory = sqlite3.Row
-
-            # Load the sqlite-vec extension
-            try:
-                import sqlite_vec  # type: ignore
-
-                sqlite_conn.enable_load_extension(True)
-                sqlite_vec.load(sqlite_conn)
-                self.console.print("[cyan]Loaded sqlite-vec extension[/cyan]")
-            except Exception as e:
-                self.console.print(
-                    f"[yellow]Warning: Could not load sqlite-vec extension: {e}[/yellow]"
-                )
-                self.console.print(
-                    "[yellow]Install sqlite-vec with[/yellow]\n[green]uv pip install sqlite-vec [/green]"
-                )
-                exit(1)
-
-            # Create LanceDB store
-            lance_store = Store(self.lancedb_path, skip_validation=True)
-
-            with Progress() as progress:
-                # Migrate documents
-                doc_task = progress.add_task(
-                    "[green]Migrating documents...", total=None
-                )
-                document_id_mapping = self._migrate_documents(
-                    sqlite_conn, lance_store, progress, doc_task
-                )
-
-                # Migrate chunks and embeddings
-                chunk_task = progress.add_task(
-                    "[yellow]Migrating chunks and embeddings...", total=None
-                )
-                self._migrate_chunks(
-                    sqlite_conn, lance_store, progress, chunk_task, document_id_mapping
-                )
-
-                # Migrate settings
-                settings_task = progress.add_task(
-                    "[blue]Migrating settings...", total=None
-                )
-                self._migrate_settings(
-                    sqlite_conn, lance_store, progress, settings_task
-                )
-
-            sqlite_conn.close()
-
-            # Optimize and cleanup using centralized vacuum
-            self.console.print("[cyan]Optimizing LanceDB...[/cyan]")
-            try:
-                await lance_store.vacuum()
-                self.console.print("[green]✅ Optimization completed[/green]")
-            except Exception as e:
-                self.console.print(
-                    f"[yellow]Warning: Optimization failed: {e}[/yellow]"
-                )
-
-            lance_store.close()
-
-            self.console.print("[green]✅ Migration completed successfully![/green]")
-            self.console.print(
-                f"[green]✅ Migrated {len(document_id_mapping)} documents[/green]"
-            )
-            return True
-
-        except Exception as e:
-            self.console.print(f"[red]❌ Migration failed: {e}[/red]")
-            import traceback
-
-            self.console.print(f"[red]{traceback.format_exc()}[/red]")
-            return False
-
-    def _migrate_documents(
-        self,
-        sqlite_conn: sqlite3.Connection,
-        lance_store: Store,
-        progress: Progress,
-        task: TaskID,
-    ) -> dict[int, str]:
-        """Migrate documents from SQLite to LanceDB and return ID mapping."""
-        cursor = sqlite_conn.cursor()
-        cursor.execute(
-            "SELECT id, content, uri, metadata, created_at, updated_at FROM documents ORDER BY id"
-        )
-
-        documents = []
-        id_mapping = {}  # Maps old integer ID to new UUID
-
-        for row in cursor.fetchall():
-            new_uuid = str(uuid4())
-            id_mapping[row["id"]] = new_uuid
-
-            doc_data = {
-                "id": new_uuid,
-                "content": row["content"],
-                "uri": row["uri"],
-                "metadata": json.loads(row["metadata"]) if row["metadata"] else {},
-                "created_at": row["created_at"],
-                "updated_at": row["updated_at"],
-            }
-            documents.append(doc_data)
-
-        # Batch insert documents to LanceDB
-        if documents:
-            from haiku.rag.store.engine import DocumentRecord
-
-            doc_records = [
-                DocumentRecord(
-                    id=doc["id"],
-                    content=doc["content"],
-                    uri=doc["uri"],
-                    metadata=json.dumps(doc["metadata"]),
-                    created_at=doc["created_at"],
-                    updated_at=doc["updated_at"],
-                )
-                for doc in documents
-            ]
-            lance_store.documents_table.add(doc_records)
-
-        progress.update(task, completed=len(documents), total=len(documents))
-        return id_mapping
-
-    def _migrate_chunks(
-        self,
-        sqlite_conn: sqlite3.Connection,
-        lance_store: Store,
-        progress: Progress,
-        task: TaskID,
-        document_id_mapping: dict[int, str],
-    ):
-        """Migrate chunks and embeddings from SQLite to LanceDB."""
-        cursor = sqlite_conn.cursor()
-
-        # Get chunks first
-        cursor.execute("""
-            SELECT id, document_id, content, metadata
-            FROM chunks
-            ORDER BY id
-        """)
-
-        chunks_data = cursor.fetchall()
-
-        # Get embeddings using the sqlite-vec virtual table
-        embeddings_map = {}
-        try:
-            # Use the virtual table to get embeddings properly
-            cursor.execute("""
-                SELECT chunk_id, embedding
-                FROM chunk_embeddings
-            """)
-
-            for row in cursor.fetchall():
-                chunk_id = row[0]
-                embedding_blob = row[1]
-                if embedding_blob and chunk_id not in embeddings_map:
-                    embeddings_map[chunk_id] = embedding_blob
-
-        except sqlite3.OperationalError as e:
-            self.console.print(
-                f"[yellow]Warning: Could not extract embeddings from virtual table: {e}[/yellow]"
-            )
-
-        chunks = []
-        for row in chunks_data:
-            # Generate new UUID for chunk
-            chunk_uuid = str(uuid4())
-
-            # Map the old document_id to new UUID
-            document_uuid = document_id_mapping.get(row["document_id"])
-            if not document_uuid:
-                self.console.print(
-                    f"[yellow]Warning: Document ID {row['document_id']} not found in mapping for chunk {row['id']}[/yellow]"
-                )
-                continue
-
-            # Get embedding for this chunk
-            embedding = []
-            embedding_blob = embeddings_map.get(row["id"])
-            if embedding_blob:
-                try:
-                    embedding = deserialize_sqlite_embedding(embedding_blob)
-                except Exception as e:
-                    self.console.print(
-                        f"[yellow]Warning: Failed to deserialize embedding for chunk {row['id']}: {e}[/yellow]"
-                    )
-                    # Generate a zero vector of the expected dimension
-                    embedding = [0.0] * lance_store.embedder._vector_dim
-            else:
-                # No embedding found, generate zero vector
-                embedding = [0.0] * lance_store.embedder._vector_dim
-
-            chunk_data = {
-                "id": chunk_uuid,
-                "document_id": document_uuid,
-                "content": row["content"],
-                "metadata": json.loads(row["metadata"]) if row["metadata"] else {},
-                "vector": embedding,
-            }
-            chunks.append(chunk_data)
-
-        # Batch insert chunks to LanceDB
-        if chunks:
-            chunk_records = [
-                lance_store.ChunkRecord(
-                    id=chunk["id"],
-                    document_id=chunk["document_id"],
-                    content=chunk["content"],
-                    metadata=json.dumps(chunk["metadata"]),
-                    vector=chunk["vector"],
-                )
-                for chunk in chunks
-            ]
-            lance_store.chunks_table.add(chunk_records)
-
-        progress.update(task, completed=len(chunks), total=len(chunks))
-
-    def _migrate_settings(
-        self,
-        sqlite_conn: sqlite3.Connection,
-        lance_store: Store,
-        progress: Progress,
-        task: TaskID,
-    ):
-        """Migrate settings from SQLite to LanceDB."""
-        cursor = sqlite_conn.cursor()
-
-        try:
-            cursor.execute("SELECT id, settings FROM settings WHERE id = 1")
-            row = cursor.fetchone()
-
-            if row:
-                settings_data = json.loads(row["settings"]) if row["settings"] else {}
-
-                # Update the existing settings in LanceDB (use string ID)
-                lance_store.settings_table.update(
-                    where="id = 'settings'",
-                    values={"settings": json.dumps(settings_data)},
-                )
-
-                progress.update(task, completed=1, total=1)
-            else:
-                progress.update(task, completed=0, total=0)
-
-        except sqlite3.OperationalError:
-            # Settings table doesn't exist in old SQLite database
-            self.console.print(
-                "[yellow]No settings table found in SQLite database[/yellow]"
-            )
-            progress.update(task, completed=0, total=0)
-
-
-async def migrate_sqlite_to_lancedb(
-    sqlite_path: Path, lancedb_path: Path | None = None
-) -> bool:
-    """
-    Migrate an existing SQLite database to LanceDB.
-
-    Args:
-        sqlite_path: Path to the existing SQLite database
-        lancedb_path: Path for the new LanceDB database (optional, will auto-generate if not provided)
-
-    Returns:
-        True if migration was successful, False otherwise
-    """
-    if lancedb_path is None:
-        # Auto-generate LanceDB path
-        lancedb_path = sqlite_path.parent / (sqlite_path.stem + ".lancedb")
-
-    migrator = SQLiteToLanceDBMigrator(sqlite_path, lancedb_path)
-    return await migrator.migrate()