haiku.rag 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of haiku.rag might be problematic. Click here for more details.
- haiku/rag/app.py +1 -1
- haiku/rag/cli.py +18 -1
- haiku/rag/client.py +23 -21
- haiku/rag/qa/anthropic.py +57 -63
- haiku/rag/qa/ollama.py +37 -40
- haiku/rag/qa/openai.py +54 -55
- haiku/rag/qa/prompts.py +18 -5
- haiku/rag/store/engine.py +95 -9
- haiku/rag/store/repositories/settings.py +78 -0
- haiku/rag/store/upgrades/__init__.py +3 -0
- haiku/rag/store/upgrades/v0_3_4.py +26 -0
- haiku/rag/utils.py +55 -0
- {haiku_rag-0.3.2.dist-info → haiku_rag-0.3.4.dist-info}/METADATA +2 -1
- {haiku_rag-0.3.2.dist-info → haiku_rag-0.3.4.dist-info}/RECORD +17 -14
- {haiku_rag-0.3.2.dist-info → haiku_rag-0.3.4.dist-info}/WHEEL +0 -0
- {haiku_rag-0.3.2.dist-info → haiku_rag-0.3.4.dist-info}/entry_points.txt +0 -0
- {haiku_rag-0.3.2.dist-info → haiku_rag-0.3.4.dist-info}/licenses/LICENSE +0 -0
haiku/rag/app.py
CHANGED
|
@@ -74,7 +74,7 @@ class HaikuRAGApp:
|
|
|
74
74
|
self.console.print(f"[red]Error: {e}[/red]")
|
|
75
75
|
|
|
76
76
|
async def rebuild(self):
|
|
77
|
-
async with HaikuRAG(db_path=self.db_path) as client:
|
|
77
|
+
async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
|
|
78
78
|
try:
|
|
79
79
|
documents = await client.list_documents()
|
|
80
80
|
total_docs = len(documents)
|
haiku/rag/cli.py
CHANGED
|
@@ -5,7 +5,7 @@ import typer
|
|
|
5
5
|
from rich.console import Console
|
|
6
6
|
|
|
7
7
|
from haiku.rag.app import HaikuRAGApp
|
|
8
|
-
from haiku.rag.utils import get_default_data_dir
|
|
8
|
+
from haiku.rag.utils import get_default_data_dir, is_up_to_date
|
|
9
9
|
|
|
10
10
|
cli = typer.Typer(
|
|
11
11
|
context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
|
|
@@ -15,6 +15,23 @@ console = Console()
|
|
|
15
15
|
event_loop = asyncio.get_event_loop()
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
async def check_version():
|
|
19
|
+
"""Check if haiku.rag is up to date and show warning if not."""
|
|
20
|
+
up_to_date, current_version, latest_version = await is_up_to_date()
|
|
21
|
+
if not up_to_date:
|
|
22
|
+
console.print(
|
|
23
|
+
f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]"
|
|
24
|
+
)
|
|
25
|
+
console.print("[yellow]Please update.[/yellow]")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@cli.callback()
|
|
29
|
+
def main():
|
|
30
|
+
"""haiku.rag CLI - SQLite-based RAG system"""
|
|
31
|
+
# Run version check before any command
|
|
32
|
+
event_loop.run_until_complete(check_version())
|
|
33
|
+
|
|
34
|
+
|
|
18
35
|
@cli.command("list", help="List all stored documents")
|
|
19
36
|
def list_documents(
|
|
20
37
|
db: Path = typer.Option(
|
haiku/rag/client.py
CHANGED
|
@@ -24,12 +24,13 @@ class HaikuRAG:
|
|
|
24
24
|
self,
|
|
25
25
|
db_path: Path | Literal[":memory:"] = Config.DEFAULT_DATA_DIR
|
|
26
26
|
/ "haiku.rag.sqlite",
|
|
27
|
+
skip_validation: bool = False,
|
|
27
28
|
):
|
|
28
29
|
"""Initialize the RAG client with a database path."""
|
|
29
30
|
if isinstance(db_path, Path):
|
|
30
31
|
if not db_path.parent.exists():
|
|
31
32
|
Path.mkdir(db_path.parent, parents=True)
|
|
32
|
-
self.store = Store(db_path)
|
|
33
|
+
self.store = Store(db_path, skip_validation=skip_validation)
|
|
33
34
|
self.document_repository = DocumentRepository(self.store)
|
|
34
35
|
self.chunk_repository = ChunkRepository(self.store)
|
|
35
36
|
|
|
@@ -165,29 +166,26 @@ class HaikuRAG:
|
|
|
165
166
|
|
|
166
167
|
# Create a temporary file with the appropriate extension
|
|
167
168
|
with tempfile.NamedTemporaryFile(
|
|
168
|
-
mode="wb", suffix=file_extension
|
|
169
|
+
mode="wb", suffix=file_extension
|
|
169
170
|
) as temp_file:
|
|
170
171
|
temp_file.write(response.content)
|
|
172
|
+
temp_file.flush() # Ensure content is written to disk
|
|
171
173
|
temp_path = Path(temp_file.name)
|
|
172
174
|
|
|
173
|
-
try:
|
|
174
175
|
# Parse the content using FileReader
|
|
175
176
|
content = FileReader.parse_file(temp_path)
|
|
176
177
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
finally:
|
|
189
|
-
# Clean up temporary file
|
|
190
|
-
temp_path.unlink(missing_ok=True)
|
|
178
|
+
# Merge metadata with contentType and md5
|
|
179
|
+
metadata.update({"contentType": content_type, "md5": md5_hash})
|
|
180
|
+
|
|
181
|
+
if existing_doc:
|
|
182
|
+
existing_doc.content = content
|
|
183
|
+
existing_doc.metadata = metadata
|
|
184
|
+
return await self.update_document(existing_doc)
|
|
185
|
+
else:
|
|
186
|
+
return await self.create_document(
|
|
187
|
+
content=content, uri=url, metadata=metadata
|
|
188
|
+
)
|
|
191
189
|
|
|
192
190
|
def _get_extension_from_content_type_or_url(
|
|
193
191
|
self, url: str, content_type: str
|
|
@@ -277,12 +275,16 @@ class HaikuRAG:
|
|
|
277
275
|
Yields:
|
|
278
276
|
int: The ID of the document currently being processed
|
|
279
277
|
"""
|
|
280
|
-
|
|
278
|
+
await self.chunk_repository.delete_all()
|
|
279
|
+
self.store.recreate_embeddings_table()
|
|
281
280
|
|
|
282
|
-
|
|
283
|
-
|
|
281
|
+
# Update settings to current config
|
|
282
|
+
from haiku.rag.store.repositories.settings import SettingsRepository
|
|
284
283
|
|
|
285
|
-
|
|
284
|
+
settings_repo = SettingsRepository(self.store)
|
|
285
|
+
settings_repo.save()
|
|
286
|
+
|
|
287
|
+
documents = await self.list_documents()
|
|
286
288
|
|
|
287
289
|
for doc in documents:
|
|
288
290
|
if doc.id is not None:
|
haiku/rag/qa/anthropic.py
CHANGED
|
@@ -37,75 +37,69 @@ try:
|
|
|
37
37
|
|
|
38
38
|
messages: list[MessageParam] = [{"role": "user", "content": question}]
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
40
|
+
max_rounds = 5 # Prevent infinite loops
|
|
41
|
+
|
|
42
|
+
for _ in range(max_rounds):
|
|
43
|
+
response = await anthropic_client.messages.create(
|
|
44
|
+
model=self._model,
|
|
45
|
+
max_tokens=4096,
|
|
46
|
+
system=self._system_prompt,
|
|
47
|
+
messages=messages,
|
|
48
|
+
tools=self.tools,
|
|
49
|
+
temperature=0.0,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
if response.stop_reason == "tool_use":
|
|
53
|
+
messages.append({"role": "assistant", "content": response.content})
|
|
54
|
+
|
|
55
|
+
# Process tool calls
|
|
56
|
+
tool_results = []
|
|
57
|
+
for content_block in response.content:
|
|
58
|
+
if isinstance(content_block, ToolUseBlock):
|
|
59
|
+
if content_block.name == "search_documents":
|
|
60
|
+
args = content_block.input
|
|
61
|
+
query = (
|
|
62
|
+
args.get("query", question)
|
|
63
|
+
if isinstance(args, dict)
|
|
64
|
+
else question
|
|
65
|
+
)
|
|
66
|
+
limit = (
|
|
67
|
+
int(args.get("limit", 3))
|
|
68
|
+
if isinstance(args, dict)
|
|
69
|
+
else 3
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
search_results = await self._client.search(
|
|
73
|
+
query, limit=limit
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
context_chunks = []
|
|
77
|
+
for chunk, score in search_results:
|
|
78
|
+
context_chunks.append(
|
|
79
|
+
f"Content: {chunk.content}\nScore: {score:.4f}"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
context = "\n\n".join(context_chunks)
|
|
83
|
+
|
|
84
|
+
tool_results.append(
|
|
85
|
+
{
|
|
86
|
+
"type": "tool_result",
|
|
87
|
+
"tool_use_id": content_block.id,
|
|
88
|
+
"content": context,
|
|
89
|
+
}
|
|
77
90
|
)
|
|
78
91
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
"content": context,
|
|
86
|
-
}
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
if tool_results:
|
|
90
|
-
messages.append({"role": "user", "content": tool_results})
|
|
91
|
-
|
|
92
|
-
final_response = await anthropic_client.messages.create(
|
|
93
|
-
model=self._model,
|
|
94
|
-
max_tokens=4096,
|
|
95
|
-
system=self._system_prompt,
|
|
96
|
-
messages=messages,
|
|
97
|
-
temperature=0.0,
|
|
98
|
-
)
|
|
99
|
-
if final_response.content:
|
|
100
|
-
first_content = final_response.content[0]
|
|
92
|
+
if tool_results:
|
|
93
|
+
messages.append({"role": "user", "content": tool_results})
|
|
94
|
+
else:
|
|
95
|
+
# No tool use, return the response
|
|
96
|
+
if response.content:
|
|
97
|
+
first_content = response.content[0]
|
|
101
98
|
if isinstance(first_content, TextBlock):
|
|
102
99
|
return first_content.text
|
|
103
100
|
return ""
|
|
104
101
|
|
|
105
|
-
|
|
106
|
-
first_content = response.content[0]
|
|
107
|
-
if isinstance(first_content, TextBlock):
|
|
108
|
-
return first_content.text
|
|
102
|
+
# If we've exhausted max rounds, return empty string
|
|
109
103
|
return ""
|
|
110
104
|
|
|
111
105
|
except ImportError:
|
haiku/rag/qa/ollama.py
CHANGED
|
@@ -14,54 +14,51 @@ class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
|
|
|
14
14
|
async def answer(self, question: str) -> str:
|
|
15
15
|
ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
|
|
16
16
|
|
|
17
|
-
# Define the search tool
|
|
18
|
-
|
|
19
17
|
messages = [
|
|
20
18
|
{"role": "system", "content": self._system_prompt},
|
|
21
19
|
{"role": "user", "content": question},
|
|
22
20
|
]
|
|
23
21
|
|
|
24
|
-
|
|
25
|
-
response = await ollama_client.chat(
|
|
26
|
-
model=self._model,
|
|
27
|
-
messages=messages,
|
|
28
|
-
tools=self.tools,
|
|
29
|
-
options=OLLAMA_OPTIONS,
|
|
30
|
-
think=False,
|
|
31
|
-
)
|
|
22
|
+
max_rounds = 5 # Prevent infinite loops
|
|
32
23
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
24
|
+
for _ in range(max_rounds):
|
|
25
|
+
response = await ollama_client.chat(
|
|
26
|
+
model=self._model,
|
|
27
|
+
messages=messages,
|
|
28
|
+
tools=self.tools,
|
|
29
|
+
options=OLLAMA_OPTIONS,
|
|
30
|
+
think=False,
|
|
31
|
+
)
|
|
39
32
|
|
|
40
|
-
|
|
33
|
+
if response.get("message", {}).get("tool_calls"):
|
|
34
|
+
messages.append(response["message"])
|
|
41
35
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
)
|
|
36
|
+
for tool_call in response["message"]["tool_calls"]:
|
|
37
|
+
if tool_call["function"]["name"] == "search_documents":
|
|
38
|
+
args = tool_call["function"]["arguments"]
|
|
39
|
+
query = args.get("query", question)
|
|
40
|
+
limit = int(args.get("limit", 3))
|
|
47
41
|
|
|
48
|
-
|
|
42
|
+
search_results = await self._client.search(query, limit=limit)
|
|
49
43
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
"tool_call_id": tool_call.get("id", "search_tool"),
|
|
56
|
-
}
|
|
57
|
-
)
|
|
44
|
+
context_chunks = []
|
|
45
|
+
for chunk, score in search_results:
|
|
46
|
+
context_chunks.append(
|
|
47
|
+
f"Content: {chunk.content}\nScore: {score:.4f}"
|
|
48
|
+
)
|
|
58
49
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
50
|
+
context = "\n\n".join(context_chunks)
|
|
51
|
+
|
|
52
|
+
messages.append(
|
|
53
|
+
{
|
|
54
|
+
"role": "tool",
|
|
55
|
+
"content": context,
|
|
56
|
+
"tool_call_id": tool_call.get("id", "search_tool"),
|
|
57
|
+
}
|
|
58
|
+
)
|
|
59
|
+
else:
|
|
60
|
+
# No tool calls, return the response
|
|
61
|
+
return response["message"]["content"]
|
|
62
|
+
|
|
63
|
+
# If we've exhausted max rounds, return empty string
|
|
64
|
+
return ""
|
haiku/rag/qa/openai.py
CHANGED
|
@@ -24,8 +24,6 @@ try:
|
|
|
24
24
|
async def answer(self, question: str) -> str:
|
|
25
25
|
openai_client = AsyncOpenAI()
|
|
26
26
|
|
|
27
|
-
# Define the search tool
|
|
28
|
-
|
|
29
27
|
messages: list[ChatCompletionMessageParam] = [
|
|
30
28
|
ChatCompletionSystemMessageParam(
|
|
31
29
|
role="system", content=self._system_prompt
|
|
@@ -33,69 +31,70 @@ try:
|
|
|
33
31
|
ChatCompletionUserMessageParam(role="user", content=question),
|
|
34
32
|
]
|
|
35
33
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
response_message = response.choices[0].message
|
|
45
|
-
|
|
46
|
-
if response_message.tool_calls:
|
|
47
|
-
messages.append(
|
|
48
|
-
ChatCompletionAssistantMessageParam(
|
|
49
|
-
role="assistant",
|
|
50
|
-
content=response_message.content,
|
|
51
|
-
tool_calls=[
|
|
52
|
-
{
|
|
53
|
-
"id": tc.id,
|
|
54
|
-
"type": "function",
|
|
55
|
-
"function": {
|
|
56
|
-
"name": tc.function.name,
|
|
57
|
-
"arguments": tc.function.arguments,
|
|
58
|
-
},
|
|
59
|
-
}
|
|
60
|
-
for tc in response_message.tool_calls
|
|
61
|
-
],
|
|
62
|
-
)
|
|
34
|
+
max_rounds = 5 # Prevent infinite loops
|
|
35
|
+
|
|
36
|
+
for _ in range(max_rounds):
|
|
37
|
+
response = await openai_client.chat.completions.create(
|
|
38
|
+
model=self._model,
|
|
39
|
+
messages=messages,
|
|
40
|
+
tools=self.tools,
|
|
41
|
+
temperature=0.0,
|
|
63
42
|
)
|
|
64
43
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
44
|
+
response_message = response.choices[0].message
|
|
45
|
+
|
|
46
|
+
if response_message.tool_calls:
|
|
47
|
+
messages.append(
|
|
48
|
+
ChatCompletionAssistantMessageParam(
|
|
49
|
+
role="assistant",
|
|
50
|
+
content=response_message.content,
|
|
51
|
+
tool_calls=[
|
|
52
|
+
{
|
|
53
|
+
"id": tc.id,
|
|
54
|
+
"type": "function",
|
|
55
|
+
"function": {
|
|
56
|
+
"name": tc.function.name,
|
|
57
|
+
"arguments": tc.function.arguments,
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
for tc in response_message.tool_calls
|
|
61
|
+
],
|
|
62
|
+
)
|
|
63
|
+
)
|
|
68
64
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
65
|
+
for tool_call in response_message.tool_calls:
|
|
66
|
+
if tool_call.function.name == "search_documents":
|
|
67
|
+
import json
|
|
72
68
|
|
|
73
|
-
|
|
69
|
+
args = json.loads(tool_call.function.arguments)
|
|
70
|
+
query = args.get("query", question)
|
|
71
|
+
limit = int(args.get("limit", 3))
|
|
74
72
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
context_chunks.append(
|
|
78
|
-
f"Content: {chunk.content}\nScore: {score:.4f}"
|
|
73
|
+
search_results = await self._client.search(
|
|
74
|
+
query, limit=limit
|
|
79
75
|
)
|
|
80
76
|
|
|
81
|
-
|
|
77
|
+
context_chunks = []
|
|
78
|
+
for chunk, score in search_results:
|
|
79
|
+
context_chunks.append(
|
|
80
|
+
f"Content: {chunk.content}\nScore: {score:.4f}"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
context = "\n\n".join(context_chunks)
|
|
82
84
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
85
|
+
messages.append(
|
|
86
|
+
ChatCompletionToolMessageParam(
|
|
87
|
+
role="tool",
|
|
88
|
+
content=context,
|
|
89
|
+
tool_call_id=tool_call.id,
|
|
90
|
+
)
|
|
88
91
|
)
|
|
89
|
-
|
|
92
|
+
else:
|
|
93
|
+
# No tool calls, return the response
|
|
94
|
+
return response_message.content or ""
|
|
90
95
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
messages=messages,
|
|
94
|
-
temperature=0.0,
|
|
95
|
-
)
|
|
96
|
-
return final_response.choices[0].message.content or ""
|
|
97
|
-
else:
|
|
98
|
-
return response_message.content or ""
|
|
96
|
+
# If we've exhausted max rounds, return empty string
|
|
97
|
+
return ""
|
|
99
98
|
|
|
100
99
|
except ImportError:
|
|
101
100
|
pass
|
haiku/rag/qa/prompts.py
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
SYSTEM_PROMPT = """
|
|
2
|
-
You are a
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
You are a knowledgeable assistant that helps users find information from a document knowledge base.
|
|
3
|
+
|
|
4
|
+
Your process:
|
|
5
|
+
1. When a user asks a question, use the search_documents tool to find relevant information
|
|
6
|
+
2. Search with specific keywords and phrases from the user's question
|
|
7
|
+
3. Review the search results and their relevance scores
|
|
8
|
+
4. If you need additional context, perform follow-up searches with different keywords
|
|
9
|
+
5. Provide a comprehensive answer based only on the retrieved documents
|
|
10
|
+
|
|
11
|
+
Guidelines:
|
|
12
|
+
- Base your answers strictly on the provided document content
|
|
13
|
+
- Quote or reference specific information when possible
|
|
14
|
+
- If multiple documents contain relevant information, synthesize them coherently
|
|
15
|
+
- Indicate when information is incomplete or when you need to search for additional context
|
|
16
|
+
- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
|
|
17
|
+
- For complex questions, consider breaking them down and performing multiple searches
|
|
18
|
+
|
|
19
|
+
Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents.
|
|
7
20
|
"""
|
haiku/rag/store/engine.py
CHANGED
|
@@ -1,23 +1,65 @@
|
|
|
1
1
|
import sqlite3
|
|
2
2
|
import struct
|
|
3
|
+
from importlib import metadata
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from typing import Literal
|
|
5
6
|
|
|
6
7
|
import sqlite_vec
|
|
8
|
+
from packaging.version import parse
|
|
9
|
+
from rich.console import Console
|
|
7
10
|
|
|
11
|
+
from haiku.rag.config import Config
|
|
8
12
|
from haiku.rag.embeddings import get_embedder
|
|
13
|
+
from haiku.rag.store.upgrades import upgrades
|
|
14
|
+
from haiku.rag.utils import int_to_semantic_version, semantic_version_to_int
|
|
9
15
|
|
|
10
16
|
|
|
11
17
|
class Store:
|
|
12
|
-
def __init__(
|
|
18
|
+
def __init__(
|
|
19
|
+
self, db_path: Path | Literal[":memory:"], skip_validation: bool = False
|
|
20
|
+
):
|
|
13
21
|
self.db_path: Path | Literal[":memory:"] = db_path
|
|
14
|
-
self.
|
|
22
|
+
self.create_or_update_db()
|
|
15
23
|
|
|
16
|
-
|
|
24
|
+
# Validate config compatibility after connection is established
|
|
25
|
+
if not skip_validation:
|
|
26
|
+
from haiku.rag.store.repositories.settings import SettingsRepository
|
|
27
|
+
|
|
28
|
+
settings_repo = SettingsRepository(self)
|
|
29
|
+
settings_repo.validate_config_compatibility()
|
|
30
|
+
current_version = metadata.version("haiku.rag")
|
|
31
|
+
self.set_user_version(current_version)
|
|
32
|
+
|
|
33
|
+
def create_or_update_db(self):
|
|
17
34
|
"""Create the database and tables with sqlite-vec support for embeddings."""
|
|
35
|
+
current_version = metadata.version("haiku.rag")
|
|
36
|
+
|
|
18
37
|
db = sqlite3.connect(self.db_path)
|
|
19
38
|
db.enable_load_extension(True)
|
|
20
39
|
sqlite_vec.load(db)
|
|
40
|
+
self._connection = db
|
|
41
|
+
existing_tables = [
|
|
42
|
+
row[0]
|
|
43
|
+
for row in db.execute(
|
|
44
|
+
"SELECT name FROM sqlite_master WHERE type='table';"
|
|
45
|
+
).fetchall()
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
# If we have a db already, perform upgrades and return
|
|
49
|
+
if self.db_path != ":memory:" and "documents" in existing_tables:
|
|
50
|
+
# Upgrade database
|
|
51
|
+
console = Console()
|
|
52
|
+
db_version = self.get_user_version()
|
|
53
|
+
for version, steps in upgrades:
|
|
54
|
+
if parse(current_version) >= parse(version) and parse(version) > parse(
|
|
55
|
+
db_version
|
|
56
|
+
):
|
|
57
|
+
for step in steps:
|
|
58
|
+
step(db)
|
|
59
|
+
console.print(
|
|
60
|
+
f"[green][b]DB Upgrade: [/b]{step.__doc__}[/green]"
|
|
61
|
+
)
|
|
62
|
+
return
|
|
21
63
|
|
|
22
64
|
# Create documents table
|
|
23
65
|
db.execute("""
|
|
@@ -30,7 +72,6 @@ class Store:
|
|
|
30
72
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
31
73
|
)
|
|
32
74
|
""")
|
|
33
|
-
|
|
34
75
|
# Create chunks table
|
|
35
76
|
db.execute("""
|
|
36
77
|
CREATE TABLE IF NOT EXISTS chunks (
|
|
@@ -41,7 +82,6 @@ class Store:
|
|
|
41
82
|
FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
|
|
42
83
|
)
|
|
43
84
|
""")
|
|
44
|
-
|
|
45
85
|
# Create vector table for chunk embeddings
|
|
46
86
|
embedder = get_embedder()
|
|
47
87
|
db.execute(f"""
|
|
@@ -50,7 +90,6 @@ class Store:
|
|
|
50
90
|
embedding FLOAT[{embedder._vector_dim}]
|
|
51
91
|
)
|
|
52
92
|
""")
|
|
53
|
-
|
|
54
93
|
# Create FTS5 table for full-text search
|
|
55
94
|
db.execute("""
|
|
56
95
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -59,14 +98,61 @@ class Store:
|
|
|
59
98
|
content_rowid='id'
|
|
60
99
|
)
|
|
61
100
|
""")
|
|
62
|
-
|
|
101
|
+
# Create settings table for storing current configuration
|
|
102
|
+
db.execute("""
|
|
103
|
+
CREATE TABLE IF NOT EXISTS settings (
|
|
104
|
+
id INTEGER PRIMARY KEY DEFAULT 1,
|
|
105
|
+
settings TEXT NOT NULL DEFAULT '{}'
|
|
106
|
+
)
|
|
107
|
+
""")
|
|
108
|
+
# Save current settings to the new database
|
|
109
|
+
settings_json = Config.model_dump_json()
|
|
110
|
+
db.execute(
|
|
111
|
+
"INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)",
|
|
112
|
+
(settings_json,),
|
|
113
|
+
)
|
|
63
114
|
# Create indexes for better performance
|
|
64
115
|
db.execute(
|
|
65
116
|
"CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
|
|
66
117
|
)
|
|
67
|
-
|
|
68
118
|
db.commit()
|
|
69
|
-
|
|
119
|
+
|
|
120
|
+
def get_user_version(self) -> str:
|
|
121
|
+
"""Returns the SQLite user version"""
|
|
122
|
+
if self._connection is None:
|
|
123
|
+
raise ValueError("Store connection is not available")
|
|
124
|
+
|
|
125
|
+
cursor = self._connection.execute("PRAGMA user_version;")
|
|
126
|
+
version = cursor.fetchone()
|
|
127
|
+
return int_to_semantic_version(version[0])
|
|
128
|
+
|
|
129
|
+
def set_user_version(self, version: str) -> None:
|
|
130
|
+
"""Updates the SQLite user version"""
|
|
131
|
+
if self._connection is None:
|
|
132
|
+
raise ValueError("Store connection is not available")
|
|
133
|
+
|
|
134
|
+
self._connection.execute(
|
|
135
|
+
f"PRAGMA user_version = {semantic_version_to_int(version)};"
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
def recreate_embeddings_table(self) -> None:
|
|
139
|
+
"""Recreate the embeddings table with current vector dimensions."""
|
|
140
|
+
if self._connection is None:
|
|
141
|
+
raise ValueError("Store connection is not available")
|
|
142
|
+
|
|
143
|
+
# Drop existing embeddings table
|
|
144
|
+
self._connection.execute("DROP TABLE IF EXISTS chunk_embeddings")
|
|
145
|
+
|
|
146
|
+
# Recreate with current dimensions
|
|
147
|
+
embedder = get_embedder()
|
|
148
|
+
self._connection.execute(f"""
|
|
149
|
+
CREATE VIRTUAL TABLE chunk_embeddings USING vec0(
|
|
150
|
+
chunk_id INTEGER PRIMARY KEY,
|
|
151
|
+
embedding FLOAT[{embedder._vector_dim}]
|
|
152
|
+
)
|
|
153
|
+
""")
|
|
154
|
+
|
|
155
|
+
self._connection.commit()
|
|
70
156
|
|
|
71
157
|
@staticmethod
|
|
72
158
|
def serialize_embedding(embedding: list[float]) -> bytes:
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from haiku.rag.store.engine import Store
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ConfigMismatchError(Exception):
|
|
8
|
+
"""Raised when current config doesn't match stored settings."""
|
|
9
|
+
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SettingsRepository:
|
|
14
|
+
def __init__(self, store: Store):
|
|
15
|
+
self.store = store
|
|
16
|
+
|
|
17
|
+
def get(self) -> dict[str, Any]:
|
|
18
|
+
"""Get all settings from the database."""
|
|
19
|
+
if self.store._connection is None:
|
|
20
|
+
raise ValueError("Store connection is not available")
|
|
21
|
+
|
|
22
|
+
cursor = self.store._connection.execute("SELECT settings FROM settings LIMIT 1")
|
|
23
|
+
row = cursor.fetchone()
|
|
24
|
+
if row:
|
|
25
|
+
return json.loads(row[0])
|
|
26
|
+
return {}
|
|
27
|
+
|
|
28
|
+
def save(self) -> None:
|
|
29
|
+
"""Sync settings from the current AppConfig to database."""
|
|
30
|
+
if self.store._connection is None:
|
|
31
|
+
raise ValueError("Store connection is not available")
|
|
32
|
+
|
|
33
|
+
from haiku.rag.config import Config
|
|
34
|
+
|
|
35
|
+
settings_json = Config.model_dump_json()
|
|
36
|
+
|
|
37
|
+
self.store._connection.execute(
|
|
38
|
+
"INSERT INTO settings (id, settings) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET settings = excluded.settings",
|
|
39
|
+
(settings_json,),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
self.store._connection.commit()
|
|
43
|
+
|
|
44
|
+
def validate_config_compatibility(self) -> None:
|
|
45
|
+
"""Check if current config is compatible with stored settings.
|
|
46
|
+
|
|
47
|
+
Raises ConfigMismatchError if there are incompatible differences.
|
|
48
|
+
If no settings exist, saves current config.
|
|
49
|
+
"""
|
|
50
|
+
db_settings = self.get()
|
|
51
|
+
if not db_settings:
|
|
52
|
+
# No settings in DB, save current config
|
|
53
|
+
self.save()
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
from haiku.rag.config import Config
|
|
57
|
+
|
|
58
|
+
current_config = Config.model_dump(mode="json")
|
|
59
|
+
|
|
60
|
+
# Critical settings that must match
|
|
61
|
+
critical_settings = [
|
|
62
|
+
"EMBEDDINGS_PROVIDER",
|
|
63
|
+
"EMBEDDINGS_MODEL",
|
|
64
|
+
"EMBEDDINGS_VECTOR_DIM",
|
|
65
|
+
"CHUNK_SIZE",
|
|
66
|
+
"CHUNK_OVERLAP",
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
errors = []
|
|
70
|
+
for setting in critical_settings:
|
|
71
|
+
if db_settings.get(setting) != current_config.get(setting):
|
|
72
|
+
errors.append(
|
|
73
|
+
f"{setting}: current={current_config.get(setting)}, stored={db_settings.get(setting)}"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
if errors:
|
|
77
|
+
error_msg = f"Config mismatch detected: {'; '.join(errors)}. Consider rebuilding the database with the current configuration."
|
|
78
|
+
raise ConfigMismatchError(error_msg)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from sqlite3 import Connection
|
|
3
|
+
|
|
4
|
+
from haiku.rag.config import Config
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def add_settings_table(db: Connection) -> None:
|
|
8
|
+
"""Create settings table for storing current configuration"""
|
|
9
|
+
db.execute("""
|
|
10
|
+
CREATE TABLE settings (
|
|
11
|
+
id INTEGER PRIMARY KEY DEFAULT 1,
|
|
12
|
+
settings TEXT NOT NULL DEFAULT '{}'
|
|
13
|
+
)
|
|
14
|
+
""")
|
|
15
|
+
|
|
16
|
+
settings_json = Config.model_dump_json()
|
|
17
|
+
db.execute(
|
|
18
|
+
"INSERT INTO settings (id, settings) VALUES (1, ?)",
|
|
19
|
+
(settings_json,),
|
|
20
|
+
)
|
|
21
|
+
db.commit()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
upgrades: list[tuple[str, list[Callable[[Connection], None]]]] = [
|
|
25
|
+
("0.3.4", [add_settings_table])
|
|
26
|
+
]
|
haiku/rag/utils.py
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import sys
|
|
2
|
+
from importlib import metadata
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
5
|
+
import httpx
|
|
6
|
+
from packaging.version import Version, parse
|
|
7
|
+
|
|
4
8
|
|
|
5
9
|
def get_default_data_dir() -> Path:
|
|
6
10
|
"""
|
|
@@ -23,3 +27,54 @@ def get_default_data_dir() -> Path:
|
|
|
23
27
|
|
|
24
28
|
data_path = system_paths[sys.platform]
|
|
25
29
|
return data_path
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def semantic_version_to_int(version: str) -> int:
|
|
33
|
+
"""
|
|
34
|
+
Convert a semantic version string to an integer.
|
|
35
|
+
|
|
36
|
+
:param version: Semantic version string
|
|
37
|
+
:type version: str
|
|
38
|
+
:return: Integer representation of semantic version
|
|
39
|
+
:rtype: int
|
|
40
|
+
"""
|
|
41
|
+
major, minor, patch = version.split(".")
|
|
42
|
+
major = int(major) << 16
|
|
43
|
+
minor = int(minor) << 8
|
|
44
|
+
patch = int(patch)
|
|
45
|
+
return major + minor + patch
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def int_to_semantic_version(version: int) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Convert an integer to a semantic version string.
|
|
51
|
+
|
|
52
|
+
:param version: Integer representation of semantic version
|
|
53
|
+
:type version: int
|
|
54
|
+
:return: Semantic version string
|
|
55
|
+
:rtype: str
|
|
56
|
+
"""
|
|
57
|
+
major = version >> 16
|
|
58
|
+
minor = (version >> 8) & 255
|
|
59
|
+
patch = version & 255
|
|
60
|
+
return f"{major}.{minor}.{patch}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def is_up_to_date() -> tuple[bool, Version, Version]:
|
|
64
|
+
"""
|
|
65
|
+
Checks whether haiku.rag is current.
|
|
66
|
+
|
|
67
|
+
:return: A tuple containing a boolean indicating whether haiku.rag is current, the running version and the latest version
|
|
68
|
+
:rtype: tuple[bool, Version, Version]
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
async with httpx.AsyncClient() as client:
|
|
72
|
+
running_version = parse(metadata.version("haiku.rag"))
|
|
73
|
+
try:
|
|
74
|
+
response = await client.get("https://pypi.org/pypi/haiku.rag/json")
|
|
75
|
+
data = response.json()
|
|
76
|
+
pypi_version = parse(data["info"]["version"])
|
|
77
|
+
except Exception:
|
|
78
|
+
# If no network connection, do not raise alarms.
|
|
79
|
+
pypi_version = running_version
|
|
80
|
+
return running_version >= pypi_version, running_version, pypi_version
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: haiku.rag
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Retrieval Augmented Generation (RAG) with SQLite
|
|
5
5
|
Author-email: Yiorgis Gozadinos <ggozadinos@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -116,3 +116,4 @@ Full documentation at: https://ggozad.github.io/haiku.rag/
|
|
|
116
116
|
- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
|
|
117
117
|
- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
|
|
118
118
|
- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
|
|
119
|
+
- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
haiku/rag/app.py,sha256=
|
|
2
|
+
haiku/rag/app.py,sha256=FpLVyP1-zAq_XPmU8CPVLkuIAeuhBOGvMqhYS8RbN40,7649
|
|
3
3
|
haiku/rag/chunker.py,sha256=lSSPWgNAe7gNZL_yNLmDtqxJix4YclOiG7gbARcEpV8,1871
|
|
4
|
-
haiku/rag/cli.py,sha256=
|
|
5
|
-
haiku/rag/client.py,sha256=
|
|
4
|
+
haiku/rag/cli.py,sha256=8PC7r5odIVLyksSm_BXor2rznIZ2KDug-YhzqbFPvms,5605
|
|
5
|
+
haiku/rag/client.py,sha256=AeRXw67E1dr6ICI6EJE1q0WwZgA6ezwFw55v6QVydYk,11014
|
|
6
6
|
haiku/rag/config.py,sha256=ctD_pu7nDOieirJofhNMO-OJIONLC5myvcru9iTm_ps,1433
|
|
7
7
|
haiku/rag/logging.py,sha256=zTTGpGq5tPdcd7RpCbd9EGw1IZlQDbYkrCg9t9pqRc4,580
|
|
8
8
|
haiku/rag/mcp.py,sha256=tMN6fNX7ZtAER1R6DL1GkC9HZozTC4HzuQs199p7icI,4551
|
|
9
9
|
haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
|
|
10
10
|
haiku/rag/reader.py,sha256=S7-Z72pDvSHedvgt4-RkTOwZadG88Oed9keJ69SVITk,962
|
|
11
|
-
haiku/rag/utils.py,sha256=
|
|
11
|
+
haiku/rag/utils.py,sha256=flQqO12OIqApINYAfkg8VDXBgRDFVR_HRaIaydk_OBQ,2310
|
|
12
12
|
haiku/rag/embeddings/__init__.py,sha256=4jUPe2FyIf8BGZ7AncWSlBdNXG3URejBbnkhQf3JiD0,1505
|
|
13
13
|
haiku/rag/embeddings/base.py,sha256=PTAWKTU-Q-hXIhbRK1o6pIdpaW7DFdzJXQ0Nzc6VI-w,379
|
|
14
14
|
haiku/rag/embeddings/ollama.py,sha256=hWdrTiuJwNSRYCqP0WP-z6XXA3RBGkAiknZMsPLH0qU,441
|
|
15
15
|
haiku/rag/embeddings/openai.py,sha256=reh8AykG2f9f5hhRDmqSsjiuCPi9SsXfe2YEZFlxXk8,550
|
|
16
16
|
haiku/rag/embeddings/voyageai.py,sha256=jc0JywdLJD3Ee1MUv1m8MhWCEo0enNnVcrIBtUvD-Ss,534
|
|
17
17
|
haiku/rag/qa/__init__.py,sha256=oso98Ypti7mBLTJ6Zk71YaSJ9Rgc89QXp9RSB6zSpYs,1501
|
|
18
|
-
haiku/rag/qa/anthropic.py,sha256=
|
|
18
|
+
haiku/rag/qa/anthropic.py,sha256=6I6cf6ySNkYbmDFdy22sA8r3GO5moiiH75tJnHcgJQA,4448
|
|
19
19
|
haiku/rag/qa/base.py,sha256=4ZTM_l5FAZ9cA0f8NeqRJiUAmjatwCTmSoclFw0gTFQ,1349
|
|
20
|
-
haiku/rag/qa/ollama.py,sha256
|
|
21
|
-
haiku/rag/qa/openai.py,sha256=
|
|
22
|
-
haiku/rag/qa/prompts.py,sha256=
|
|
20
|
+
haiku/rag/qa/ollama.py,sha256=-UtNFErYlA_66g3WLU6lK38a1Y5zhAL6s_uZ5AP0TFs,2381
|
|
21
|
+
haiku/rag/qa/openai.py,sha256=dF32sGgVt8mZi5oVxByaeECs9NqLjvDiZnnpJBsrHm8,3968
|
|
22
|
+
haiku/rag/qa/prompts.py,sha256=578LJGZJ0LQ_q7ccyj5hLabtHo8Zcfw5-DiLGN9lC-w,1200
|
|
23
23
|
haiku/rag/store/__init__.py,sha256=hq0W0DAC7ysqhWSP2M2uHX8cbG6kbr-sWHxhq6qQcY0,103
|
|
24
|
-
haiku/rag/store/engine.py,sha256=
|
|
24
|
+
haiku/rag/store/engine.py,sha256=4ouAD0s-TFwEoEHjVVw_KnV6aaw5nwhe9fdT8PRXfok,6061
|
|
25
25
|
haiku/rag/store/models/__init__.py,sha256=s0E72zneGlowvZrFWaNxHYjOAUjgWdLxzdYsnvNRVlY,88
|
|
26
26
|
haiku/rag/store/models/chunk.py,sha256=lmbPOOTz-N4PXhrA5XCUxyRcSTZBo135fqkV1mwnGcE,309
|
|
27
27
|
haiku/rag/store/models/document.py,sha256=TVXVY-nQs-1vCORQEs9rA7zOtndeGC4dgCoujLAS054,396
|
|
@@ -29,8 +29,11 @@ haiku/rag/store/repositories/__init__.py,sha256=uIBhxjQh-4o3O-ck8b7BQ58qXQTuJdPv
|
|
|
29
29
|
haiku/rag/store/repositories/base.py,sha256=cm3VyQXhtxvRfk1uJHpA0fDSxMpYN-mjQmRiDiLsQ68,1008
|
|
30
30
|
haiku/rag/store/repositories/chunk.py,sha256=gik7ZPOK3gCoG6tU1pGueAZBPmJxIb7obYFUhwINrYg,16497
|
|
31
31
|
haiku/rag/store/repositories/document.py,sha256=xpWOpjHFbhVwNJ1gpusEKNY6l_Qyibg9y_bdHCwcfpk,7133
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
haiku_rag-0.3.
|
|
36
|
-
haiku_rag-0.3.
|
|
32
|
+
haiku/rag/store/repositories/settings.py,sha256=dme3_ulQdQvyF9daavSjAd-SjZ5hh0MJoxP7iXgap-A,2492
|
|
33
|
+
haiku/rag/store/upgrades/__init__.py,sha256=kKS1YWT_P-CYKhKtokOLTIFNKf9jlfjFFr8lyIMeogM,100
|
|
34
|
+
haiku/rag/store/upgrades/v0_3_4.py,sha256=GLogKZdZ40NX1vBHKdOJju7fFzNUCHoEnjSZg17Hm2U,663
|
|
35
|
+
haiku_rag-0.3.4.dist-info/METADATA,sha256=9FEVS2pZkPrRYVGd1qaMmfjyxr4fc9sHx1NTeyCbTo0,4019
|
|
36
|
+
haiku_rag-0.3.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
37
|
+
haiku_rag-0.3.4.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
|
|
38
|
+
haiku_rag-0.3.4.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
|
|
39
|
+
haiku_rag-0.3.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|