haiku.rag 0.10.2__py3-none-any.whl → 0.19.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +172 -0
- {haiku_rag-0.10.2.dist-info → haiku_rag-0.19.3.dist-info}/METADATA +79 -51
- haiku_rag-0.19.3.dist-info/RECORD +6 -0
- {haiku_rag-0.10.2.dist-info → haiku_rag-0.19.3.dist-info}/WHEEL +1 -1
- haiku/rag/__init__.py +0 -0
- haiku/rag/app.py +0 -437
- haiku/rag/chunker.py +0 -51
- haiku/rag/cli.py +0 -466
- haiku/rag/client.py +0 -605
- haiku/rag/config.py +0 -81
- haiku/rag/embeddings/__init__.py +0 -35
- haiku/rag/embeddings/base.py +0 -15
- haiku/rag/embeddings/ollama.py +0 -17
- haiku/rag/embeddings/openai.py +0 -16
- haiku/rag/embeddings/vllm.py +0 -19
- haiku/rag/embeddings/voyageai.py +0 -17
- haiku/rag/logging.py +0 -56
- haiku/rag/mcp.py +0 -156
- haiku/rag/migration.py +0 -316
- haiku/rag/monitor.py +0 -73
- haiku/rag/qa/__init__.py +0 -15
- haiku/rag/qa/agent.py +0 -91
- haiku/rag/qa/prompts.py +0 -60
- haiku/rag/reader.py +0 -115
- haiku/rag/reranking/__init__.py +0 -34
- haiku/rag/reranking/base.py +0 -13
- haiku/rag/reranking/cohere.py +0 -34
- haiku/rag/reranking/mxbai.py +0 -28
- haiku/rag/reranking/vllm.py +0 -44
- haiku/rag/research/__init__.py +0 -20
- haiku/rag/research/common.py +0 -53
- haiku/rag/research/dependencies.py +0 -47
- haiku/rag/research/graph.py +0 -29
- haiku/rag/research/models.py +0 -70
- haiku/rag/research/nodes/evaluate.py +0 -80
- haiku/rag/research/nodes/plan.py +0 -63
- haiku/rag/research/nodes/search.py +0 -93
- haiku/rag/research/nodes/synthesize.py +0 -51
- haiku/rag/research/prompts.py +0 -114
- haiku/rag/research/state.py +0 -25
- haiku/rag/store/__init__.py +0 -4
- haiku/rag/store/engine.py +0 -269
- haiku/rag/store/models/__init__.py +0 -4
- haiku/rag/store/models/chunk.py +0 -17
- haiku/rag/store/models/document.py +0 -17
- haiku/rag/store/repositories/__init__.py +0 -9
- haiku/rag/store/repositories/chunk.py +0 -424
- haiku/rag/store/repositories/document.py +0 -237
- haiku/rag/store/repositories/settings.py +0 -155
- haiku/rag/store/upgrades/__init__.py +0 -62
- haiku/rag/store/upgrades/v0_10_1.py +0 -64
- haiku/rag/store/upgrades/v0_9_3.py +0 -112
- haiku/rag/utils.py +0 -199
- haiku_rag-0.10.2.dist-info/RECORD +0 -54
- {haiku_rag-0.10.2.dist-info → haiku_rag-0.19.3.dist-info}/entry_points.txt +0 -0
- {haiku_rag-0.10.2.dist-info → haiku_rag-0.19.3.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,155 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
|
-
from haiku.rag.config import Config
|
|
4
|
-
from haiku.rag.store.engine import SettingsRecord, Store
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class ConfigMismatchError(Exception):
|
|
8
|
-
"""Raised when stored config doesn't match current config."""
|
|
9
|
-
|
|
10
|
-
pass
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class SettingsRepository:
|
|
14
|
-
"""Repository for Settings operations."""
|
|
15
|
-
|
|
16
|
-
def __init__(self, store: Store) -> None:
|
|
17
|
-
self.store = store
|
|
18
|
-
|
|
19
|
-
async def create(self, entity: dict) -> dict:
|
|
20
|
-
"""Create settings in the database."""
|
|
21
|
-
settings_record = SettingsRecord(id="settings", settings=json.dumps(entity))
|
|
22
|
-
self.store.settings_table.add([settings_record])
|
|
23
|
-
return entity
|
|
24
|
-
|
|
25
|
-
async def get_by_id(self, entity_id: str) -> dict | None:
|
|
26
|
-
"""Get settings by ID."""
|
|
27
|
-
results = list(
|
|
28
|
-
self.store.settings_table.search()
|
|
29
|
-
.where(f"id = '{entity_id}'")
|
|
30
|
-
.limit(1)
|
|
31
|
-
.to_pydantic(SettingsRecord)
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
if not results:
|
|
35
|
-
return None
|
|
36
|
-
|
|
37
|
-
return json.loads(results[0].settings) if results[0].settings else {}
|
|
38
|
-
|
|
39
|
-
async def update(self, entity: dict) -> dict:
|
|
40
|
-
"""Update existing settings."""
|
|
41
|
-
self.store.settings_table.update(
|
|
42
|
-
where="id = 'settings'", values={"settings": json.dumps(entity)}
|
|
43
|
-
)
|
|
44
|
-
return entity
|
|
45
|
-
|
|
46
|
-
async def delete(self, entity_id: str) -> bool:
|
|
47
|
-
"""Delete settings by ID."""
|
|
48
|
-
self.store.settings_table.delete(f"id = '{entity_id}'")
|
|
49
|
-
return True
|
|
50
|
-
|
|
51
|
-
async def list_all(
|
|
52
|
-
self, limit: int | None = None, offset: int | None = None
|
|
53
|
-
) -> list[dict]:
|
|
54
|
-
"""List all settings."""
|
|
55
|
-
results = list(self.store.settings_table.search().to_pydantic(SettingsRecord))
|
|
56
|
-
return [
|
|
57
|
-
json.loads(record.settings) if record.settings else {} for record in results
|
|
58
|
-
]
|
|
59
|
-
|
|
60
|
-
def get_current_settings(self) -> dict:
|
|
61
|
-
"""Get the current settings."""
|
|
62
|
-
results = list(
|
|
63
|
-
self.store.settings_table.search()
|
|
64
|
-
.where("id = 'settings'")
|
|
65
|
-
.limit(1)
|
|
66
|
-
.to_pydantic(SettingsRecord)
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
if not results:
|
|
70
|
-
return {}
|
|
71
|
-
|
|
72
|
-
return json.loads(results[0].settings) if results[0].settings else {}
|
|
73
|
-
|
|
74
|
-
def save_current_settings(self) -> None:
|
|
75
|
-
"""Save the current configuration to the database."""
|
|
76
|
-
current_config = Config.model_dump(mode="json")
|
|
77
|
-
|
|
78
|
-
# Check if settings exist
|
|
79
|
-
existing = list(
|
|
80
|
-
self.store.settings_table.search()
|
|
81
|
-
.where("id = 'settings'")
|
|
82
|
-
.limit(1)
|
|
83
|
-
.to_pydantic(SettingsRecord)
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
if existing:
|
|
87
|
-
# Preserve existing version if present to avoid interfering with upgrade flow
|
|
88
|
-
try:
|
|
89
|
-
existing_settings = (
|
|
90
|
-
json.loads(existing[0].settings) if existing[0].settings else {}
|
|
91
|
-
)
|
|
92
|
-
except Exception:
|
|
93
|
-
existing_settings = {}
|
|
94
|
-
if "version" in existing_settings:
|
|
95
|
-
current_config["version"] = existing_settings["version"]
|
|
96
|
-
|
|
97
|
-
# Update existing settings
|
|
98
|
-
if existing_settings != current_config:
|
|
99
|
-
self.store.settings_table.update(
|
|
100
|
-
where="id = 'settings'",
|
|
101
|
-
values={"settings": json.dumps(current_config)},
|
|
102
|
-
)
|
|
103
|
-
else:
|
|
104
|
-
# Create new settings
|
|
105
|
-
settings_record = SettingsRecord(
|
|
106
|
-
id="settings", settings=json.dumps(current_config)
|
|
107
|
-
)
|
|
108
|
-
self.store.settings_table.add([settings_record])
|
|
109
|
-
|
|
110
|
-
def validate_config_compatibility(self) -> None:
|
|
111
|
-
"""Validate that the current configuration is compatible with stored settings."""
|
|
112
|
-
stored_settings = self.get_current_settings()
|
|
113
|
-
|
|
114
|
-
# If no stored settings, this is a new database - save current config and return
|
|
115
|
-
if not stored_settings:
|
|
116
|
-
self.save_current_settings()
|
|
117
|
-
return
|
|
118
|
-
|
|
119
|
-
current_config = Config.model_dump(mode="json")
|
|
120
|
-
|
|
121
|
-
# Check if embedding provider or model has changed
|
|
122
|
-
stored_provider = stored_settings.get("EMBEDDINGS_PROVIDER")
|
|
123
|
-
current_provider = current_config.get("EMBEDDINGS_PROVIDER")
|
|
124
|
-
|
|
125
|
-
stored_model = stored_settings.get("EMBEDDINGS_MODEL")
|
|
126
|
-
current_model = current_config.get("EMBEDDINGS_MODEL")
|
|
127
|
-
|
|
128
|
-
stored_vector_dim = stored_settings.get("EMBEDDINGS_VECTOR_DIM")
|
|
129
|
-
current_vector_dim = current_config.get("EMBEDDINGS_VECTOR_DIM")
|
|
130
|
-
|
|
131
|
-
# Check for incompatible changes
|
|
132
|
-
incompatible_changes = []
|
|
133
|
-
|
|
134
|
-
if stored_provider and stored_provider != current_provider:
|
|
135
|
-
incompatible_changes.append(
|
|
136
|
-
f"Embedding provider changed from '{stored_provider}' to '{current_provider}'"
|
|
137
|
-
)
|
|
138
|
-
|
|
139
|
-
if stored_model and stored_model != current_model:
|
|
140
|
-
incompatible_changes.append(
|
|
141
|
-
f"Embedding model changed from '{stored_model}' to '{current_model}'"
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
if stored_vector_dim and stored_vector_dim != current_vector_dim:
|
|
145
|
-
incompatible_changes.append(
|
|
146
|
-
f"Vector dimension changed from {stored_vector_dim} to {current_vector_dim}"
|
|
147
|
-
)
|
|
148
|
-
|
|
149
|
-
if incompatible_changes:
|
|
150
|
-
error_msg = (
|
|
151
|
-
"Database configuration is incompatible with current settings:\n"
|
|
152
|
-
+ "\n".join(f" - {change}" for change in incompatible_changes)
|
|
153
|
-
)
|
|
154
|
-
error_msg += "\n\nPlease rebuild the database using: haiku-rag rebuild"
|
|
155
|
-
raise ConfigMismatchError(error_msg)
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from collections.abc import Callable
|
|
3
|
-
from dataclasses import dataclass
|
|
4
|
-
|
|
5
|
-
from packaging.version import Version, parse
|
|
6
|
-
|
|
7
|
-
from haiku.rag.store.engine import Store
|
|
8
|
-
|
|
9
|
-
logger = logging.getLogger(__name__)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
@dataclass
|
|
13
|
-
class Upgrade:
|
|
14
|
-
"""Represents a database upgrade step."""
|
|
15
|
-
|
|
16
|
-
version: str
|
|
17
|
-
apply: Callable[[Store], None]
|
|
18
|
-
description: str = ""
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# Registry of upgrade steps (ordered by version)
|
|
22
|
-
upgrades: list[Upgrade] = []
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> None:
|
|
26
|
-
"""Run upgrades where from_version < step.version <= to_version."""
|
|
27
|
-
v_from: Version = parse(from_version)
|
|
28
|
-
v_to: Version = parse(to_version)
|
|
29
|
-
|
|
30
|
-
# Ensure that tests/development run available code upgrades even if the
|
|
31
|
-
# installed package version hasn't been bumped to include them yet.
|
|
32
|
-
if upgrades:
|
|
33
|
-
highest_step_version: Version = max(parse(u.version) for u in upgrades)
|
|
34
|
-
if highest_step_version > v_to:
|
|
35
|
-
v_to = highest_step_version
|
|
36
|
-
|
|
37
|
-
# Determine applicable steps
|
|
38
|
-
sorted_steps = sorted(upgrades, key=lambda u: parse(u.version))
|
|
39
|
-
applicable = [s for s in sorted_steps if v_from < parse(s.version) <= v_to]
|
|
40
|
-
if applicable:
|
|
41
|
-
logger.info("%d upgrade step(s) pending", len(applicable))
|
|
42
|
-
|
|
43
|
-
# Apply in ascending order
|
|
44
|
-
for idx, step in enumerate(applicable, start=1):
|
|
45
|
-
logger.info(
|
|
46
|
-
"Applying upgrade %s: %s (%d/%d)",
|
|
47
|
-
step.version,
|
|
48
|
-
step.description or "",
|
|
49
|
-
idx,
|
|
50
|
-
len(applicable),
|
|
51
|
-
)
|
|
52
|
-
step.apply(store)
|
|
53
|
-
logger.info("Completed upgrade %s", step.version)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts # noqa: E402
|
|
57
|
-
from .v0_9_3 import upgrade_order as upgrade_0_9_3_order # noqa: E402
|
|
58
|
-
from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title # noqa: E402
|
|
59
|
-
|
|
60
|
-
upgrades.append(upgrade_0_9_3_order)
|
|
61
|
-
upgrades.append(upgrade_0_9_3_fts)
|
|
62
|
-
upgrades.append(upgrade_0_10_1_add_title)
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
|
-
from lancedb.pydantic import LanceModel
|
|
4
|
-
from pydantic import Field
|
|
5
|
-
|
|
6
|
-
from haiku.rag.store.engine import Store
|
|
7
|
-
from haiku.rag.store.upgrades import Upgrade
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def _apply_add_document_title(store: Store) -> None:
|
|
11
|
-
"""Add a nullable 'title' column to the documents table."""
|
|
12
|
-
|
|
13
|
-
# Read existing rows using Arrow for schema-agnostic access
|
|
14
|
-
try:
|
|
15
|
-
docs_arrow = store.documents_table.search().to_arrow()
|
|
16
|
-
rows = docs_arrow.to_pylist()
|
|
17
|
-
except Exception:
|
|
18
|
-
rows = []
|
|
19
|
-
|
|
20
|
-
class DocumentRecordV2(LanceModel):
|
|
21
|
-
id: str
|
|
22
|
-
content: str
|
|
23
|
-
uri: str | None = None
|
|
24
|
-
title: str | None = None
|
|
25
|
-
metadata: str = Field(default="{}")
|
|
26
|
-
created_at: str = Field(default_factory=lambda: "")
|
|
27
|
-
updated_at: str = Field(default_factory=lambda: "")
|
|
28
|
-
|
|
29
|
-
# Drop and recreate documents table with the new schema
|
|
30
|
-
try:
|
|
31
|
-
store.db.drop_table("documents")
|
|
32
|
-
except Exception:
|
|
33
|
-
pass
|
|
34
|
-
|
|
35
|
-
store.documents_table = store.db.create_table("documents", schema=DocumentRecordV2)
|
|
36
|
-
|
|
37
|
-
# Reinsert previous rows with title=None
|
|
38
|
-
if rows:
|
|
39
|
-
backfilled = []
|
|
40
|
-
for row in rows:
|
|
41
|
-
backfilled.append(
|
|
42
|
-
DocumentRecordV2(
|
|
43
|
-
id=row.get("id"),
|
|
44
|
-
content=row.get("content", ""),
|
|
45
|
-
uri=row.get("uri"),
|
|
46
|
-
title=None,
|
|
47
|
-
metadata=(
|
|
48
|
-
row.get("metadata")
|
|
49
|
-
if isinstance(row.get("metadata"), str)
|
|
50
|
-
else json.dumps(row.get("metadata") or {})
|
|
51
|
-
),
|
|
52
|
-
created_at=row.get("created_at", ""),
|
|
53
|
-
updated_at=row.get("updated_at", ""),
|
|
54
|
-
)
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
store.documents_table.add(backfilled)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
upgrade_add_title = Upgrade(
|
|
61
|
-
version="0.10.1",
|
|
62
|
-
apply=_apply_add_document_title,
|
|
63
|
-
description="Add nullable 'title' column to documents table",
|
|
64
|
-
)
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
|
|
3
|
-
from lancedb.pydantic import LanceModel, Vector
|
|
4
|
-
from pydantic import Field
|
|
5
|
-
|
|
6
|
-
from haiku.rag.store.engine import Store
|
|
7
|
-
from haiku.rag.store.upgrades import Upgrade
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def _infer_vector_dim(store: Store) -> int:
|
|
11
|
-
"""Infer vector dimension from existing data; fallback to embedder config."""
|
|
12
|
-
try:
|
|
13
|
-
arrow = store.chunks_table.search().limit(1).to_arrow()
|
|
14
|
-
rows = arrow.to_pylist()
|
|
15
|
-
if rows:
|
|
16
|
-
vec = rows[0].get("vector")
|
|
17
|
-
if isinstance(vec, list) and vec:
|
|
18
|
-
return len(vec)
|
|
19
|
-
except Exception:
|
|
20
|
-
pass
|
|
21
|
-
# Fallback to configured embedder vector dim
|
|
22
|
-
return getattr(store.embedder, "_vector_dim", 1024)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def _apply_chunk_order(store: Store) -> None:
|
|
26
|
-
"""Add integer 'order' column to chunks and backfill from metadata."""
|
|
27
|
-
|
|
28
|
-
vector_dim = _infer_vector_dim(store)
|
|
29
|
-
|
|
30
|
-
class ChunkRecordV2(LanceModel):
|
|
31
|
-
id: str
|
|
32
|
-
document_id: str
|
|
33
|
-
content: str
|
|
34
|
-
metadata: str = Field(default="{}")
|
|
35
|
-
order: int = Field(default=0)
|
|
36
|
-
vector: Vector(vector_dim) = Field( # type: ignore
|
|
37
|
-
default_factory=lambda: [0.0] * vector_dim
|
|
38
|
-
)
|
|
39
|
-
|
|
40
|
-
# Read existing chunks
|
|
41
|
-
try:
|
|
42
|
-
chunks_arrow = store.chunks_table.search().to_arrow()
|
|
43
|
-
rows = chunks_arrow.to_pylist()
|
|
44
|
-
except Exception:
|
|
45
|
-
rows = []
|
|
46
|
-
|
|
47
|
-
new_chunk_records: list[ChunkRecordV2] = []
|
|
48
|
-
for row in rows:
|
|
49
|
-
md_raw = row.get("metadata") or "{}"
|
|
50
|
-
try:
|
|
51
|
-
md = json.loads(md_raw) if isinstance(md_raw, str) else md_raw
|
|
52
|
-
except Exception:
|
|
53
|
-
md = {}
|
|
54
|
-
# Extract and normalize order
|
|
55
|
-
order_val = 0
|
|
56
|
-
try:
|
|
57
|
-
if isinstance(md, dict) and "order" in md:
|
|
58
|
-
order_val = int(md["order"]) # type: ignore[arg-type]
|
|
59
|
-
except Exception:
|
|
60
|
-
order_val = 0
|
|
61
|
-
|
|
62
|
-
if isinstance(md, dict) and "order" in md:
|
|
63
|
-
md = {k: v for k, v in md.items() if k != "order"}
|
|
64
|
-
|
|
65
|
-
vec = row.get("vector") or [0.0] * vector_dim
|
|
66
|
-
|
|
67
|
-
new_chunk_records.append(
|
|
68
|
-
ChunkRecordV2(
|
|
69
|
-
id=row.get("id"),
|
|
70
|
-
document_id=row.get("document_id"),
|
|
71
|
-
content=row.get("content", ""),
|
|
72
|
-
metadata=json.dumps(md),
|
|
73
|
-
order=order_val,
|
|
74
|
-
vector=vec,
|
|
75
|
-
)
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
# Recreate chunks table with new schema
|
|
79
|
-
try:
|
|
80
|
-
store.db.drop_table("chunks")
|
|
81
|
-
except Exception:
|
|
82
|
-
pass
|
|
83
|
-
|
|
84
|
-
store.chunks_table = store.db.create_table("chunks", schema=ChunkRecordV2)
|
|
85
|
-
store.chunks_table.create_fts_index("content", replace=True)
|
|
86
|
-
|
|
87
|
-
if new_chunk_records:
|
|
88
|
-
store.chunks_table.add(new_chunk_records)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
upgrade_order = Upgrade(
|
|
92
|
-
version="0.9.3",
|
|
93
|
-
apply=_apply_chunk_order,
|
|
94
|
-
description="Add 'order' column to chunks and backfill from metadata",
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def _apply_fts_phrase_support(store: Store) -> None:
|
|
99
|
-
"""Recreate FTS index with phrase query support and no stop-word removal."""
|
|
100
|
-
try:
|
|
101
|
-
store.chunks_table.create_fts_index(
|
|
102
|
-
"content", replace=True, with_position=True, remove_stop_words=False
|
|
103
|
-
)
|
|
104
|
-
except Exception:
|
|
105
|
-
pass
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
upgrade_fts_phrase = Upgrade(
|
|
109
|
-
version="0.9.3",
|
|
110
|
-
apply=_apply_fts_phrase_support,
|
|
111
|
-
description="Enable FTS phrase queries (with positions) and keep stop-words",
|
|
112
|
-
)
|
haiku/rag/utils.py
DELETED
|
@@ -1,199 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import importlib
|
|
3
|
-
import importlib.util
|
|
4
|
-
import sys
|
|
5
|
-
from collections.abc import Callable
|
|
6
|
-
from functools import wraps
|
|
7
|
-
from importlib import metadata
|
|
8
|
-
from io import BytesIO
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from types import ModuleType
|
|
11
|
-
|
|
12
|
-
from packaging.version import Version, parse
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def debounce(wait: float) -> Callable:
|
|
16
|
-
"""
|
|
17
|
-
A decorator to debounce a function, ensuring it is called only after a specified delay
|
|
18
|
-
and always executes after the last call.
|
|
19
|
-
|
|
20
|
-
Args:
|
|
21
|
-
wait (float): The debounce delay in seconds.
|
|
22
|
-
|
|
23
|
-
Returns:
|
|
24
|
-
Callable: The decorated function.
|
|
25
|
-
"""
|
|
26
|
-
|
|
27
|
-
def decorator(func: Callable) -> Callable:
|
|
28
|
-
last_call = None
|
|
29
|
-
task = None
|
|
30
|
-
|
|
31
|
-
@wraps(func)
|
|
32
|
-
async def debounced(*args, **kwargs):
|
|
33
|
-
nonlocal last_call, task
|
|
34
|
-
last_call = asyncio.get_event_loop().time()
|
|
35
|
-
|
|
36
|
-
if task:
|
|
37
|
-
task.cancel()
|
|
38
|
-
|
|
39
|
-
async def call_func():
|
|
40
|
-
await asyncio.sleep(wait)
|
|
41
|
-
if asyncio.get_event_loop().time() - last_call >= wait: # type: ignore
|
|
42
|
-
await func(*args, **kwargs)
|
|
43
|
-
|
|
44
|
-
task = asyncio.create_task(call_func())
|
|
45
|
-
|
|
46
|
-
return debounced
|
|
47
|
-
|
|
48
|
-
return decorator
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def get_default_data_dir() -> Path:
|
|
52
|
-
"""Get the user data directory for the current system platform.
|
|
53
|
-
|
|
54
|
-
Linux: ~/.local/share/haiku.rag
|
|
55
|
-
macOS: ~/Library/Application Support/haiku.rag
|
|
56
|
-
Windows: C:/Users/<USER>/AppData/Roaming/haiku.rag
|
|
57
|
-
|
|
58
|
-
Returns:
|
|
59
|
-
User Data Path.
|
|
60
|
-
"""
|
|
61
|
-
home = Path.home()
|
|
62
|
-
|
|
63
|
-
system_paths = {
|
|
64
|
-
"win32": home / "AppData/Roaming/haiku.rag",
|
|
65
|
-
"linux": home / ".local/share/haiku.rag",
|
|
66
|
-
"darwin": home / "Library/Application Support/haiku.rag",
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
data_path = system_paths[sys.platform]
|
|
70
|
-
return data_path
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
async def is_up_to_date() -> tuple[bool, Version, Version]:
|
|
74
|
-
"""Check whether haiku.rag is current.
|
|
75
|
-
|
|
76
|
-
Returns:
|
|
77
|
-
A tuple containing a boolean indicating whether haiku.rag is current,
|
|
78
|
-
the running version and the latest version.
|
|
79
|
-
"""
|
|
80
|
-
|
|
81
|
-
# Lazy import to avoid pulling httpx (and its deps) on module import
|
|
82
|
-
import httpx
|
|
83
|
-
|
|
84
|
-
async with httpx.AsyncClient() as client:
|
|
85
|
-
running_version = parse(metadata.version("haiku.rag"))
|
|
86
|
-
try:
|
|
87
|
-
response = await client.get("https://pypi.org/pypi/haiku.rag/json")
|
|
88
|
-
data = response.json()
|
|
89
|
-
pypi_version = parse(data["info"]["version"])
|
|
90
|
-
except Exception:
|
|
91
|
-
# If no network connection, do not raise alarms.
|
|
92
|
-
pypi_version = running_version
|
|
93
|
-
return running_version >= pypi_version, running_version, pypi_version
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def text_to_docling_document(text: str, name: str = "content.md"):
|
|
97
|
-
"""Convert text content to a DoclingDocument.
|
|
98
|
-
|
|
99
|
-
Args:
|
|
100
|
-
text: The text content to convert.
|
|
101
|
-
name: The name to use for the document stream (defaults to "content.md").
|
|
102
|
-
|
|
103
|
-
Returns:
|
|
104
|
-
A DoclingDocument created from the text content.
|
|
105
|
-
"""
|
|
106
|
-
# Lazy import docling deps to keep import-time light
|
|
107
|
-
from docling.document_converter import DocumentConverter # type: ignore
|
|
108
|
-
from docling_core.types.io import DocumentStream # type: ignore
|
|
109
|
-
|
|
110
|
-
bytes_io = BytesIO(text.encode("utf-8"))
|
|
111
|
-
doc_stream = DocumentStream(name=name, stream=bytes_io)
|
|
112
|
-
converter = DocumentConverter()
|
|
113
|
-
result = converter.convert(doc_stream)
|
|
114
|
-
return result.document
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
def load_callable(path: str):
|
|
118
|
-
"""Load a callable from a dotted path or file path.
|
|
119
|
-
|
|
120
|
-
Supported formats:
|
|
121
|
-
- "package.module:func" or "package.module.func"
|
|
122
|
-
- "path/to/file.py:func"
|
|
123
|
-
|
|
124
|
-
Returns the loaded callable. Raises ValueError on failure.
|
|
125
|
-
"""
|
|
126
|
-
if not path:
|
|
127
|
-
raise ValueError("Empty callable path provided")
|
|
128
|
-
|
|
129
|
-
module_part = None
|
|
130
|
-
func_name = None
|
|
131
|
-
|
|
132
|
-
if ":" in path:
|
|
133
|
-
module_part, func_name = path.split(":", 1)
|
|
134
|
-
else:
|
|
135
|
-
# split by last dot for module.attr
|
|
136
|
-
if "." in path:
|
|
137
|
-
module_part, func_name = path.rsplit(".", 1)
|
|
138
|
-
else:
|
|
139
|
-
raise ValueError(
|
|
140
|
-
"Invalid callable path format. Use 'module:func' or 'module.func' or 'file.py:func'."
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
# Try file path first
|
|
144
|
-
mod: ModuleType | None = None
|
|
145
|
-
module_path = Path(module_part)
|
|
146
|
-
if module_path.suffix == ".py" and module_path.exists():
|
|
147
|
-
spec = importlib.util.spec_from_file_location(module_path.stem, module_path)
|
|
148
|
-
if spec and spec.loader:
|
|
149
|
-
mod = importlib.util.module_from_spec(spec)
|
|
150
|
-
spec.loader.exec_module(mod)
|
|
151
|
-
else:
|
|
152
|
-
# Import as a module path
|
|
153
|
-
try:
|
|
154
|
-
mod = importlib.import_module(module_part)
|
|
155
|
-
except Exception as e:
|
|
156
|
-
raise ValueError(f"Failed to import module '{module_part}': {e}")
|
|
157
|
-
|
|
158
|
-
if not hasattr(mod, func_name):
|
|
159
|
-
raise ValueError(f"Callable '{func_name}' not found in module '{module_part}'")
|
|
160
|
-
func = getattr(mod, func_name)
|
|
161
|
-
if not callable(func):
|
|
162
|
-
raise ValueError(
|
|
163
|
-
f"Attribute '{func_name}' in module '{module_part}' is not callable"
|
|
164
|
-
)
|
|
165
|
-
return func
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
def prefetch_models():
|
|
169
|
-
"""Prefetch runtime models (Docling + Ollama as configured)."""
|
|
170
|
-
import httpx
|
|
171
|
-
from docling.utils.model_downloader import download_models
|
|
172
|
-
|
|
173
|
-
from haiku.rag.config import Config
|
|
174
|
-
|
|
175
|
-
download_models()
|
|
176
|
-
|
|
177
|
-
# Collect Ollama models from config
|
|
178
|
-
required_models: set[str] = set()
|
|
179
|
-
if Config.EMBEDDINGS_PROVIDER == "ollama":
|
|
180
|
-
required_models.add(Config.EMBEDDINGS_MODEL)
|
|
181
|
-
if Config.QA_PROVIDER == "ollama":
|
|
182
|
-
required_models.add(Config.QA_MODEL)
|
|
183
|
-
if Config.RESEARCH_PROVIDER == "ollama":
|
|
184
|
-
required_models.add(Config.RESEARCH_MODEL)
|
|
185
|
-
if Config.RERANK_PROVIDER == "ollama":
|
|
186
|
-
required_models.add(Config.RERANK_MODEL)
|
|
187
|
-
|
|
188
|
-
if not required_models:
|
|
189
|
-
return
|
|
190
|
-
|
|
191
|
-
base_url = Config.OLLAMA_BASE_URL
|
|
192
|
-
|
|
193
|
-
with httpx.Client(timeout=None) as client:
|
|
194
|
-
for model in sorted(required_models):
|
|
195
|
-
with client.stream(
|
|
196
|
-
"POST", f"{base_url}/api/pull", json={"model": model}
|
|
197
|
-
) as r:
|
|
198
|
-
for _ in r.iter_lines():
|
|
199
|
-
pass
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
haiku/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
haiku/rag/app.py,sha256=-nhri6MLIZvp8DEndfeixTCVj6kutWwfLguyAcNe9S4,18023
|
|
3
|
-
haiku/rag/chunker.py,sha256=PVe6ysv8UlacUd4Zb3_8RFWIaWDXnzBAy2VDJ4TaUsE,1555
|
|
4
|
-
haiku/rag/cli.py,sha256=wreAxyXSRnn7f09t9SGe4uAXQjlieUQIpNpOapJT7y8,12910
|
|
5
|
-
haiku/rag/client.py,sha256=iUaa6YUac3CXFniIm8DsaaNsiyHsi4cp8-fPhF5XuVU,22925
|
|
6
|
-
haiku/rag/config.py,sha256=SEV2OzaKavYwHZ0LmRzBj-0dbI6YFIRuNiTw9el7SO0,2307
|
|
7
|
-
haiku/rag/logging.py,sha256=dm65AwADpcQsH5OAPtRA-4hsw0w5DK-sGOvzYkj6jzw,1720
|
|
8
|
-
haiku/rag/mcp.py,sha256=H7XibtSNUviFeaJVsXzHiRqUm0nJCpA7A1QHuBv6SKQ,5057
|
|
9
|
-
haiku/rag/migration.py,sha256=zm0-60PiS1hIQnZz65B7qfsgM7GwZVXFqMFowjpVBs8,11058
|
|
10
|
-
haiku/rag/monitor.py,sha256=r386nkhdlsU8UECwIuVwnrSlgMk3vNIuUZGNIzkZuec,2770
|
|
11
|
-
haiku/rag/reader.py,sha256=qkPTMJuQ_o4sK-8zpDl9WFYe_MJ7aL_gUw6rczIpW-g,3274
|
|
12
|
-
haiku/rag/utils.py,sha256=dBzhKaOHI9KRiJqHErcXUnqtnXY2AgOK8PCLA3rhO0A,6115
|
|
13
|
-
haiku/rag/embeddings/__init__.py,sha256=44IfDITGIFTflGT6UEmiYOwpWFVbYv5smLY59D0YeCs,1419
|
|
14
|
-
haiku/rag/embeddings/base.py,sha256=BnSviKrlzjv3L0sZJs_T-pxfawd-bcTak-rsX-D2f3A,497
|
|
15
|
-
haiku/rag/embeddings/ollama.py,sha256=LuLlHH6RGoO9_gFCIlbmesuXOj017gTw6z-p8Ez0CfE,595
|
|
16
|
-
haiku/rag/embeddings/openai.py,sha256=fIFCk-jpUtaW0xsnrQnJ824O0UCjaGG2sgvBzREhilc,503
|
|
17
|
-
haiku/rag/embeddings/vllm.py,sha256=vhaUnCn6VMkfSluLhWKtSV-sekFaPsp4pKo2N7-SBCY,626
|
|
18
|
-
haiku/rag/embeddings/voyageai.py,sha256=UW-MW4tJKnPB6Fs2P7A3yt-ZeRm46H9npckchSriPX8,661
|
|
19
|
-
haiku/rag/qa/__init__.py,sha256=Sl7Kzrg9CuBOcMF01wc1NtQhUNWjJI0MhIHfCWrb8V4,434
|
|
20
|
-
haiku/rag/qa/agent.py,sha256=rtUkEmnD8lMHIxpPPVY6TdmF4aSlZnLjad5eDefrlBw,3145
|
|
21
|
-
haiku/rag/qa/prompts.py,sha256=Lqwn3m4zCsu_CJiC4s9cLsuPNbb9nq6j2PqEF3lw1eA,3380
|
|
22
|
-
haiku/rag/reranking/__init__.py,sha256=IRXHs4qPu6VbGJQpzSwhgtVWWumURH_vEoVFE-extlo,894
|
|
23
|
-
haiku/rag/reranking/base.py,sha256=LM9yUSSJ414UgBZhFTgxGprlRqzfTe4I1vgjricz2JY,405
|
|
24
|
-
haiku/rag/reranking/cohere.py,sha256=1iTdiaa8vvb6oHVB2qpWzUOVkyfUcimVSZp6Qr4aq4c,1049
|
|
25
|
-
haiku/rag/reranking/mxbai.py,sha256=uveGFIdmNmepd2EQsvYr64wv0ra2_wB845hdSZXy5Cw,908
|
|
26
|
-
haiku/rag/reranking/vllm.py,sha256=xVGH9ss-ISWdJ5SKUUHUbTqBo7PIEmA_SQv0ScdJ6XA,1479
|
|
27
|
-
haiku/rag/research/__init__.py,sha256=t4JAmIXcKaWqvpFGX5yaehsNrfblskEMn-4mDmdKn9c,502
|
|
28
|
-
haiku/rag/research/common.py,sha256=EUnsA6VZ3-WMweXESuUYezH1ALit8N38064bsZFqtBE,1688
|
|
29
|
-
haiku/rag/research/dependencies.py,sha256=ZiSQdV6jHti4DuUp4WCaJL73TqYDr5vC8ppB34M2cNg,1639
|
|
30
|
-
haiku/rag/research/graph.py,sha256=m3vDP1nPXWzfS7VeTQzmTOk-lFpoaTvKHvRIF2mbxvs,798
|
|
31
|
-
haiku/rag/research/models.py,sha256=Q92oxBNq3qp3DyUzTim9YGDOBtGzXH25K_mmfLAA7Y8,2329
|
|
32
|
-
haiku/rag/research/prompts.py,sha256=0_EMA5CS7O37QhKJM7OCDdrdgMcoF2DgehBHR4L7xmk,5103
|
|
33
|
-
haiku/rag/research/state.py,sha256=vFwO8c2JmwwfkELE5Mwjt9Oat-bHn5tayf31MIG2SRs,623
|
|
34
|
-
haiku/rag/research/nodes/evaluate.py,sha256=Cp2J-jXYZothiQV3zRZFaCsBLaUU0Tm_-ri-hlgQQII,2897
|
|
35
|
-
haiku/rag/research/nodes/plan.py,sha256=9AkTls01Q3zTLKGgIgSCX9X4VYC8IWjEWii8A_f77YQ,2439
|
|
36
|
-
haiku/rag/research/nodes/search.py,sha256=2ioc5Ba3ciq2zpFxgzoGkZOvVsJ1TBX9zseURLDJpBg,3591
|
|
37
|
-
haiku/rag/research/nodes/synthesize.py,sha256=4acKduqWnE11ML7elUksKLozxzWJTkBLSJ2li_YMxgY,1736
|
|
38
|
-
haiku/rag/store/__init__.py,sha256=R2IRcxtkFDxqa2sgMirqLq3l2-FPdWr6ydYStaqm5OQ,104
|
|
39
|
-
haiku/rag/store/engine.py,sha256=BceAeTpDgV92B1A3GVcjsTwlD-c0cZPPvGiXW2Gola0,10215
|
|
40
|
-
haiku/rag/store/models/__init__.py,sha256=kc7Ctf53Jr483tk4QTIrcgqBbXDz4ZoeYSkFXfPnpks,89
|
|
41
|
-
haiku/rag/store/models/chunk.py,sha256=3EuZav4QekJIeHBCub48EM8SjNX8HEJ6wVDXGot4PEQ,421
|
|
42
|
-
haiku/rag/store/models/document.py,sha256=cZXy_jEti-hnhq7FKhuhCfd99ccY9fIHMLovB_Thbb8,425
|
|
43
|
-
haiku/rag/store/repositories/__init__.py,sha256=Olv5dLfBQINRV3HrsfUpjzkZ7Qm7goEYyMNykgo_DaY,291
|
|
44
|
-
haiku/rag/store/repositories/chunk.py,sha256=UfajEWf5VmMuSozGRDlWBjJNR0ngvOVFDrp6_augzBg,15217
|
|
45
|
-
haiku/rag/store/repositories/document.py,sha256=C9GbIl8sa2-Djaml4hlaPTtjV2HwHaz_Wzs35sdbdhg,7876
|
|
46
|
-
haiku/rag/store/repositories/settings.py,sha256=7XMBMavU8zRgdBoQzQg0Obfa7UKjuVnBugidTC6sEW0,5548
|
|
47
|
-
haiku/rag/store/upgrades/__init__.py,sha256=RQ8A6rEXBASLb5PD9vdDnEas_m_GgRzzdVu4B88Snqc,1975
|
|
48
|
-
haiku/rag/store/upgrades/v0_10_1.py,sha256=qNGnxj6hoHaHJ1rKTiALfw0c9NQOi0KAK-VZCD_073A,1959
|
|
49
|
-
haiku/rag/store/upgrades/v0_9_3.py,sha256=NrjNilQSgDtFWRbL3ZUtzQzJ8tf9u0dDRJtnDFwwbdw,3322
|
|
50
|
-
haiku_rag-0.10.2.dist-info/METADATA,sha256=Q-yJHHYbis9EtHkAW6I9IhATxh8a2XOinzLj6Z40NZY,5973
|
|
51
|
-
haiku_rag-0.10.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
52
|
-
haiku_rag-0.10.2.dist-info/entry_points.txt,sha256=G1U3nAkNd5YDYd4v0tuYFbriz0i-JheCsFuT9kIoGCI,48
|
|
53
|
-
haiku_rag-0.10.2.dist-info/licenses/LICENSE,sha256=eXZrWjSk9PwYFNK9yUczl3oPl95Z4V9UXH7bPN46iPo,1065
|
|
54
|
-
haiku_rag-0.10.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|