deeptutor-knowledge 1.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. deeptutor/api/routers/knowledge.py +5225 -0
  2. deeptutor/api/routers/marginnote4.py +278 -0
  3. deeptutor/capabilities/ima/__init__.py +6 -0
  4. deeptutor/capabilities/ima/binding.py +118 -0
  5. deeptutor/capabilities/ima/capability.py +92 -0
  6. deeptutor/capabilities/ima/prompts/en/system.md +33 -0
  7. deeptutor/capabilities/ima/prompts/zh/system.md +24 -0
  8. deeptutor/capabilities/ima/tools.py +476 -0
  9. deeptutor/capabilities/marginnote4/__init__.py +13 -0
  10. deeptutor/capabilities/marginnote4/binding.py +66 -0
  11. deeptutor/capabilities/marginnote4/capability.py +80 -0
  12. deeptutor/capabilities/marginnote4/models.py +173 -0
  13. deeptutor/capabilities/marginnote4/prompts/en/system.md +54 -0
  14. deeptutor/capabilities/marginnote4/prompts/zh/system.md +44 -0
  15. deeptutor/capabilities/marginnote4/store.py +562 -0
  16. deeptutor/capabilities/marginnote4/tools.py +329 -0
  17. deeptutor/capabilities/obsidian/__init__.py +6 -0
  18. deeptutor/capabilities/obsidian/binding.py +75 -0
  19. deeptutor/capabilities/obsidian/capability.py +82 -0
  20. deeptutor/capabilities/obsidian/prompts/en/system.md +32 -0
  21. deeptutor/capabilities/obsidian/prompts/zh/system.md +28 -0
  22. deeptutor/capabilities/obsidian/tools.py +399 -0
  23. deeptutor/capabilities/obsidian/vault.py +341 -0
  24. deeptutor/knowledge/__init__.py +28 -0
  25. deeptutor/knowledge/add_documents.py +734 -0
  26. deeptutor/knowledge/indexing_run.py +432 -0
  27. deeptutor/knowledge/initializer.py +448 -0
  28. deeptutor/knowledge/manager.py +2570 -0
  29. deeptutor/knowledge/module_service.py +29 -0
  30. deeptutor/knowledge/naming.py +40 -0
  31. deeptutor/knowledge/progress_events.py +45 -0
  32. deeptutor/knowledge/progress_tracker.py +355 -0
  33. deeptutor/knowledge/staging.py +69 -0
  34. deeptutor/services/config/knowledge_base_config.py +239 -0
  35. deeptutor/services/embedding/__init__.py +46 -0
  36. deeptutor/services/embedding/adapters/__init__.py +40 -0
  37. deeptutor/services/embedding/adapters/base.py +196 -0
  38. deeptutor/services/embedding/adapters/cohere.py +176 -0
  39. deeptutor/services/embedding/adapters/dashscope_native.py +242 -0
  40. deeptutor/services/embedding/adapters/gemini.py +462 -0
  41. deeptutor/services/embedding/adapters/jina.py +161 -0
  42. deeptutor/services/embedding/adapters/ollama.py +145 -0
  43. deeptutor/services/embedding/adapters/openai_compatible.py +406 -0
  44. deeptutor/services/embedding/adapters/openai_sdk.py +150 -0
  45. deeptutor/services/embedding/client.py +323 -0
  46. deeptutor/services/embedding/config.py +114 -0
  47. deeptutor/services/embedding/request_options.py +46 -0
  48. deeptutor/services/embedding/validation.py +128 -0
  49. deeptutor/services/github_source/__init__.py +5 -0
  50. deeptutor/services/github_source/client.py +164 -0
  51. deeptutor/services/github_source/sync.py +384 -0
  52. deeptutor/services/github_source/sync_service.py +72 -0
  53. deeptutor/services/rag/__init__.py +21 -0
  54. deeptutor/services/rag/embedding_binding.py +331 -0
  55. deeptutor/services/rag/embedding_signature.py +63 -0
  56. deeptutor/services/rag/eval/__init__.py +92 -0
  57. deeptutor/services/rag/eval/dataset.py +244 -0
  58. deeptutor/services/rag/eval/diff.py +312 -0
  59. deeptutor/services/rag/eval/matching.py +218 -0
  60. deeptutor/services/rag/eval/metrics.py +213 -0
  61. deeptutor/services/rag/eval/report.py +111 -0
  62. deeptutor/services/rag/eval/runner.py +309 -0
  63. deeptutor/services/rag/factory.py +369 -0
  64. deeptutor/services/rag/file_routing.py +5 -0
  65. deeptutor/services/rag/index_probe.py +327 -0
  66. deeptutor/services/rag/index_versioning.py +387 -0
  67. deeptutor/services/rag/kb_paths.py +50 -0
  68. deeptutor/services/rag/linked_kb.py +237 -0
  69. deeptutor/services/rag/pipelines/__init__.py +7 -0
  70. deeptutor/services/rag/pipelines/base.py +41 -0
  71. deeptutor/services/rag/pipelines/graphrag/__init__.py +18 -0
  72. deeptutor/services/rag/pipelines/graphrag/compatibility.py +42 -0
  73. deeptutor/services/rag/pipelines/graphrag/completion_adapter.py +251 -0
  74. deeptutor/services/rag/pipelines/graphrag/config.py +357 -0
  75. deeptutor/services/rag/pipelines/graphrag/engine.py +479 -0
  76. deeptutor/services/rag/pipelines/graphrag/errors.py +328 -0
  77. deeptutor/services/rag/pipelines/graphrag/ingestion.py +135 -0
  78. deeptutor/services/rag/pipelines/graphrag/pandas_compat.py +68 -0
  79. deeptutor/services/rag/pipelines/graphrag/pipeline.py +284 -0
  80. deeptutor/services/rag/pipelines/graphrag/provider.py +134 -0
  81. deeptutor/services/rag/pipelines/graphrag/storage.py +127 -0
  82. deeptutor/services/rag/pipelines/ima/__init__.py +28 -0
  83. deeptutor/services/rag/pipelines/ima/client.py +364 -0
  84. deeptutor/services/rag/pipelines/ima/config.py +140 -0
  85. deeptutor/services/rag/pipelines/ima/envelope.py +109 -0
  86. deeptutor/services/rag/pipelines/ima/inventory.py +209 -0
  87. deeptutor/services/rag/pipelines/ima/media.py +190 -0
  88. deeptutor/services/rag/pipelines/ima/models.py +348 -0
  89. deeptutor/services/rag/pipelines/ima/notes.py +240 -0
  90. deeptutor/services/rag/pipelines/ima/pipeline.py +238 -0
  91. deeptutor/services/rag/pipelines/ima/probe.py +105 -0
  92. deeptutor/services/rag/pipelines/ima/sources.py +105 -0
  93. deeptutor/services/rag/pipelines/ima/transport.py +130 -0
  94. deeptutor/services/rag/pipelines/kiwix/__init__.py +1 -0
  95. deeptutor/services/rag/pipelines/kiwix/client.py +369 -0
  96. deeptutor/services/rag/pipelines/kiwix/pipeline.py +75 -0
  97. deeptutor/services/rag/pipelines/lightrag/README.md +78 -0
  98. deeptutor/services/rag/pipelines/lightrag/__init__.py +15 -0
  99. deeptutor/services/rag/pipelines/lightrag/block_policy.py +266 -0
  100. deeptutor/services/rag/pipelines/lightrag/cache_reuse.py +137 -0
  101. deeptutor/services/rag/pipelines/lightrag/config.py +453 -0
  102. deeptutor/services/rag/pipelines/lightrag/engine.py +536 -0
  103. deeptutor/services/rag/pipelines/lightrag/indexing_policy.py +696 -0
  104. deeptutor/services/rag/pipelines/lightrag/ingress.py +364 -0
  105. deeptutor/services/rag/pipelines/lightrag/parser.py +99 -0
  106. deeptutor/services/rag/pipelines/lightrag/pipeline.py +692 -0
  107. deeptutor/services/rag/pipelines/lightrag/roles.py +132 -0
  108. deeptutor/services/rag/pipelines/lightrag/sidecar.py +320 -0
  109. deeptutor/services/rag/pipelines/lightrag/storage.py +338 -0
  110. deeptutor/services/rag/pipelines/lightrag/worker.py +259 -0
  111. deeptutor/services/rag/pipelines/lightrag/write_lock.py +55 -0
  112. deeptutor/services/rag/pipelines/lightrag_server/__init__.py +15 -0
  113. deeptutor/services/rag/pipelines/lightrag_server/client.py +149 -0
  114. deeptutor/services/rag/pipelines/lightrag_server/config.py +68 -0
  115. deeptutor/services/rag/pipelines/lightrag_server/pipeline.py +120 -0
  116. deeptutor/services/rag/pipelines/lightrag_server/probe.py +110 -0
  117. deeptutor/services/rag/pipelines/llamaindex/__init__.py +3 -0
  118. deeptutor/services/rag/pipelines/llamaindex/config.py +199 -0
  119. deeptutor/services/rag/pipelines/llamaindex/document_loader.py +648 -0
  120. deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.py +264 -0
  121. deeptutor/services/rag/pipelines/llamaindex/errors.py +76 -0
  122. deeptutor/services/rag/pipelines/llamaindex/exercise_lookup.py +373 -0
  123. deeptutor/services/rag/pipelines/llamaindex/ingestion.py +129 -0
  124. deeptutor/services/rag/pipelines/llamaindex/pipeline.py +556 -0
  125. deeptutor/services/rag/pipelines/llamaindex/rerank.py +130 -0
  126. deeptutor/services/rag/pipelines/llamaindex/retrievers.py +360 -0
  127. deeptutor/services/rag/pipelines/llamaindex/storage.py +655 -0
  128. deeptutor/services/rag/pipelines/llamaindex/vector_store.py +288 -0
  129. deeptutor/services/rag/pipelines/modes.py +48 -0
  130. deeptutor/services/rag/pipelines/pageindex/__init__.py +47 -0
  131. deeptutor/services/rag/pipelines/pageindex/client.py +146 -0
  132. deeptutor/services/rag/pipelines/pageindex/config.py +49 -0
  133. deeptutor/services/rag/pipelines/pageindex/pipeline.py +313 -0
  134. deeptutor/services/rag/pipelines/pageindex/reasoning.py +262 -0
  135. deeptutor/services/rag/pipelines/pageindex/storage.py +133 -0
  136. deeptutor/services/rag/pipelines/pageindex/tools.py +221 -0
  137. deeptutor/services/rag/pipelines/weknora/__init__.py +13 -0
  138. deeptutor/services/rag/pipelines/weknora/client.py +106 -0
  139. deeptutor/services/rag/pipelines/weknora/config.py +56 -0
  140. deeptutor/services/rag/pipelines/weknora/pipeline.py +74 -0
  141. deeptutor/services/rag/pipelines/weknora/probe.py +80 -0
  142. deeptutor/services/rag/preflight.py +274 -0
  143. deeptutor/services/rag/provider_binding.py +70 -0
  144. deeptutor/services/rag/query_rewrite.py +186 -0
  145. deeptutor/services/rag/service.py +379 -0
  146. deeptutor/services/rag/smart_retriever.py +81 -0
  147. deeptutor/services/rag/source_pages.py +292 -0
  148. deeptutor/services/rag/source_visuals.py +434 -0
  149. deeptutor/services/rag/visual_assets.py +430 -0
  150. deeptutor/services/rag/visual_coverage.py +165 -0
  151. deeptutor/services/web_source/__init__.py +1 -0
  152. deeptutor/services/web_source/bilingual.py +255 -0
  153. deeptutor/services/web_source/crawler.py +681 -0
  154. deeptutor/services/web_source/html_extractor.py +643 -0
  155. deeptutor/services/web_source/markdown.py +28 -0
  156. deeptutor/services/web_source/navigation.py +72 -0
  157. deeptutor/services/web_source/repository.py +640 -0
  158. deeptutor/services/web_source/robots.py +182 -0
  159. deeptutor/services/web_source/scheduler.py +336 -0
  160. deeptutor/services/web_source/snapshot_assets.py +171 -0
  161. deeptutor/services/web_source/sync.py +253 -0
  162. deeptutor/services/workspace/kb_move.py +446 -0
  163. deeptutor/tools/rag_tool.py +110 -0
  164. deeptutor_cli/kb.py +756 -0
  165. deeptutor_knowledge-1.7.0.dist-info/METADATA +15 -0
  166. deeptutor_knowledge-1.7.0.dist-info/RECORD +168 -0
  167. deeptutor_knowledge-1.7.0.dist-info/WHEEL +5 -0
  168. deeptutor_knowledge-1.7.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,387 @@
1
+ """Flat per-embedding index versions for knowledge bases.
2
+
3
+ New layout::
4
+
5
+ data/knowledge_bases/<kb_name>/
6
+ raw/ # source files (untouched)
7
+ version-1/ # LlamaIndex storage files live directly here
8
+ docstore.json
9
+ index_store.json
10
+ default__vector_store.json
11
+ graph_store.json
12
+ image__vector_store.json
13
+ meta.json # {"version", "signature", "model", ...}
14
+ version-2/
15
+ ...
16
+ metadata.json
17
+
18
+ Older layouts remain readable for backward compatibility, but all new writes
19
+ go to flat ``version-N`` directories:
20
+
21
+ * ``llamaindex_storage/`` at KB root (legacy single-store)
22
+ * ``index_versions/<signature>/llamaindex_storage`` (legacy nested versions)
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from dataclasses import asdict, dataclass
28
+ from datetime import datetime, timezone
29
+ import hashlib
30
+ import json
31
+ import logging
32
+ from pathlib import Path
33
+ import re
34
+ from typing import Any, Optional
35
+
36
+ logger = logging.getLogger(__name__)
37
+
38
+ VERSION_PREFIX = "version-"
39
+ LEGACY_VERSION_DIRNAME = "index_versions"
40
+ LEGACY_STORAGE_DIRNAME = "llamaindex_storage"
41
+ META_FILENAME = "meta.json"
42
+
43
+ _VERSION_RE = re.compile(r"^version-(\d+)$")
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class EmbeddingSignature:
48
+ """Stable identity of an embedding configuration."""
49
+
50
+ binding: str
51
+ model: str
52
+ dimension: int
53
+ base_url: str
54
+ api_version: str
55
+ role_semantics: str = ""
56
+
57
+ def hash(self) -> str:
58
+ """Short hex digest used as the stable signature."""
59
+ fields = asdict(self)
60
+ role_semantics = fields.pop("role_semantics")
61
+ if role_semantics:
62
+ fields["role_semantics"] = role_semantics
63
+ canonical = json.dumps(fields, sort_keys=True, ensure_ascii=False)
64
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
65
+
66
+
67
+ def kb_versions_dir(kb_dir: Path) -> Path:
68
+ """Return the legacy nested versions directory."""
69
+ return kb_dir / LEGACY_VERSION_DIRNAME
70
+
71
+
72
+ def legacy_storage_dir(kb_dir: Path) -> Path:
73
+ return kb_dir / LEGACY_STORAGE_DIRNAME
74
+
75
+
76
+ def _legacy_version_dir_for_signature(kb_dir: Path, sig_hash: str) -> Path:
77
+ return kb_versions_dir(kb_dir) / sig_hash
78
+
79
+
80
+ def _legacy_storage_dir_for_signature(kb_dir: Path, sig_hash: str) -> Path:
81
+ return _legacy_version_dir_for_signature(kb_dir, sig_hash) / LEGACY_STORAGE_DIRNAME
82
+
83
+
84
+ def _is_storage_ready(storage_dir: Path) -> bool:
85
+ return storage_dir.is_dir() and any(
86
+ child.name != META_FILENAME for child in storage_dir.iterdir()
87
+ )
88
+
89
+
90
+ def _read_json(path: Path) -> Optional[dict[str, Any]]:
91
+ if not path.exists():
92
+ return None
93
+ try:
94
+ with open(path, encoding="utf-8") as handle:
95
+ payload = json.load(handle)
96
+ return payload if isinstance(payload, dict) else None
97
+ except Exception as exc:
98
+ logger.warning(f"Failed to read version meta {path}: {exc}")
99
+ return None
100
+
101
+
102
+ def _flat_version_number(path: Path) -> int:
103
+ match = _VERSION_RE.match(path.name)
104
+ return int(match.group(1)) if match else 0
105
+
106
+
107
+ def _flat_version_dirs(kb_dir: Path) -> list[Path]:
108
+ if not kb_dir.is_dir():
109
+ return []
110
+ return sorted(
111
+ (child for child in kb_dir.iterdir() if child.is_dir() and _VERSION_RE.match(child.name)),
112
+ key=_flat_version_number,
113
+ )
114
+
115
+
116
+ def _next_flat_version_dir(kb_dir: Path) -> Path:
117
+ existing = [_flat_version_number(path) for path in _flat_version_dirs(kb_dir)]
118
+ return kb_dir / f"{VERSION_PREFIX}{(max(existing) if existing else 0) + 1}"
119
+
120
+
121
+ def _native_lightrag_output_ready(version_dir: Path, meta: dict[str, Any]) -> bool:
122
+ workspace = str(meta.get("workspace") or "").strip()
123
+ if workspace:
124
+ workspace_path = Path(workspace)
125
+ if workspace_path.is_absolute() or workspace_path.name != workspace:
126
+ return False
127
+ status_path = version_dir / workspace / "kv_store_doc_status.json"
128
+ else:
129
+ # Transitional/test layouts may still keep the status store flat.
130
+ status_path = version_dir / "kv_store_doc_status.json"
131
+ payload = _read_json(status_path)
132
+ if not payload:
133
+ return False
134
+ return any(
135
+ isinstance(row, dict)
136
+ and (str(row.get("status") or "").lower() == "processed" or bool(row.get("chunks_list")))
137
+ for row in payload.values()
138
+ )
139
+
140
+
141
+ def _entry_from_flat_version(version_dir: Path) -> dict[str, Any]:
142
+ stored_meta = _read_json(version_dir / META_FILENAME)
143
+ meta = dict(stored_meta or {})
144
+ meta.setdefault("version", version_dir.name)
145
+ meta.setdefault("signature", meta.get("signature") or version_dir.name)
146
+ has_provider_output = _is_storage_ready(version_dir)
147
+ # Legacy flat stores from other providers predate metadata, so their
148
+ # established readiness rule remains output-based. A native LightRAG
149
+ # candidate is identifiable before publication by its doc-status/ingress
150
+ # files and must fail closed until schema-2 metadata is atomically written.
151
+ is_unpublished_lightrag_candidate = stored_meta is None and (
152
+ (version_dir / "kv_store_doc_status.json").exists()
153
+ or (version_dir / "deeptutor_ingress").exists()
154
+ )
155
+ ready = has_provider_output and not is_unpublished_lightrag_candidate
156
+ if (version_dir / ".building.json").exists():
157
+ ready = ready and bool(stored_meta and stored_meta.get("state") == "published")
158
+ if stored_meta is not None and stored_meta.get("provider") == "lightrag":
159
+ adapter_schema = stored_meta.get("lightrag_adapter_schema")
160
+ if adapter_schema is not None:
161
+ ready = bool(
162
+ adapter_schema == 2
163
+ and stored_meta.get("state") == "published"
164
+ and _native_lightrag_output_ready(version_dir, stored_meta)
165
+ )
166
+ meta["ready"] = ready
167
+ meta["storage_path"] = str(version_dir)
168
+ meta["version_path"] = str(version_dir)
169
+ meta["layout"] = "flat"
170
+ return meta
171
+
172
+
173
+ def _entry_from_legacy_nested(kb_dir: Path, version_dir: Path) -> dict[str, Any]:
174
+ storage = version_dir / LEGACY_STORAGE_DIRNAME
175
+ meta = _read_json(version_dir / META_FILENAME) or {"signature": version_dir.name}
176
+ meta.setdefault("version", version_dir.name)
177
+ meta.setdefault("signature", version_dir.name)
178
+ meta["ready"] = _is_storage_ready(storage)
179
+ meta["storage_path"] = str(storage)
180
+ meta["version_path"] = str(version_dir)
181
+ meta["layout"] = "nested_legacy"
182
+ meta["legacy"] = True
183
+ # Keep the parent KB path discoverable for callers that want to migrate.
184
+ meta["kb_path"] = str(kb_dir)
185
+ return meta
186
+
187
+
188
+ def _entry_from_root_legacy(kb_dir: Path) -> dict[str, Any]:
189
+ storage = legacy_storage_dir(kb_dir)
190
+ return {
191
+ "signature": "legacy",
192
+ "version": "legacy",
193
+ "ready": _is_storage_ready(storage),
194
+ "storage_path": str(storage),
195
+ "version_path": str(storage),
196
+ "layout": "root_legacy",
197
+ "legacy": True,
198
+ }
199
+
200
+
201
+ def _sort_versions(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
202
+ def key(entry: dict[str, Any]) -> tuple[str, int, int]:
203
+ layout_priority = 2 if entry.get("layout") == "flat" else 1
204
+ version_num = 0
205
+ version_name = str(entry.get("version") or "")
206
+ match = _VERSION_RE.match(version_name)
207
+ if match:
208
+ version_num = int(match.group(1))
209
+ return (str(entry.get("created_at", "")), layout_priority, version_num)
210
+
211
+ return sorted(versions, key=key, reverse=True)
212
+
213
+
214
+ def list_kb_versions(kb_dir: Path) -> list[dict[str, Any]]:
215
+ """Return all index versions for a KB, newest first.
216
+
217
+ New flat ``version-N`` directories are returned alongside legacy nested and
218
+ root stores so older KBs stay queryable while new re-indexes use the simpler
219
+ flat layout.
220
+ """
221
+ versions: list[dict[str, Any]] = []
222
+
223
+ for version_dir in _flat_version_dirs(kb_dir):
224
+ versions.append(_entry_from_flat_version(version_dir))
225
+
226
+ legacy_versions_root = kb_versions_dir(kb_dir)
227
+ if legacy_versions_root.is_dir():
228
+ for child in sorted(legacy_versions_root.iterdir(), key=lambda path: path.name):
229
+ if child.is_dir():
230
+ versions.append(_entry_from_legacy_nested(kb_dir, child))
231
+
232
+ root_legacy = legacy_storage_dir(kb_dir)
233
+ if root_legacy.is_dir() and any(root_legacy.iterdir()):
234
+ versions.append(_entry_from_root_legacy(kb_dir))
235
+
236
+ return _sort_versions(versions)
237
+
238
+
239
+ def _find_flat_version_by_signature(
240
+ kb_dir: Path, sig_hash: str, *, ready_only: bool
241
+ ) -> Optional[dict[str, Any]]:
242
+ for entry in list_kb_versions(kb_dir):
243
+ if entry.get("layout") != "flat":
244
+ continue
245
+ if entry.get("signature") != sig_hash:
246
+ continue
247
+ if ready_only and not entry.get("ready"):
248
+ continue
249
+ return entry
250
+ return None
251
+
252
+
253
+ def _latest_ready_flat_version(kb_dir: Path) -> Optional[dict[str, Any]]:
254
+ for entry in list_kb_versions(kb_dir):
255
+ if entry.get("layout") == "flat" and entry.get("ready"):
256
+ return entry
257
+ return None
258
+
259
+
260
+ def read_version_meta(kb_dir: Path, sig_hash: str) -> Optional[dict[str, Any]]:
261
+ """Read metadata for a matching signature from any supported layout."""
262
+ for entry in list_kb_versions(kb_dir):
263
+ if entry.get("signature") == sig_hash:
264
+ return entry
265
+ return None
266
+
267
+
268
+ def find_matching_version(kb_dir: Path, signature: EmbeddingSignature) -> Optional[dict[str, Any]]:
269
+ """Return a ready version whose signature matches ``signature``.
270
+
271
+ Prefer the new flat layout when both flat and legacy entries exist for the
272
+ same signature.
273
+ """
274
+ target = signature.hash()
275
+ matches = [
276
+ entry
277
+ for entry in list_kb_versions(kb_dir)
278
+ if entry.get("signature") == target and entry.get("ready")
279
+ ]
280
+ if not matches:
281
+ return None
282
+ flat_matches = [entry for entry in matches if entry.get("layout") == "flat"]
283
+ return flat_matches[0] if flat_matches else matches[0]
284
+
285
+
286
+ def version_dir_for_signature(kb_dir: Path, sig_hash: str) -> Path:
287
+ """Return the existing flat version dir for a signature, or the next flat dir."""
288
+ entry = _find_flat_version_by_signature(kb_dir, sig_hash, ready_only=False)
289
+ if entry:
290
+ return Path(str(entry["version_path"]))
291
+ return _next_flat_version_dir(kb_dir)
292
+
293
+
294
+ def storage_dir_for_signature(kb_dir: Path, sig_hash: str) -> Path:
295
+ """Return a storage dir for a signature in any supported layout.
296
+
297
+ This helper is kept for backward-compatible imports. New write paths should
298
+ use ``resolve_storage_dir_for_write``.
299
+ """
300
+ for entry in list_kb_versions(kb_dir):
301
+ if entry.get("signature") == sig_hash:
302
+ return Path(str(entry["storage_path"]))
303
+ legacy = _legacy_storage_dir_for_signature(kb_dir, sig_hash)
304
+ if legacy.exists():
305
+ return legacy
306
+ return version_dir_for_signature(kb_dir, sig_hash)
307
+
308
+
309
+ def write_version_meta(
310
+ kb_dir: Path, signature: EmbeddingSignature, storage_dir: Path | None = None
311
+ ) -> None:
312
+ """Persist metadata next to the LlamaIndex store."""
313
+ target = storage_dir or resolve_storage_dir_for_write(kb_dir, signature)
314
+ target.mkdir(parents=True, exist_ok=True)
315
+ payload: dict[str, Any] = {
316
+ "version": target.name,
317
+ "signature": signature.hash(),
318
+ **asdict(signature),
319
+ "layout": "flat" if target.parent == kb_dir else "nested_legacy",
320
+ "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
321
+ }
322
+ from deeptutor.services.file_io import atomic_write_json
323
+
324
+ if (target / ".building.json").exists():
325
+ payload.update(provider="llamaindex", state="published", readiness_verified=True)
326
+ atomic_write_json(target / META_FILENAME, payload)
327
+
328
+
329
+ def resolve_storage_dir_for_read(
330
+ kb_dir: Path, signature: Optional[EmbeddingSignature]
331
+ ) -> Optional[Path]:
332
+ """Return the storage dir to read for the active embedding signature."""
333
+ if signature is not None:
334
+ match = find_matching_version(kb_dir, signature)
335
+ if match:
336
+ return Path(str(match["storage_path"]))
337
+ else:
338
+ latest_flat = _latest_ready_flat_version(kb_dir)
339
+ if latest_flat:
340
+ return Path(str(latest_flat["storage_path"]))
341
+
342
+ root_legacy = legacy_storage_dir(kb_dir)
343
+ if _is_storage_ready(root_legacy):
344
+ return root_legacy
345
+
346
+ return None
347
+
348
+
349
+ def resolve_storage_dir_for_write(kb_dir: Path, signature: Optional[EmbeddingSignature]) -> Path:
350
+ """Return the flat storage dir to write for the active embedding signature.
351
+
352
+ Existing flat versions are reused. Legacy nested/root stores are never used
353
+ for new writes so re-indexing naturally converges KBs onto the flat layout.
354
+ """
355
+ if signature is None:
356
+ target = _next_flat_version_dir(kb_dir)
357
+ else:
358
+ entry = _find_flat_version_by_signature(kb_dir, signature.hash(), ready_only=False)
359
+ target = Path(str(entry["storage_path"])) if entry else _next_flat_version_dir(kb_dir)
360
+ target.mkdir(parents=True, exist_ok=True)
361
+ return target
362
+
363
+
364
+ def resolve_storage_dir_for_rebuild(
365
+ kb_dir: Path, signature: Optional[EmbeddingSignature], *, publication_guard: bool = False
366
+ ) -> Path:
367
+ """Return a fresh flat storage dir for a full index rebuild.
368
+
369
+ Incremental writes reuse a matching flat version, but a full rebuild should
370
+ not overwrite the last matching version until the new index has been
371
+ persisted successfully. Keeping the old version in place lets a failed
372
+ rebuild remain diagnosable and avoids stale vector-store files.
373
+ """
374
+ _ = signature
375
+ kb_dir.mkdir(parents=True, exist_ok=True)
376
+ while True:
377
+ target = _next_flat_version_dir(kb_dir)
378
+ try:
379
+ target.mkdir()
380
+ break
381
+ except FileExistsError:
382
+ continue
383
+ from deeptutor.services.file_io import atomic_write_json
384
+
385
+ if publication_guard:
386
+ atomic_write_json(target / ".building.json", {"state": "building"})
387
+ return target
@@ -0,0 +1,50 @@
1
+ """Resolve the on-disk directory backing a knowledge base.
2
+
3
+ Ordinary KBs live at ``<kb_base_dir>/<kb_name>``. A *linked* KB is a pointer to
4
+ an engine index the user already built elsewhere: its ``kb_config.json`` entry
5
+ carries an ``external_path`` we resolve to instead, so retrieval reads that
6
+ folder in place — no copy, no re-index.
7
+
8
+ This is the single seam every pipeline goes through to find a KB's storage
9
+ root. Pipelines must never compute ``Path(kb_base_dir) / kb_name`` directly, or
10
+ linked KBs would resolve to a non-existent local folder and silently return no
11
+ results.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from pathlib import Path
18
+
19
+ from deeptutor.knowledge.kb_types import external_root_of
20
+
21
+ KB_CONFIG_FILENAME = "kb_config.json"
22
+
23
+
24
+ def resolve_kb_dir(kb_base_dir: str | Path, kb_name: str) -> Path:
25
+ """Return the directory holding ``kb_name``'s index.
26
+
27
+ For a linked KB this is the user's external folder; for every other KB it
28
+ is the conventional ``<kb_base_dir>/<kb_name>``.
29
+ """
30
+ base = Path(kb_base_dir)
31
+ external = _external_path(base, kb_name)
32
+ if external:
33
+ return Path(external).expanduser()
34
+ return base / kb_name
35
+
36
+
37
+ def _external_path(base: Path, kb_name: str) -> str | None:
38
+ """Read a KB entry's external pointer from ``kb_config.json``, if any."""
39
+ cfg = base / KB_CONFIG_FILENAME
40
+ if not cfg.exists():
41
+ return None
42
+ try:
43
+ with open(cfg, encoding="utf-8") as handle:
44
+ entry = json.load(handle).get("knowledge_bases", {}).get(kb_name, {})
45
+ except Exception:
46
+ return None
47
+ return external_root_of(entry)
48
+
49
+
50
+ __all__ = ["resolve_kb_dir"]
@@ -0,0 +1,237 @@
1
+ """Probe an external folder before mounting it as a *linked* knowledge base.
2
+
3
+ Linking reuses a self-contained index a user already built — no copy, no
4
+ re-index (see :data:`deeptutor.knowledge.kb_types.LINKED_KB_TYPE`). Before we
5
+ register a pointer we must answer two questions for the user:
6
+
7
+ 1. **Does this folder actually hold a ready index for the chosen engine?**
8
+ We reuse the standard version discovery (``list_kb_versions``) so the same
9
+ "is this KB ready?" logic that lists ordinary KBs decides linkability.
10
+ 2. **Was that index built with a compatible embedding model?** A local vector /
11
+ graph index is only queryable by the embedding model that built it. A
12
+ mismatch makes LlamaIndex error loudly and the graph engines fail silently,
13
+ so we surface a clear verdict the UI can confirm.
14
+
15
+ PageIndex is deliberately not linkable: its index lives in the cloud (the local
16
+ folder holds only a doc-id manifest), so there is nothing self-contained to
17
+ mount.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from dataclasses import asdict, dataclass, field
23
+ import os
24
+ from pathlib import Path
25
+ from typing import Any, Optional
26
+
27
+ # Optional ops-set allowlist of filesystem roots a linked folder must live
28
+ # under, as an ``os.pathsep``-separated list. Unset (the default) means no
29
+ # restriction, which is correct for the local/self-hosted single-trust-domain
30
+ # deployments this feature targets. Shared multi-user servers should set it so a
31
+ # non-admin cannot point a KB at another user's data or system paths.
32
+ LINK_ROOTS_ENV = "DEEPTUTOR_LINKED_FOLDER_ROOTS"
33
+
34
+ from deeptutor.services.rag.factory import (
35
+ DEFAULT_PROVIDER,
36
+ GRAPHRAG_PROVIDER,
37
+ LIGHTRAG_PROVIDER,
38
+ PAGEINDEX_PROVIDER,
39
+ normalize_provider_name,
40
+ )
41
+
42
+ # Engines whose index is self-contained on disk and therefore mountable. The
43
+ # cloud-backed PageIndex is excluded — see module docstring.
44
+ LINKABLE_PROVIDERS = frozenset({DEFAULT_PROVIDER, GRAPHRAG_PROVIDER, LIGHTRAG_PROVIDER})
45
+
46
+
47
+ @dataclass
48
+ class EmbeddingCompat:
49
+ # True/False once both signatures are known; None when it can't be verified.
50
+ compatible: Optional[bool] = None
51
+ index_model: Optional[str] = None
52
+ current_model: Optional[str] = None
53
+
54
+
55
+ @dataclass
56
+ class ProbeResult:
57
+ provider: str
58
+ external_path: str
59
+ ok: bool = False
60
+ version: Optional[str] = None
61
+ doc_count: Optional[int] = None
62
+ embedding: EmbeddingCompat = field(default_factory=EmbeddingCompat)
63
+ warnings: list[str] = field(default_factory=list)
64
+ error: Optional[str] = None
65
+
66
+ def to_dict(self) -> dict[str, Any]:
67
+ data = asdict(self)
68
+ return data
69
+
70
+
71
+ def provider_is_linkable(provider: str) -> bool:
72
+ return normalize_provider_name(provider) in LINKABLE_PROVIDERS
73
+
74
+
75
+ def allowed_link_roots() -> list[Path]:
76
+ """Resolved allowlist of roots a linked folder may live under (may be empty)."""
77
+ raw = os.environ.get(LINK_ROOTS_ENV, "").strip()
78
+ if not raw:
79
+ return []
80
+ roots: list[Path] = []
81
+ for chunk in raw.split(os.pathsep):
82
+ chunk = chunk.strip()
83
+ if not chunk:
84
+ continue
85
+ try:
86
+ roots.append(Path(chunk).expanduser().resolve())
87
+ except OSError:
88
+ continue
89
+ return roots
90
+
91
+
92
+ def assert_path_allowed(folder_path: str) -> Path:
93
+ """Resolve ``folder_path`` and enforce the optional root allowlist.
94
+
95
+ Returns the resolved absolute path. Raises ``ValueError`` if the folder is
96
+ missing/not a directory, or escapes the configured allowlist (symlinks are
97
+ resolved first so they can't tunnel out). With no allowlist set, any
98
+ existing directory is permitted — the self-hosted default.
99
+ """
100
+ folder = Path(folder_path).expanduser()
101
+ if not folder.exists():
102
+ raise ValueError(f"Folder does not exist: {folder}")
103
+ if not folder.is_dir():
104
+ raise ValueError(f"Not a directory: {folder}")
105
+ resolved = folder.resolve()
106
+
107
+ roots = allowed_link_roots()
108
+ if roots and not any(_is_within(resolved, root) for root in roots):
109
+ raise ValueError("This folder is outside the locations allowed for linking.")
110
+ return resolved
111
+
112
+
113
+ def _is_within(path: Path, root: Path) -> bool:
114
+ try:
115
+ return path == root or root in path.parents
116
+ except OSError:
117
+ return False
118
+
119
+
120
+ def probe_linked_folder(folder_path: str, provider: str) -> ProbeResult:
121
+ """Inspect ``folder_path`` for a ready ``provider`` index.
122
+
123
+ Always returns a :class:`ProbeResult` (never raises): ``error`` set means
124
+ the folder cannot be linked; ``warnings`` are non-fatal cautions the user
125
+ confirms. The caller is responsible for any path-jail / access checks.
126
+ """
127
+ provider = normalize_provider_name(provider)
128
+ folder = Path(folder_path).expanduser()
129
+ result = ProbeResult(provider=provider, external_path=str(folder))
130
+
131
+ if provider == PAGEINDEX_PROVIDER:
132
+ result.error = (
133
+ "PageIndex indexes live in the cloud, not in a local folder, so they "
134
+ "cannot be linked. Create a PageIndex knowledge base instead."
135
+ )
136
+ return result
137
+ if provider not in LINKABLE_PROVIDERS:
138
+ result.error = f"Engine '{provider}' does not support linking an existing folder."
139
+ return result
140
+
141
+ if not folder.exists():
142
+ result.error = f"Folder does not exist: {folder}"
143
+ return result
144
+ if not folder.is_dir():
145
+ result.error = f"Not a directory: {folder}"
146
+ return result
147
+
148
+ result.external_path = str(folder.resolve())
149
+
150
+ from deeptutor.services.rag.index_probe import inspect_kb_versions
151
+
152
+ versions = inspect_kb_versions(folder, provider)
153
+ ready = [v for v in versions if v.get("ready")]
154
+ if not ready:
155
+ failure = next(
156
+ (str(v.get("failure_summary") or "") for v in versions if v.get("failure_summary")),
157
+ "",
158
+ )
159
+ result.error = (
160
+ failure
161
+ or "No ready index found in this folder. Point at a knowledge base folder "
162
+ "that contains a built index (a 'version-N' directory)."
163
+ )
164
+ return result
165
+
166
+ latest = ready[0] # inspect_kb_versions preserves newest-first ordering.
167
+ result.version = str(latest.get("version") or "")
168
+ _check_embedding(provider, latest, result)
169
+ doc_count = latest.get("doc_count")
170
+ result.doc_count = (
171
+ int(doc_count) if isinstance(doc_count, int) else _best_effort_doc_count(folder)
172
+ )
173
+ result.ok = result.error is None
174
+ return result
175
+
176
+
177
+ def _check_embedding(provider: str, version: dict, result: ProbeResult) -> None:
178
+ """Compare the index's embedding identity against the active config."""
179
+ from deeptutor.services.rag.embedding_signature import signature_from_embedding_config
180
+
181
+ current = signature_from_embedding_config()
182
+ compat = result.embedding
183
+ compat.current_model = current.model if current else None
184
+
185
+ # LlamaIndex stores the embedding hash in the version signature; the graph
186
+ # engines stamp it separately (see embedding_meta_fields).
187
+ if provider == DEFAULT_PROVIDER:
188
+ index_hash = str(version.get("signature") or "")
189
+ else:
190
+ index_hash = str(version.get("embedding_signature") or "")
191
+ compat.index_model = version.get("embedding_model") or version.get("model")
192
+
193
+ if current is None:
194
+ compat.compatible = None
195
+ result.warnings.append(
196
+ "No embedding model is configured, so embedding compatibility could not "
197
+ "be verified. Configure one under Settings before querying."
198
+ )
199
+ return
200
+ if not index_hash:
201
+ compat.compatible = None
202
+ result.warnings.append(
203
+ "This index does not record which embedding model built it. Make sure it "
204
+ "matches your current embedding model, or retrieval may fail."
205
+ )
206
+ return
207
+
208
+ compat.compatible = index_hash == current.hash()
209
+ if not compat.compatible:
210
+ built_with = compat.index_model or "a different model"
211
+ result.warnings.append(
212
+ f"This index was built with {built_with}, which differs from your current "
213
+ f"embedding model ({compat.current_model}). Retrieval will not work until "
214
+ "you switch back to the matching embedding model."
215
+ )
216
+
217
+
218
+ def _best_effort_doc_count(folder: Path) -> Optional[int]:
219
+ """Count source files under ``raw/`` if the linked folder ships them."""
220
+ raw = folder / "raw"
221
+ if not raw.is_dir():
222
+ return None
223
+ try:
224
+ return sum(1 for p in raw.rglob("*") if p.is_file())
225
+ except OSError:
226
+ return None
227
+
228
+
229
+ __all__ = [
230
+ "LINKABLE_PROVIDERS",
231
+ "EmbeddingCompat",
232
+ "ProbeResult",
233
+ "probe_linked_folder",
234
+ "provider_is_linkable",
235
+ "allowed_link_roots",
236
+ "assert_path_allowed",
237
+ ]
@@ -0,0 +1,7 @@
1
+ """Pre-configured RAG pipelines.
2
+
3
+ DeepTutor currently ships with a single built-in provider (`llamaindex`).
4
+ Additional providers can still be registered dynamically via the factory layer.
5
+ """
6
+
7
+ __all__: list[str] = []