deeptutor-knowledge 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeptutor/api/routers/knowledge.py +5225 -0
- deeptutor/api/routers/marginnote4.py +278 -0
- deeptutor/capabilities/ima/__init__.py +6 -0
- deeptutor/capabilities/ima/binding.py +118 -0
- deeptutor/capabilities/ima/capability.py +92 -0
- deeptutor/capabilities/ima/prompts/en/system.md +33 -0
- deeptutor/capabilities/ima/prompts/zh/system.md +24 -0
- deeptutor/capabilities/ima/tools.py +476 -0
- deeptutor/capabilities/marginnote4/__init__.py +13 -0
- deeptutor/capabilities/marginnote4/binding.py +66 -0
- deeptutor/capabilities/marginnote4/capability.py +80 -0
- deeptutor/capabilities/marginnote4/models.py +173 -0
- deeptutor/capabilities/marginnote4/prompts/en/system.md +54 -0
- deeptutor/capabilities/marginnote4/prompts/zh/system.md +44 -0
- deeptutor/capabilities/marginnote4/store.py +562 -0
- deeptutor/capabilities/marginnote4/tools.py +329 -0
- deeptutor/capabilities/obsidian/__init__.py +6 -0
- deeptutor/capabilities/obsidian/binding.py +75 -0
- deeptutor/capabilities/obsidian/capability.py +82 -0
- deeptutor/capabilities/obsidian/prompts/en/system.md +32 -0
- deeptutor/capabilities/obsidian/prompts/zh/system.md +28 -0
- deeptutor/capabilities/obsidian/tools.py +399 -0
- deeptutor/capabilities/obsidian/vault.py +341 -0
- deeptutor/knowledge/__init__.py +28 -0
- deeptutor/knowledge/add_documents.py +734 -0
- deeptutor/knowledge/indexing_run.py +432 -0
- deeptutor/knowledge/initializer.py +448 -0
- deeptutor/knowledge/manager.py +2570 -0
- deeptutor/knowledge/module_service.py +29 -0
- deeptutor/knowledge/naming.py +40 -0
- deeptutor/knowledge/progress_events.py +45 -0
- deeptutor/knowledge/progress_tracker.py +355 -0
- deeptutor/knowledge/staging.py +69 -0
- deeptutor/services/config/knowledge_base_config.py +239 -0
- deeptutor/services/embedding/__init__.py +46 -0
- deeptutor/services/embedding/adapters/__init__.py +40 -0
- deeptutor/services/embedding/adapters/base.py +196 -0
- deeptutor/services/embedding/adapters/cohere.py +176 -0
- deeptutor/services/embedding/adapters/dashscope_native.py +242 -0
- deeptutor/services/embedding/adapters/gemini.py +462 -0
- deeptutor/services/embedding/adapters/jina.py +161 -0
- deeptutor/services/embedding/adapters/ollama.py +145 -0
- deeptutor/services/embedding/adapters/openai_compatible.py +406 -0
- deeptutor/services/embedding/adapters/openai_sdk.py +150 -0
- deeptutor/services/embedding/client.py +323 -0
- deeptutor/services/embedding/config.py +114 -0
- deeptutor/services/embedding/request_options.py +46 -0
- deeptutor/services/embedding/validation.py +128 -0
- deeptutor/services/github_source/__init__.py +5 -0
- deeptutor/services/github_source/client.py +164 -0
- deeptutor/services/github_source/sync.py +384 -0
- deeptutor/services/github_source/sync_service.py +72 -0
- deeptutor/services/rag/__init__.py +21 -0
- deeptutor/services/rag/embedding_binding.py +331 -0
- deeptutor/services/rag/embedding_signature.py +63 -0
- deeptutor/services/rag/eval/__init__.py +92 -0
- deeptutor/services/rag/eval/dataset.py +244 -0
- deeptutor/services/rag/eval/diff.py +312 -0
- deeptutor/services/rag/eval/matching.py +218 -0
- deeptutor/services/rag/eval/metrics.py +213 -0
- deeptutor/services/rag/eval/report.py +111 -0
- deeptutor/services/rag/eval/runner.py +309 -0
- deeptutor/services/rag/factory.py +369 -0
- deeptutor/services/rag/file_routing.py +5 -0
- deeptutor/services/rag/index_probe.py +327 -0
- deeptutor/services/rag/index_versioning.py +387 -0
- deeptutor/services/rag/kb_paths.py +50 -0
- deeptutor/services/rag/linked_kb.py +237 -0
- deeptutor/services/rag/pipelines/__init__.py +7 -0
- deeptutor/services/rag/pipelines/base.py +41 -0
- deeptutor/services/rag/pipelines/graphrag/__init__.py +18 -0
- deeptutor/services/rag/pipelines/graphrag/compatibility.py +42 -0
- deeptutor/services/rag/pipelines/graphrag/completion_adapter.py +251 -0
- deeptutor/services/rag/pipelines/graphrag/config.py +357 -0
- deeptutor/services/rag/pipelines/graphrag/engine.py +479 -0
- deeptutor/services/rag/pipelines/graphrag/errors.py +328 -0
- deeptutor/services/rag/pipelines/graphrag/ingestion.py +135 -0
- deeptutor/services/rag/pipelines/graphrag/pandas_compat.py +68 -0
- deeptutor/services/rag/pipelines/graphrag/pipeline.py +284 -0
- deeptutor/services/rag/pipelines/graphrag/provider.py +134 -0
- deeptutor/services/rag/pipelines/graphrag/storage.py +127 -0
- deeptutor/services/rag/pipelines/ima/__init__.py +28 -0
- deeptutor/services/rag/pipelines/ima/client.py +364 -0
- deeptutor/services/rag/pipelines/ima/config.py +140 -0
- deeptutor/services/rag/pipelines/ima/envelope.py +109 -0
- deeptutor/services/rag/pipelines/ima/inventory.py +209 -0
- deeptutor/services/rag/pipelines/ima/media.py +190 -0
- deeptutor/services/rag/pipelines/ima/models.py +348 -0
- deeptutor/services/rag/pipelines/ima/notes.py +240 -0
- deeptutor/services/rag/pipelines/ima/pipeline.py +238 -0
- deeptutor/services/rag/pipelines/ima/probe.py +105 -0
- deeptutor/services/rag/pipelines/ima/sources.py +105 -0
- deeptutor/services/rag/pipelines/ima/transport.py +130 -0
- deeptutor/services/rag/pipelines/kiwix/__init__.py +1 -0
- deeptutor/services/rag/pipelines/kiwix/client.py +369 -0
- deeptutor/services/rag/pipelines/kiwix/pipeline.py +75 -0
- deeptutor/services/rag/pipelines/lightrag/README.md +78 -0
- deeptutor/services/rag/pipelines/lightrag/__init__.py +15 -0
- deeptutor/services/rag/pipelines/lightrag/block_policy.py +266 -0
- deeptutor/services/rag/pipelines/lightrag/cache_reuse.py +137 -0
- deeptutor/services/rag/pipelines/lightrag/config.py +453 -0
- deeptutor/services/rag/pipelines/lightrag/engine.py +536 -0
- deeptutor/services/rag/pipelines/lightrag/indexing_policy.py +696 -0
- deeptutor/services/rag/pipelines/lightrag/ingress.py +364 -0
- deeptutor/services/rag/pipelines/lightrag/parser.py +99 -0
- deeptutor/services/rag/pipelines/lightrag/pipeline.py +692 -0
- deeptutor/services/rag/pipelines/lightrag/roles.py +132 -0
- deeptutor/services/rag/pipelines/lightrag/sidecar.py +320 -0
- deeptutor/services/rag/pipelines/lightrag/storage.py +338 -0
- deeptutor/services/rag/pipelines/lightrag/worker.py +259 -0
- deeptutor/services/rag/pipelines/lightrag/write_lock.py +55 -0
- deeptutor/services/rag/pipelines/lightrag_server/__init__.py +15 -0
- deeptutor/services/rag/pipelines/lightrag_server/client.py +149 -0
- deeptutor/services/rag/pipelines/lightrag_server/config.py +68 -0
- deeptutor/services/rag/pipelines/lightrag_server/pipeline.py +120 -0
- deeptutor/services/rag/pipelines/lightrag_server/probe.py +110 -0
- deeptutor/services/rag/pipelines/llamaindex/__init__.py +3 -0
- deeptutor/services/rag/pipelines/llamaindex/config.py +199 -0
- deeptutor/services/rag/pipelines/llamaindex/document_loader.py +648 -0
- deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.py +264 -0
- deeptutor/services/rag/pipelines/llamaindex/errors.py +76 -0
- deeptutor/services/rag/pipelines/llamaindex/exercise_lookup.py +373 -0
- deeptutor/services/rag/pipelines/llamaindex/ingestion.py +129 -0
- deeptutor/services/rag/pipelines/llamaindex/pipeline.py +556 -0
- deeptutor/services/rag/pipelines/llamaindex/rerank.py +130 -0
- deeptutor/services/rag/pipelines/llamaindex/retrievers.py +360 -0
- deeptutor/services/rag/pipelines/llamaindex/storage.py +655 -0
- deeptutor/services/rag/pipelines/llamaindex/vector_store.py +288 -0
- deeptutor/services/rag/pipelines/modes.py +48 -0
- deeptutor/services/rag/pipelines/pageindex/__init__.py +47 -0
- deeptutor/services/rag/pipelines/pageindex/client.py +146 -0
- deeptutor/services/rag/pipelines/pageindex/config.py +49 -0
- deeptutor/services/rag/pipelines/pageindex/pipeline.py +313 -0
- deeptutor/services/rag/pipelines/pageindex/reasoning.py +262 -0
- deeptutor/services/rag/pipelines/pageindex/storage.py +133 -0
- deeptutor/services/rag/pipelines/pageindex/tools.py +221 -0
- deeptutor/services/rag/pipelines/weknora/__init__.py +13 -0
- deeptutor/services/rag/pipelines/weknora/client.py +106 -0
- deeptutor/services/rag/pipelines/weknora/config.py +56 -0
- deeptutor/services/rag/pipelines/weknora/pipeline.py +74 -0
- deeptutor/services/rag/pipelines/weknora/probe.py +80 -0
- deeptutor/services/rag/preflight.py +274 -0
- deeptutor/services/rag/provider_binding.py +70 -0
- deeptutor/services/rag/query_rewrite.py +186 -0
- deeptutor/services/rag/service.py +379 -0
- deeptutor/services/rag/smart_retriever.py +81 -0
- deeptutor/services/rag/source_pages.py +292 -0
- deeptutor/services/rag/source_visuals.py +434 -0
- deeptutor/services/rag/visual_assets.py +430 -0
- deeptutor/services/rag/visual_coverage.py +165 -0
- deeptutor/services/web_source/__init__.py +1 -0
- deeptutor/services/web_source/bilingual.py +255 -0
- deeptutor/services/web_source/crawler.py +681 -0
- deeptutor/services/web_source/html_extractor.py +643 -0
- deeptutor/services/web_source/markdown.py +28 -0
- deeptutor/services/web_source/navigation.py +72 -0
- deeptutor/services/web_source/repository.py +640 -0
- deeptutor/services/web_source/robots.py +182 -0
- deeptutor/services/web_source/scheduler.py +336 -0
- deeptutor/services/web_source/snapshot_assets.py +171 -0
- deeptutor/services/web_source/sync.py +253 -0
- deeptutor/services/workspace/kb_move.py +446 -0
- deeptutor/tools/rag_tool.py +110 -0
- deeptutor_cli/kb.py +756 -0
- deeptutor_knowledge-1.7.0.dist-info/METADATA +15 -0
- deeptutor_knowledge-1.7.0.dist-info/RECORD +168 -0
- deeptutor_knowledge-1.7.0.dist-info/WHEEL +5 -0
- deeptutor_knowledge-1.7.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Sync logic: crawl a doc site and ingest pages into a KB.
|
|
2
|
+
|
|
3
|
+
``sync_source()`` mirrors the GitHub-source sync flow:
|
|
4
|
+
1. Crawl the site from the configured base URL.
|
|
5
|
+
2. Compare page hashes with those stored in metadata.
|
|
6
|
+
3. Write new/changed pages to the KB ``raw/`` directory as ``.md`` files.
|
|
7
|
+
4. Remove pages that were present before but are no longer on the site.
|
|
8
|
+
5. Feed changed files through ``add_documents()`` for indexing.
|
|
9
|
+
6. Persist updated page hashes + sync status to ``metadata.json``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
import logging
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from deeptutor.knowledge.add_documents import DEFAULT_BASE_DIR
|
|
21
|
+
from deeptutor.services.web_source.crawler import _contained_path, crawl_and_diff
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
WEB_SYNC_INTERVAL_HOURS = 24
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class WebSyncResult:
|
|
30
|
+
"""Outcome of a single ``sync_source()`` invocation."""
|
|
31
|
+
|
|
32
|
+
ok: bool
|
|
33
|
+
pages_added: int = 0
|
|
34
|
+
pages_updated: int = 0
|
|
35
|
+
pages_removed: int = 0
|
|
36
|
+
pages_unchanged: int = 0
|
|
37
|
+
error: str = ""
|
|
38
|
+
bilingual_pairs_count: int = 0
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def total_changes(self) -> int:
|
|
42
|
+
return self.pages_added + self.pages_updated + self.pages_removed
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _utcnow_iso() -> str:
|
|
46
|
+
return datetime.now(timezone.utc).isoformat()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _url_to_filename(url: str, base_path_prefix: str) -> str:
|
|
50
|
+
"""Stable ``.md`` filename derived from a page URL."""
|
|
51
|
+
from deeptutor.services.web_source.crawler import _to_filename
|
|
52
|
+
|
|
53
|
+
return _to_filename(url, base_path_prefix)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _record_sync_failure(
|
|
57
|
+
kb_name: str,
|
|
58
|
+
source_id: str,
|
|
59
|
+
base_dir: str,
|
|
60
|
+
error: str,
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Persist a visible error without advancing hashes past failed indexing."""
|
|
63
|
+
from deeptutor.knowledge.manager import KnowledgeBaseManager
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
KnowledgeBaseManager(base_dir=base_dir).update_web_source_state(
|
|
67
|
+
kb_name=kb_name,
|
|
68
|
+
source_id=source_id,
|
|
69
|
+
last_synced_at=_utcnow_iso(),
|
|
70
|
+
last_sync_status="error",
|
|
71
|
+
last_sync_error=error,
|
|
72
|
+
)
|
|
73
|
+
except Exception:
|
|
74
|
+
logger.exception("Failed to persist web-source sync failure state")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def _rebuild_index_after_removal(
|
|
78
|
+
kb_name: str,
|
|
79
|
+
raw_dir: Path,
|
|
80
|
+
base_dir: str,
|
|
81
|
+
) -> int:
|
|
82
|
+
"""Rebuild the bound provider so removed pages cannot remain retrievable."""
|
|
83
|
+
from deeptutor.services.rag.service import RAGService
|
|
84
|
+
from deeptutor.utils.file_types import FileTypeRouter
|
|
85
|
+
|
|
86
|
+
files = FileTypeRouter.collect_supported_files(raw_dir, recursive=True)
|
|
87
|
+
if not files:
|
|
88
|
+
raise RuntimeError("No source files remain after removing deleted web pages")
|
|
89
|
+
success = await RAGService(kb_base_dir=base_dir).initialize(
|
|
90
|
+
kb_name=kb_name,
|
|
91
|
+
file_paths=[str(path) for path in files],
|
|
92
|
+
)
|
|
93
|
+
if not success:
|
|
94
|
+
raise RuntimeError("The knowledge-base index rebuild produced no documents")
|
|
95
|
+
return len(files)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def sync_source(
|
|
99
|
+
kb_name: str,
|
|
100
|
+
source: dict[str, Any],
|
|
101
|
+
*,
|
|
102
|
+
base_dir: str = DEFAULT_BASE_DIR,
|
|
103
|
+
max_depth: int | None = None,
|
|
104
|
+
max_pages: int | None = None,
|
|
105
|
+
) -> WebSyncResult:
|
|
106
|
+
"""Crawl and sync one web source into the named KB (legacy per-file path).
|
|
107
|
+
|
|
108
|
+
Uses :func:`crawl_and_diff` for the shared crawl-write-diff pipeline,
|
|
109
|
+
then indexes changed files individually via ``add_documents``.
|
|
110
|
+
|
|
111
|
+
This is the bounded, on-demand sync path used by the API and CLI.
|
|
112
|
+
"""
|
|
113
|
+
kb_dir = Path(base_dir) / kb_name
|
|
114
|
+
raw_dir = kb_dir / "raw"
|
|
115
|
+
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
diff = await crawl_and_diff(
|
|
119
|
+
source,
|
|
120
|
+
raw_dir,
|
|
121
|
+
max_depth=max_depth,
|
|
122
|
+
max_pages=max_pages,
|
|
123
|
+
)
|
|
124
|
+
except Exception as exc:
|
|
125
|
+
logger.exception("Crawl failed for %s", source["url"])
|
|
126
|
+
error = str(exc)
|
|
127
|
+
_record_sync_failure(kb_name, source["id"], base_dir, error)
|
|
128
|
+
return WebSyncResult(ok=False, error=error)
|
|
129
|
+
if not diff.ok:
|
|
130
|
+
_record_sync_failure(kb_name, source["id"], base_dir, diff.error)
|
|
131
|
+
return WebSyncResult(ok=False, error=diff.error)
|
|
132
|
+
|
|
133
|
+
# Remove deleted raw pages and their file-hash records first. A full
|
|
134
|
+
# provider rebuild below is required because most providers do not expose
|
|
135
|
+
# a reliable per-document vector deletion operation.
|
|
136
|
+
removed_count = 0
|
|
137
|
+
removal_errors: list[str] = []
|
|
138
|
+
for fname in diff.pages_removed:
|
|
139
|
+
target = _contained_path(raw_dir, fname)
|
|
140
|
+
if target is None:
|
|
141
|
+
removal_errors.append(f"{fname}: path escapes the knowledge base")
|
|
142
|
+
continue
|
|
143
|
+
try:
|
|
144
|
+
from deeptutor.knowledge.add_documents import remove_raw_document
|
|
145
|
+
|
|
146
|
+
remove_raw_document(kb_dir, target)
|
|
147
|
+
removed_count += 1
|
|
148
|
+
except Exception as exc:
|
|
149
|
+
logger.warning("Failed to remove %s: %s", fname, exc)
|
|
150
|
+
removal_errors.append(f"{fname}: {exc}")
|
|
151
|
+
|
|
152
|
+
if removal_errors:
|
|
153
|
+
error = "Failed to remove deleted pages: " + "; ".join(removal_errors)
|
|
154
|
+
_record_sync_failure(kb_name, source["id"], base_dir, error)
|
|
155
|
+
return WebSyncResult(ok=False, error=error)
|
|
156
|
+
|
|
157
|
+
indexed = 0
|
|
158
|
+
try:
|
|
159
|
+
if diff.pages_removed:
|
|
160
|
+
indexed = await _rebuild_index_after_removal(kb_name, raw_dir, base_dir)
|
|
161
|
+
elif diff.changed_paths:
|
|
162
|
+
from deeptutor.knowledge.add_documents import add_documents
|
|
163
|
+
|
|
164
|
+
indexed = await add_documents(
|
|
165
|
+
kb_name=kb_name,
|
|
166
|
+
source_files=diff.changed_paths,
|
|
167
|
+
base_dir=base_dir,
|
|
168
|
+
allow_duplicates=False,
|
|
169
|
+
)
|
|
170
|
+
except Exception as exc:
|
|
171
|
+
logger.warning("Indexing failed for web source files: %s", exc)
|
|
172
|
+
error = f"Indexing failed: {exc}"
|
|
173
|
+
_record_sync_failure(kb_name, source["id"], base_dir, error)
|
|
174
|
+
return WebSyncResult(ok=False, error=error)
|
|
175
|
+
|
|
176
|
+
# Persist sync state.
|
|
177
|
+
from deeptutor.knowledge.manager import KnowledgeBaseManager
|
|
178
|
+
|
|
179
|
+
manager = KnowledgeBaseManager(base_dir=base_dir)
|
|
180
|
+
manager.update_web_source_state(
|
|
181
|
+
kb_name=kb_name,
|
|
182
|
+
source_id=source["id"],
|
|
183
|
+
page_hashes=diff.page_hashes,
|
|
184
|
+
page_count=diff.page_count,
|
|
185
|
+
last_synced_at=_utcnow_iso(),
|
|
186
|
+
last_sync_status="success",
|
|
187
|
+
last_sync_error=None,
|
|
188
|
+
navigation=diff.navigation,
|
|
189
|
+
bilingual_pairings=diff.pairings,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
try:
|
|
193
|
+
from deeptutor.multi_user.context import get_current_user
|
|
194
|
+
from deeptutor.services.web_source.scheduler import get_web_source_sync_scheduler
|
|
195
|
+
|
|
196
|
+
user = get_current_user()
|
|
197
|
+
scheduler = get_web_source_sync_scheduler()
|
|
198
|
+
scheduler.repo.record_pairings(
|
|
199
|
+
owner_id=user.id,
|
|
200
|
+
kb_name=kb_name,
|
|
201
|
+
source_id=source["id"],
|
|
202
|
+
pairings=diff.pairings,
|
|
203
|
+
)
|
|
204
|
+
except Exception:
|
|
205
|
+
logger.debug("Failed to record pairings in SQLite repo", exc_info=True)
|
|
206
|
+
|
|
207
|
+
logger.info(
|
|
208
|
+
"Web sync %s: +%d ~%d -%d (%d unchanged), %d indexed, %d bilingual pairs",
|
|
209
|
+
diff.url,
|
|
210
|
+
len(diff.pages_added),
|
|
211
|
+
len(diff.pages_updated),
|
|
212
|
+
removed_count,
|
|
213
|
+
len(diff.pages_unchanged),
|
|
214
|
+
indexed,
|
|
215
|
+
len(diff.pairings),
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
return WebSyncResult(
|
|
219
|
+
ok=True,
|
|
220
|
+
pages_added=len(diff.pages_added),
|
|
221
|
+
pages_updated=len(diff.pages_updated),
|
|
222
|
+
pages_removed=removed_count,
|
|
223
|
+
pages_unchanged=len(diff.pages_unchanged),
|
|
224
|
+
bilingual_pairs_count=len(diff.pairings),
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# Navigation helpers live in navigation.py now.
|
|
229
|
+
# These aliases keep backward compatibility for existing imports/tests.
|
|
230
|
+
from deeptutor.services.web_source.navigation import ( # noqa: F401, E402
|
|
231
|
+
build_navigation_manifest as _build_navigation_manifest_impl,
|
|
232
|
+
)
|
|
233
|
+
from deeptutor.services.web_source.navigation import (
|
|
234
|
+
flat_to_tree as _flat_to_tree_impl,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _build_navigation_manifest(
|
|
239
|
+
nav_links: list[dict],
|
|
240
|
+
nav_kind: str,
|
|
241
|
+
page_urls: dict[str, str],
|
|
242
|
+
base_path_prefix: str = "",
|
|
243
|
+
) -> dict:
|
|
244
|
+
"""Backward-compatible wrapper around navigation.build_navigation_manifest."""
|
|
245
|
+
return _build_navigation_manifest_impl(nav_links, nav_kind, page_urls)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _flat_to_tree(
|
|
249
|
+
links: list[dict],
|
|
250
|
+
url_to_file: dict[str, str],
|
|
251
|
+
) -> list[dict]:
|
|
252
|
+
"""Backward-compatible alias for navigation.flat_to_tree."""
|
|
253
|
+
return _flat_to_tree_impl(links, url_to_file)
|
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
"""Move one owned knowledge base between account-local storage roots.
|
|
2
|
+
|
|
3
|
+
The index and raw documents are copied as a unit. The source remains intact
|
|
4
|
+
until the destination, catalog entries, and redirects have been published.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import shutil
|
|
15
|
+
from typing import Any, cast
|
|
16
|
+
import uuid
|
|
17
|
+
|
|
18
|
+
from deeptutor.knowledge.kb_types import MARGINNOTE4_KB_TYPE, is_connected_kb
|
|
19
|
+
from deeptutor.multi_user.context import get_current_user
|
|
20
|
+
from deeptutor.services.file_io import atomic_write_json
|
|
21
|
+
from deeptutor.services.path_service import get_path_service
|
|
22
|
+
from deeptutor.services.rag.factory import has_ready_provider_index
|
|
23
|
+
from deeptutor.services.workspace.activity import data_activity
|
|
24
|
+
from deeptutor.services.workspace.context import workspace_context
|
|
25
|
+
from deeptutor.services.workspace.data_migration import (
|
|
26
|
+
_files,
|
|
27
|
+
_snapshot,
|
|
28
|
+
assert_no_pending_recovery,
|
|
29
|
+
)
|
|
30
|
+
from deeptutor.services.workspace.knowledge import (
|
|
31
|
+
_move_aliases_path,
|
|
32
|
+
canonical_kb_id,
|
|
33
|
+
move_aliases,
|
|
34
|
+
parse_kb_id,
|
|
35
|
+
qualified_kb_id,
|
|
36
|
+
resolve_qualified,
|
|
37
|
+
)
|
|
38
|
+
from deeptutor.services.workspace.models import WorkspaceError
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _config(root: Path) -> dict:
|
|
42
|
+
path = root / "kb_config.json"
|
|
43
|
+
if path.is_symlink():
|
|
44
|
+
raise WorkspaceError("Knowledge configuration cannot be a symbolic link.")
|
|
45
|
+
if not path.exists():
|
|
46
|
+
return {"knowledge_bases": {}}
|
|
47
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
48
|
+
if not isinstance(value, dict) or not isinstance(value.get("knowledge_bases"), dict):
|
|
49
|
+
raise WorkspaceError("Knowledge configuration is damaged.")
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _root(workspace_id: str) -> tuple[Path, bool]:
|
|
54
|
+
with workspace_context(workspace_id) as scope:
|
|
55
|
+
return get_path_service().get_knowledge_bases_root(), scope.archived
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _shared_grants(name: str) -> list[str]:
|
|
59
|
+
from deeptutor.services.auth import AUTH_ENABLED
|
|
60
|
+
|
|
61
|
+
if not AUTH_ENABLED or not get_current_user().is_admin:
|
|
62
|
+
return []
|
|
63
|
+
from deeptutor.multi_user.grants import load_grant
|
|
64
|
+
from deeptutor.multi_user.identity import list_user_info
|
|
65
|
+
|
|
66
|
+
return [
|
|
67
|
+
user["id"]
|
|
68
|
+
for user in list_user_info()
|
|
69
|
+
if user.get("role") != "admin"
|
|
70
|
+
and any(
|
|
71
|
+
str(item.get("name") or item.get("kb_name") or "") == name
|
|
72
|
+
or str(item.get("resource_id") or item.get("id") or "") == f"admin:kb:{name}"
|
|
73
|
+
for item in load_grant(user["id"]).get("knowledge_bases", []) or []
|
|
74
|
+
)
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _legacy_account_id(name: str) -> str:
|
|
79
|
+
# Historical general-chat selections used a role-specific prefix. Keep
|
|
80
|
+
# that alias only for the owning account; custom workspaces can have
|
|
81
|
+
# same-name KBs, so their legacy IDs cannot be redirected globally.
|
|
82
|
+
role = "admin" if get_current_user().is_admin else "user"
|
|
83
|
+
return f"{role}:kb:{name}"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def preview_kb_move(source_id: str, target_workspace_id: str) -> dict:
|
|
87
|
+
"""Return a collision and assignment preview without changing either store."""
|
|
88
|
+
from deeptutor.services.workspace import get_content_workspace_service
|
|
89
|
+
|
|
90
|
+
source_id = canonical_kb_id(str(source_id or ""))
|
|
91
|
+
parsed = parse_kb_id(source_id)
|
|
92
|
+
if parsed is None or not parsed[1]:
|
|
93
|
+
raise WorkspaceError("Choose a knowledge base from the library.")
|
|
94
|
+
source_workspace_id, name = parsed
|
|
95
|
+
target_workspace_id = str(target_workspace_id or "")
|
|
96
|
+
# Move is a library operation. Chat selection is a read ceiling for turns,
|
|
97
|
+
# not an ownership check for managing a different stored KB.
|
|
98
|
+
resolve_qualified(source_id, require_write=True)
|
|
99
|
+
source_root, source_archived = _root(source_workspace_id)
|
|
100
|
+
target_root, target_archived = _root(target_workspace_id)
|
|
101
|
+
source_config = _config(source_root)
|
|
102
|
+
target_config = _config(target_root)
|
|
103
|
+
entry = source_config["knowledge_bases"].get(name)
|
|
104
|
+
source_dir = source_root / name
|
|
105
|
+
target_dir = target_root / name
|
|
106
|
+
blockers: list[str] = []
|
|
107
|
+
if source_workspace_id == target_workspace_id or source_root == target_root:
|
|
108
|
+
blockers.append("Choose a different storage workspace.")
|
|
109
|
+
if source_archived or target_archived:
|
|
110
|
+
blockers.append("Restore the workspace before moving its knowledge base.")
|
|
111
|
+
if not isinstance(entry, dict):
|
|
112
|
+
blockers.append("The source knowledge base is missing from its configuration.")
|
|
113
|
+
elif entry.get("type") == MARGINNOTE4_KB_TYPE:
|
|
114
|
+
blockers.append(
|
|
115
|
+
"MarginNote 4 stores its synced database outside the knowledge-base folder. "
|
|
116
|
+
"Move is unavailable until that database can be transferred safely."
|
|
117
|
+
)
|
|
118
|
+
elif not is_connected_kb(entry):
|
|
119
|
+
if entry.get("path", name) != name:
|
|
120
|
+
blockers.append(
|
|
121
|
+
"This knowledge base has a nonstandard storage path and cannot be moved safely."
|
|
122
|
+
)
|
|
123
|
+
if not source_dir.is_dir() or source_dir.is_symlink():
|
|
124
|
+
blockers.append("The source knowledge base directory is unavailable.")
|
|
125
|
+
if str(entry.get("status") or "ready") not in {"ready", "error", "needs_reindex"}:
|
|
126
|
+
blockers.append("Wait for this knowledge base to finish processing before moving it.")
|
|
127
|
+
if name in target_config["knowledge_bases"] or target_dir.exists() or target_dir.is_symlink():
|
|
128
|
+
blockers.append(f"Destination already contains a knowledge base named '{name}'.")
|
|
129
|
+
target_id = qualified_kb_id(name, target_workspace_id)
|
|
130
|
+
if target_id in move_aliases():
|
|
131
|
+
blockers.append("Destination ID is reserved by an earlier knowledge-base move.")
|
|
132
|
+
if not source_workspace_id and _shared_grants(name):
|
|
133
|
+
blockers.append(
|
|
134
|
+
"This knowledge base is shared with other accounts. Remove those grants before moving it."
|
|
135
|
+
)
|
|
136
|
+
if source_dir.is_dir() and not source_dir.is_symlink():
|
|
137
|
+
try:
|
|
138
|
+
files = list(_files(source_dir))
|
|
139
|
+
file_count = len(files)
|
|
140
|
+
byte_count = sum(path.stat().st_size for path in files)
|
|
141
|
+
except (OSError, WorkspaceError) as exc:
|
|
142
|
+
blockers.append(str(exc))
|
|
143
|
+
file_count = byte_count = 0
|
|
144
|
+
else:
|
|
145
|
+
file_count = byte_count = 0
|
|
146
|
+
aliases = move_aliases()
|
|
147
|
+
old_ids = {source_id, *(key for key in aliases if canonical_kb_id(key) == source_id)}
|
|
148
|
+
if not source_workspace_id:
|
|
149
|
+
old_ids.add(_legacy_account_id(name))
|
|
150
|
+
assignments = [
|
|
151
|
+
{"workspace_id": row["workspace_id"], "display_name": row["display_name"]}
|
|
152
|
+
for row in get_content_workspace_service()._catalog()
|
|
153
|
+
if any(ref in old_ids for ref in (row.get("resources") or {}).get("knowledge_bases") or [])
|
|
154
|
+
]
|
|
155
|
+
return {
|
|
156
|
+
"source_id": source_id,
|
|
157
|
+
"target_id": target_id,
|
|
158
|
+
"source_workspace_id": source_workspace_id,
|
|
159
|
+
"target_workspace_id": target_workspace_id,
|
|
160
|
+
"name": name,
|
|
161
|
+
"files": file_count,
|
|
162
|
+
"bytes": byte_count,
|
|
163
|
+
"assignments": assignments,
|
|
164
|
+
"blockers": blockers,
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _rewrite_assignments(old_ids: set[str], target_id: str) -> None:
|
|
169
|
+
from deeptutor.services.workspace import get_content_workspace_service
|
|
170
|
+
|
|
171
|
+
service = get_content_workspace_service()
|
|
172
|
+
owner = get_current_user().scope.user_id
|
|
173
|
+
with service._catalog_connection() as conn:
|
|
174
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
175
|
+
for workspace_id, payload in conn.execute("SELECT id,payload FROM workspaces"):
|
|
176
|
+
row = json.loads(payload)
|
|
177
|
+
if row.get("owner_id") != owner:
|
|
178
|
+
continue
|
|
179
|
+
resources = row.get("resources") or {}
|
|
180
|
+
selected = resources.get("knowledge_bases")
|
|
181
|
+
if not isinstance(selected, list) or not any(ref in old_ids for ref in selected):
|
|
182
|
+
continue
|
|
183
|
+
resources["knowledge_bases"] = list(
|
|
184
|
+
dict.fromkeys(target_id if ref in old_ids else ref for ref in selected)
|
|
185
|
+
)
|
|
186
|
+
row["resources"] = resources
|
|
187
|
+
conn.execute(
|
|
188
|
+
"UPDATE workspaces SET payload = ? WHERE id = ?",
|
|
189
|
+
(json.dumps(row, ensure_ascii=False), workspace_id),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _restore_assignments(previous: dict[str, dict]) -> None:
|
|
194
|
+
from deeptutor.services.workspace import get_content_workspace_service
|
|
195
|
+
|
|
196
|
+
service = get_content_workspace_service()
|
|
197
|
+
owner = get_current_user().scope.user_id
|
|
198
|
+
with service._catalog_connection() as conn:
|
|
199
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
200
|
+
for workspace_id, resources in previous.items():
|
|
201
|
+
record = conn.execute(
|
|
202
|
+
"SELECT payload FROM workspaces WHERE id = ?", (workspace_id,)
|
|
203
|
+
).fetchone()
|
|
204
|
+
if record is None:
|
|
205
|
+
continue
|
|
206
|
+
row = json.loads(record[0])
|
|
207
|
+
if row.get("owner_id") != owner:
|
|
208
|
+
continue
|
|
209
|
+
row["resources"] = resources
|
|
210
|
+
conn.execute(
|
|
211
|
+
"UPDATE workspaces SET payload = ? WHERE id = ?",
|
|
212
|
+
(json.dumps(row, ensure_ascii=False), workspace_id),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _copy_kb(source: Path, destination: Path) -> None:
|
|
217
|
+
destination.mkdir(parents=True)
|
|
218
|
+
# _snapshot verifies every byte and backs up SQLite with WAL state.
|
|
219
|
+
_snapshot(source, destination)
|
|
220
|
+
for directory, dirs, _ in os.walk(source, followlinks=False):
|
|
221
|
+
for name in dirs:
|
|
222
|
+
candidate = Path(directory) / name
|
|
223
|
+
if candidate.is_symlink():
|
|
224
|
+
raise WorkspaceError("Knowledge base contains a symbolic link.")
|
|
225
|
+
(destination / candidate.relative_to(source)).mkdir(parents=True, exist_ok=True)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _parse_cache_root(workspace_id: str) -> Path:
|
|
229
|
+
with workspace_context(workspace_id):
|
|
230
|
+
return get_path_service().get_parse_cache_root()
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _rebase_llamaindex_paths(
|
|
234
|
+
stage: Path, source: Path, target: Path, source_cache_root: Path
|
|
235
|
+
) -> None:
|
|
236
|
+
"""Rebase persisted citations and keep indexed parse-cache images with the KB.
|
|
237
|
+
|
|
238
|
+
FAISS vector files are binary and contain no paths. LlamaIndex stores node
|
|
239
|
+
paths in docstore JSON and, for older simple indexes, vector-store JSON.
|
|
240
|
+
BM25 sidecars can contain serialized nodes, so discard them; retrieval
|
|
241
|
+
rebuilds BM25 from the corrected docstore when needed.
|
|
242
|
+
"""
|
|
243
|
+
copied_images: dict[str, str] = {}
|
|
244
|
+
source_prefix = str(source) + os.sep
|
|
245
|
+
cache_root = source_cache_root.resolve()
|
|
246
|
+
|
|
247
|
+
def rewrite_path(value: str, key: str) -> str:
|
|
248
|
+
if value.startswith(source_prefix):
|
|
249
|
+
relative = Path(value).relative_to(source)
|
|
250
|
+
if not (stage / relative).is_file():
|
|
251
|
+
raise WorkspaceError(f"Indexed source is missing: {relative}")
|
|
252
|
+
return str(target / relative)
|
|
253
|
+
if key != "image_path" or not Path(value).is_absolute():
|
|
254
|
+
return value
|
|
255
|
+
if value in copied_images:
|
|
256
|
+
return copied_images[value]
|
|
257
|
+
image = Path(value)
|
|
258
|
+
try:
|
|
259
|
+
resolved = image.resolve(strict=True)
|
|
260
|
+
relative = resolved.relative_to(cache_root)
|
|
261
|
+
except (OSError, ValueError) as exc:
|
|
262
|
+
raise WorkspaceError(
|
|
263
|
+
"An indexed image is outside the movable knowledge base and parse cache."
|
|
264
|
+
) from exc
|
|
265
|
+
path_parts = [image, *list(image.parents)[: len(image.parts) - len(cache_root.parts)]]
|
|
266
|
+
if (
|
|
267
|
+
resolved != image
|
|
268
|
+
or not image.is_file()
|
|
269
|
+
or any(part.is_symlink() for part in path_parts)
|
|
270
|
+
):
|
|
271
|
+
raise WorkspaceError("An indexed image cannot be copied safely.")
|
|
272
|
+
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
273
|
+
asset = Path(".index_assets") / digest / image.name
|
|
274
|
+
_snapshot(image, stage / asset.parent)
|
|
275
|
+
copied_images[value] = str(target / asset)
|
|
276
|
+
return copied_images[value]
|
|
277
|
+
|
|
278
|
+
def rewrite(value: object, key: str = "") -> tuple[object, bool]:
|
|
279
|
+
if isinstance(value, dict):
|
|
280
|
+
changed = False
|
|
281
|
+
result = {}
|
|
282
|
+
for child_key, child_value in value.items():
|
|
283
|
+
result[child_key], child_changed = rewrite(child_value, child_key)
|
|
284
|
+
changed |= child_changed
|
|
285
|
+
return result, changed
|
|
286
|
+
if isinstance(value, list):
|
|
287
|
+
children = [rewrite(child) for child in value]
|
|
288
|
+
return [child for child, _ in children], any(changed for _, changed in children)
|
|
289
|
+
if isinstance(value, str):
|
|
290
|
+
if value.startswith(source_prefix) or key == "image_path":
|
|
291
|
+
updated = rewrite_path(value, key)
|
|
292
|
+
return updated, updated != value
|
|
293
|
+
# Some LlamaIndex stores embed node objects as serialized JSON.
|
|
294
|
+
if key == "__data__" and value.startswith("{") and source_prefix in value:
|
|
295
|
+
parsed = json.loads(value)
|
|
296
|
+
rebased_data, changed = rewrite(parsed)
|
|
297
|
+
if changed:
|
|
298
|
+
return json.dumps(rebased_data, ensure_ascii=False), True
|
|
299
|
+
return value, False
|
|
300
|
+
|
|
301
|
+
for docstore in list(stage.rglob("docstore.json")):
|
|
302
|
+
relative = docstore.relative_to(stage)
|
|
303
|
+
if relative.parts[0] != "llamaindex_storage" and not (
|
|
304
|
+
relative.parts[0].startswith("version-") or relative.parts[0] == "index_versions"
|
|
305
|
+
):
|
|
306
|
+
continue
|
|
307
|
+
storage = docstore.parent
|
|
308
|
+
if not (storage / "index_store.json").is_file():
|
|
309
|
+
continue
|
|
310
|
+
for path in (docstore, *storage.glob("*vector_store.json")):
|
|
311
|
+
# The FAISS store keeps binary bytes under a .json filename.
|
|
312
|
+
if path != docstore:
|
|
313
|
+
with path.open("rb") as handle:
|
|
314
|
+
if handle.read(1) != b"{":
|
|
315
|
+
continue
|
|
316
|
+
try:
|
|
317
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
318
|
+
updated, changed = rewrite(payload)
|
|
319
|
+
except (ValueError, UnicodeError) as exc:
|
|
320
|
+
raise WorkspaceError(f"Cannot rebase LlamaIndex paths in {path.name}.") from exc
|
|
321
|
+
if changed:
|
|
322
|
+
atomic_write_json(path, cast(dict[str, Any], updated))
|
|
323
|
+
shutil.rmtree(storage / "bm25_retriever", ignore_errors=True)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def move_kb(source_id: str, target_workspace_id: str) -> dict:
|
|
327
|
+
"""Publish one verified KB copy and preserve qualified saved references."""
|
|
328
|
+
with data_activity(exclusive=True):
|
|
329
|
+
assert_no_pending_recovery(reject_unreadable=True)
|
|
330
|
+
plan = preview_kb_move(source_id, target_workspace_id)
|
|
331
|
+
if plan["blockers"]:
|
|
332
|
+
raise WorkspaceError(" ".join(plan["blockers"]))
|
|
333
|
+
name = plan["name"]
|
|
334
|
+
source_root, _ = _root(plan["source_workspace_id"])
|
|
335
|
+
target_root, _ = _root(plan["target_workspace_id"])
|
|
336
|
+
source_path = source_root / name
|
|
337
|
+
target_path = target_root / name
|
|
338
|
+
source_before = _config(source_root)
|
|
339
|
+
target_before = _config(target_root)
|
|
340
|
+
aliases_before = move_aliases()
|
|
341
|
+
aliases_path = _move_aliases_path()
|
|
342
|
+
aliases_existed = aliases_path.exists()
|
|
343
|
+
source_config_path = source_root / "kb_config.json"
|
|
344
|
+
target_config_path = target_root / "kb_config.json"
|
|
345
|
+
source_config_existed = source_config_path.exists()
|
|
346
|
+
target_config_existed = target_config_path.exists()
|
|
347
|
+
old_ids = {
|
|
348
|
+
plan["source_id"],
|
|
349
|
+
*(key for key in aliases_before if canonical_kb_id(key) == plan["source_id"]),
|
|
350
|
+
}
|
|
351
|
+
if not plan["source_workspace_id"]:
|
|
352
|
+
old_ids.add(_legacy_account_id(name))
|
|
353
|
+
stage = target_root / f".kb-move-{uuid.uuid4().hex}"
|
|
354
|
+
published = False
|
|
355
|
+
source_hidden = None
|
|
356
|
+
from deeptutor.services.workspace import get_content_workspace_service
|
|
357
|
+
|
|
358
|
+
catalog = get_content_workspace_service()
|
|
359
|
+
assignment_before = {
|
|
360
|
+
row["workspace_id"]: deepcopy(row.get("resources") or {})
|
|
361
|
+
for row in catalog._catalog()
|
|
362
|
+
if any(
|
|
363
|
+
ref in old_ids for ref in (row.get("resources") or {}).get("knowledge_bases") or []
|
|
364
|
+
)
|
|
365
|
+
}
|
|
366
|
+
try:
|
|
367
|
+
if source_path.is_dir():
|
|
368
|
+
_copy_kb(source_path, stage)
|
|
369
|
+
entry = source_before["knowledge_bases"][name]
|
|
370
|
+
provider = entry.get("rag_provider")
|
|
371
|
+
if provider in {None, "", "llamaindex"}:
|
|
372
|
+
_rebase_llamaindex_paths(
|
|
373
|
+
stage,
|
|
374
|
+
source_path,
|
|
375
|
+
target_path,
|
|
376
|
+
_parse_cache_root(plan["source_workspace_id"]),
|
|
377
|
+
)
|
|
378
|
+
if (
|
|
379
|
+
not is_connected_kb(entry)
|
|
380
|
+
and has_ready_provider_index(source_path, provider)
|
|
381
|
+
and not has_ready_provider_index(stage, provider)
|
|
382
|
+
):
|
|
383
|
+
raise WorkspaceError(
|
|
384
|
+
"The copied index did not pass its provider readiness check."
|
|
385
|
+
)
|
|
386
|
+
if target_path.exists():
|
|
387
|
+
raise WorkspaceError(
|
|
388
|
+
f"Destination already contains a knowledge base named '{name}'."
|
|
389
|
+
)
|
|
390
|
+
stage.rename(target_path)
|
|
391
|
+
published = True
|
|
392
|
+
target_next = deepcopy(target_before)
|
|
393
|
+
target_next["knowledge_bases"][name] = deepcopy(source_before["knowledge_bases"][name])
|
|
394
|
+
source_next = deepcopy(source_before)
|
|
395
|
+
del source_next["knowledge_bases"][name]
|
|
396
|
+
source_defaults = source_next.get("defaults") or {}
|
|
397
|
+
if source_defaults.get("default_kb") == name:
|
|
398
|
+
source_defaults["default_kb"] = None
|
|
399
|
+
source_next["defaults"] = source_defaults
|
|
400
|
+
target_defaults = target_next.setdefault("defaults", {})
|
|
401
|
+
if not target_defaults.get("default_kb"):
|
|
402
|
+
target_defaults["default_kb"] = name
|
|
403
|
+
atomic_write_json(target_config_path, target_next)
|
|
404
|
+
_rewrite_assignments(old_ids, plan["target_id"])
|
|
405
|
+
aliases_next = {
|
|
406
|
+
key: (plan["target_id"] if value in old_ids else value)
|
|
407
|
+
for key, value in aliases_before.items()
|
|
408
|
+
}
|
|
409
|
+
aliases_next[plan["source_id"]] = plan["target_id"]
|
|
410
|
+
if not plan["source_workspace_id"]:
|
|
411
|
+
aliases_next[_legacy_account_id(name)] = plan["target_id"]
|
|
412
|
+
atomic_write_json(aliases_path, aliases_next)
|
|
413
|
+
atomic_write_json(source_config_path, source_next)
|
|
414
|
+
if source_path.is_dir():
|
|
415
|
+
source_hidden = source_root / f".kb-moved-{uuid.uuid4().hex}"
|
|
416
|
+
source_path.rename(source_hidden)
|
|
417
|
+
except BaseException:
|
|
418
|
+
if source_hidden is not None and source_hidden.exists():
|
|
419
|
+
source_hidden.rename(source_path)
|
|
420
|
+
if source_config_existed:
|
|
421
|
+
atomic_write_json(source_config_path, source_before)
|
|
422
|
+
else:
|
|
423
|
+
source_config_path.unlink(missing_ok=True)
|
|
424
|
+
if target_config_existed:
|
|
425
|
+
atomic_write_json(target_config_path, target_before)
|
|
426
|
+
else:
|
|
427
|
+
target_config_path.unlink(missing_ok=True)
|
|
428
|
+
_restore_assignments(assignment_before)
|
|
429
|
+
if aliases_existed:
|
|
430
|
+
atomic_write_json(aliases_path, aliases_before)
|
|
431
|
+
else:
|
|
432
|
+
aliases_path.unlink(missing_ok=True)
|
|
433
|
+
if published:
|
|
434
|
+
shutil.rmtree(target_path)
|
|
435
|
+
if stage.exists():
|
|
436
|
+
shutil.rmtree(stage)
|
|
437
|
+
raise
|
|
438
|
+
if source_hidden is not None:
|
|
439
|
+
try:
|
|
440
|
+
shutil.rmtree(source_hidden)
|
|
441
|
+
except OSError:
|
|
442
|
+
# The hidden original remains available for manual recovery;
|
|
443
|
+
# it does not appear as an active knowledge base.
|
|
444
|
+
plan["cleanup_path"] = str(source_hidden)
|
|
445
|
+
plan["status"] = "moved"
|
|
446
|
+
return plan
|