kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/commands/__init__.py
ADDED
ytk/commands/merge.py
ADDED
|
@@ -0,0 +1,600 @@
|
|
|
1
|
+
"""T41.1 — cross-source merge for v2 knowledge bundles.
|
|
2
|
+
|
|
3
|
+
The merge order is fixed by the plan (§5.5 T41.1): **validate each input
|
|
4
|
+
schema → merge sources, contents and the evidence registry → merge knowledge
|
|
5
|
+
atoms last**. Everything the merged artifact cites is carried with it:
|
|
6
|
+
|
|
7
|
+
* evidence is deduplicated by ``(source_id, source_revision, evidence_id)``;
|
|
8
|
+
the same evidence id reused by another source/revision is namespaced and the
|
|
9
|
+
atom references are rewritten, never silently overwritten ("同一局部 ID /
|
|
10
|
+
同来源不同修订不能互相覆盖");
|
|
11
|
+
* a genuinely conflicting entry (same identity, different payload) is a hard
|
|
12
|
+
:class:`MergeConflictError`;
|
|
13
|
+
* atoms merge only when type *and* normalized content *and* the negation /
|
|
14
|
+
condition signature all match — opposite conclusions stay separate;
|
|
15
|
+
* confidence is never raised because more sources repeat a claim;
|
|
16
|
+
* referenced local assets are copied into ``<output_dir>/assets/<source>/`` and
|
|
17
|
+
re-pointed with package-relative paths, so the merged product still resolves
|
|
18
|
+
after the original input directories are moved.
|
|
19
|
+
|
|
20
|
+
Legacy v1 artifacts keep working:
|
|
21
|
+
|
|
22
|
+
* a *minimal* v1 ``knowledge.json`` (``video_id`` + ``knowledge_atoms`` only)
|
|
23
|
+
merges through the historical dictionary path, byte-compatible with the old
|
|
24
|
+
CLI output;
|
|
25
|
+
* a full v1 ``ExtractionBundle`` is migrated with
|
|
26
|
+
:func:`ytk.models.migrations.migrate_extraction_bundle_v1` and merges as v2.
|
|
27
|
+
|
|
28
|
+
Anything else (bad JSON, unsupported ``schema_version``, non-extraction JSON,
|
|
29
|
+
missing/conflicting evidence, unreachable assets) is rejected with a clear
|
|
30
|
+
:class:`MergeInputError` instead of producing a half-resolvable artifact.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import hashlib
|
|
36
|
+
import json
|
|
37
|
+
import re
|
|
38
|
+
import shutil
|
|
39
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Any, Literal
|
|
43
|
+
|
|
44
|
+
from pydantic import ValidationError
|
|
45
|
+
|
|
46
|
+
from ytk.dedup.service import DeduplicationService
|
|
47
|
+
from ytk.errors import UserFacingError
|
|
48
|
+
from ytk.models.content import (
|
|
49
|
+
ContentBundle,
|
|
50
|
+
EvidenceRefV2,
|
|
51
|
+
ExtractionBundleV2,
|
|
52
|
+
KnowledgeAtomV2,
|
|
53
|
+
MediaAsset,
|
|
54
|
+
TopicGroup,
|
|
55
|
+
locator_key,
|
|
56
|
+
)
|
|
57
|
+
from ytk.models.knowledge import ExtractionBundle
|
|
58
|
+
from ytk.models.migrations import migrate_extraction_bundle_v1
|
|
59
|
+
from ytk.models.source import ContentRef
|
|
60
|
+
|
|
61
|
+
InputKind = Literal["v2", "v1", "legacy", "unknown"]
|
|
62
|
+
|
|
63
|
+
_V1_REQUIRED_KEYS = {"metadata", "transcript_source", "topic_segments", "skill_spec"}
|
|
64
|
+
_SAFE_SLUG_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class MergeInputError(UserFacingError):
|
|
68
|
+
"""An input artifact cannot be merged (bad schema, missing or fake evidence)."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class MergeConflictError(MergeInputError):
|
|
72
|
+
"""Two inputs make incompatible claims about the same identity."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class _Artifact:
|
|
77
|
+
path: Path
|
|
78
|
+
root: Path
|
|
79
|
+
bundle: ExtractionBundleV2 | None = None
|
|
80
|
+
payload: dict[str, Any] = field(default_factory=dict)
|
|
81
|
+
kind: InputKind = "unknown"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
# Loading / schema validation
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def load_knowledge(path: Path) -> dict[str, Any]:
|
|
90
|
+
"""Parse one JSON artifact; raise :class:`MergeInputError` when unreadable."""
|
|
91
|
+
path = Path(path)
|
|
92
|
+
try:
|
|
93
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
94
|
+
except OSError as exc:
|
|
95
|
+
raise MergeInputError(f"cannot read merge input {path}: {exc}") from exc
|
|
96
|
+
except (json.JSONDecodeError, UnicodeError) as exc:
|
|
97
|
+
raise MergeInputError(f"merge input {path} is not valid JSON: {exc}") from exc
|
|
98
|
+
if not isinstance(payload, dict):
|
|
99
|
+
raise MergeInputError(f"merge input {path} is not a JSON object")
|
|
100
|
+
return payload
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def classify_payload(path: Path, payload: Mapping[str, Any]) -> InputKind:
|
|
104
|
+
"""Classify a parsed artifact as v2 / full v1 / legacy minimal / unknown."""
|
|
105
|
+
if "schema_version" in payload:
|
|
106
|
+
version = payload["schema_version"]
|
|
107
|
+
if version != 2:
|
|
108
|
+
raise MergeInputError(
|
|
109
|
+
f"merge input {path} declares unsupported schema_version={version!r}"
|
|
110
|
+
)
|
|
111
|
+
return "v2"
|
|
112
|
+
if "knowledge_atoms" in payload and "video_id" in payload:
|
|
113
|
+
if _V1_REQUIRED_KEYS & set(payload):
|
|
114
|
+
return "v1"
|
|
115
|
+
return "legacy"
|
|
116
|
+
return "unknown"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _format_validation_error(exc: ValidationError) -> str:
|
|
120
|
+
return "; ".join(
|
|
121
|
+
f"{'.'.join(str(part) for part in error.get('loc', ())) or '<root>'}: "
|
|
122
|
+
f"{error.get('msg', 'invalid value')}"
|
|
123
|
+
for error in exc.errors()[:5]
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def load_bundle(path: Path, payload: Mapping[str, Any] | None = None) -> ExtractionBundleV2:
|
|
128
|
+
"""Validate one v1/v2 artifact and return a v2 bundle (strict, never guesses)."""
|
|
129
|
+
path = Path(path)
|
|
130
|
+
data = dict(payload) if payload is not None else load_knowledge(path)
|
|
131
|
+
kind = classify_payload(path, data)
|
|
132
|
+
if kind == "legacy":
|
|
133
|
+
raise MergeInputError(
|
|
134
|
+
f"merge input {path} is a legacy minimal knowledge.json and cannot be "
|
|
135
|
+
"combined with extraction bundles"
|
|
136
|
+
)
|
|
137
|
+
if kind == "unknown":
|
|
138
|
+
raise MergeInputError(
|
|
139
|
+
f"merge input {path} is not a YTK knowledge extraction artifact"
|
|
140
|
+
)
|
|
141
|
+
if kind == "v2":
|
|
142
|
+
try:
|
|
143
|
+
return ExtractionBundleV2.model_validate(data)
|
|
144
|
+
except ValidationError as exc:
|
|
145
|
+
raise MergeInputError(
|
|
146
|
+
f"merge input {path} is not a valid v2 bundle: {_format_validation_error(exc)}"
|
|
147
|
+
) from exc
|
|
148
|
+
try:
|
|
149
|
+
v1 = ExtractionBundle.model_validate(data)
|
|
150
|
+
except ValidationError as exc:
|
|
151
|
+
raise MergeInputError(
|
|
152
|
+
f"merge input {path} is not a valid v1 knowledge bundle: "
|
|
153
|
+
f"{_format_validation_error(exc)}"
|
|
154
|
+
) from exc
|
|
155
|
+
return migrate_extraction_bundle_v1(v1)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
# Assets
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _slug(value: str) -> str:
|
|
164
|
+
slug = _SAFE_SLUG_RE.sub("-", value).strip("-")
|
|
165
|
+
return slug or "source"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _asset_candidates(asset: MediaAsset, artifacts: Sequence[_Artifact], content_key: str) -> list[Path]:
|
|
169
|
+
candidates: list[Path] = []
|
|
170
|
+
for artifact in artifacts:
|
|
171
|
+
if artifact.bundle is None:
|
|
172
|
+
continue
|
|
173
|
+
if content_key not in {content.content_key for content in artifact.bundle.contents}:
|
|
174
|
+
continue
|
|
175
|
+
for candidate in (
|
|
176
|
+
artifact.path.parent / asset.relative_path,
|
|
177
|
+
artifact.root / asset.relative_path,
|
|
178
|
+
):
|
|
179
|
+
if candidate.resolve().is_relative_to(artifact.root.resolve()):
|
|
180
|
+
candidates.append(candidate)
|
|
181
|
+
seen: set[Path] = set()
|
|
182
|
+
unique: list[Path] = []
|
|
183
|
+
for candidate in candidates:
|
|
184
|
+
if candidate not in seen:
|
|
185
|
+
seen.add(candidate)
|
|
186
|
+
unique.append(candidate)
|
|
187
|
+
return unique
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _copy_assets(
|
|
191
|
+
bundle: ExtractionBundleV2,
|
|
192
|
+
artifacts: Sequence[_Artifact],
|
|
193
|
+
output_dir: Path,
|
|
194
|
+
) -> list[ContentBundle]:
|
|
195
|
+
"""Copy every referenced asset into the package and rewrite relative paths."""
|
|
196
|
+
relocated: list[ContentBundle] = []
|
|
197
|
+
for content in bundle.contents:
|
|
198
|
+
new_assets: list[MediaAsset] = []
|
|
199
|
+
for asset in content.assets:
|
|
200
|
+
candidates = _asset_candidates(asset, artifacts, content.content_key)
|
|
201
|
+
available = [path for path in candidates if path.is_file()]
|
|
202
|
+
source_path = next((path for path in available if (
|
|
203
|
+
path.stat().st_size == asset.byte_size
|
|
204
|
+
and hashlib.sha256(path.read_bytes()).hexdigest() == asset.sha256
|
|
205
|
+
)), None)
|
|
206
|
+
if source_path is None:
|
|
207
|
+
if available:
|
|
208
|
+
raise MergeInputError(
|
|
209
|
+
f"asset {asset.asset_id!r} of {content.content_key} has a mismatching digest or size"
|
|
210
|
+
)
|
|
211
|
+
raise MergeInputError(
|
|
212
|
+
f"asset {asset.asset_id!r} of {asset.source_id} was not found "
|
|
213
|
+
f"(expected at {asset.relative_path}); the merged output would not "
|
|
214
|
+
"resolve after moving the input directories"
|
|
215
|
+
)
|
|
216
|
+
target_dir = output_dir / "assets" / _slug(asset.source_id)
|
|
217
|
+
if not target_dir.resolve().is_relative_to(output_dir.resolve()):
|
|
218
|
+
raise MergeInputError("asset destination escapes the merged output package")
|
|
219
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
220
|
+
target = target_dir / source_path.name
|
|
221
|
+
if target.is_symlink():
|
|
222
|
+
raise MergeInputError("asset destination must not be a symbolic link")
|
|
223
|
+
if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest() != asset.sha256:
|
|
224
|
+
target = target.with_name(f"{target.stem}-{asset.sha256[:20]}{target.suffix}")
|
|
225
|
+
if target.exists():
|
|
226
|
+
if target.read_bytes() != source_path.read_bytes():
|
|
227
|
+
raise MergeConflictError(
|
|
228
|
+
f"two different assets map to {target} for source {asset.source_id}"
|
|
229
|
+
)
|
|
230
|
+
else:
|
|
231
|
+
shutil.copyfile(source_path, target)
|
|
232
|
+
relative = target.relative_to(output_dir).as_posix()
|
|
233
|
+
new_assets.append(asset.model_copy(update={"relative_path": relative}))
|
|
234
|
+
relocated.append(content.model_copy(update={"assets": new_assets}))
|
|
235
|
+
return relocated
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# ---------------------------------------------------------------------------
|
|
239
|
+
# Core merge
|
|
240
|
+
# ---------------------------------------------------------------------------
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _merge_sources(
|
|
244
|
+
collected: dict[str, ContentRef], incoming_sequence: Sequence[ContentRef]
|
|
245
|
+
) -> dict[str, ContentRef]:
|
|
246
|
+
for ref in incoming_sequence:
|
|
247
|
+
current = collected.get(ref.source_id)
|
|
248
|
+
if current is None:
|
|
249
|
+
collected[ref.source_id] = ref
|
|
250
|
+
continue
|
|
251
|
+
if current.platform != ref.platform or current.content_type != ref.content_type:
|
|
252
|
+
raise MergeConflictError(
|
|
253
|
+
f"source {ref.source_id!r} is declared with conflicting platform/content "
|
|
254
|
+
f"type ({current.platform}/{current.content_type} vs "
|
|
255
|
+
f"{ref.platform}/{ref.content_type})"
|
|
256
|
+
)
|
|
257
|
+
aliases = list(current.alias_urls)
|
|
258
|
+
for alias in ref.alias_urls:
|
|
259
|
+
if alias not in aliases:
|
|
260
|
+
aliases.append(alias)
|
|
261
|
+
display = {**dict(ref.display), **dict(current.display)}
|
|
262
|
+
collected[ref.source_id] = current.model_copy(
|
|
263
|
+
update={"alias_urls": aliases, "display": display}
|
|
264
|
+
)
|
|
265
|
+
return collected
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _same_content(left: ContentBundle, right: ContentBundle) -> bool:
|
|
269
|
+
left_blocks = {block.block_id: block.model_dump(mode="json") for block in left.blocks}
|
|
270
|
+
right_blocks = {block.block_id: block.model_dump(mode="json") for block in right.blocks}
|
|
271
|
+
return (
|
|
272
|
+
left_blocks == right_blocks
|
|
273
|
+
and [asset.model_dump(exclude={"relative_path"}) for asset in left.assets]
|
|
274
|
+
== [asset.model_dump(exclude={"relative_path"}) for asset in right.assets]
|
|
275
|
+
and left.completeness.status == right.completeness.status
|
|
276
|
+
and left.source.source_id == right.source.source_id
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _claim_evidence_id(
|
|
281
|
+
evidence: EvidenceRefV2, used: dict[str, tuple[str, str, str]]
|
|
282
|
+
) -> str:
|
|
283
|
+
"""Return the merged id for ``evidence`` (namespace on a real collision)."""
|
|
284
|
+
identity = (evidence.source_id, evidence.source_revision, evidence.evidence_id)
|
|
285
|
+
current = used.get(evidence.evidence_id)
|
|
286
|
+
if current is None or current == identity:
|
|
287
|
+
used[evidence.evidence_id] = identity
|
|
288
|
+
return evidence.evidence_id
|
|
289
|
+
candidate = f"{evidence.source_id}@{evidence.source_revision}#{evidence.evidence_id}"
|
|
290
|
+
counter = 2
|
|
291
|
+
while candidate in used and used[candidate] != identity:
|
|
292
|
+
candidate = (
|
|
293
|
+
f"{evidence.source_id}@{evidence.source_revision}#{evidence.evidence_id}~{counter}"
|
|
294
|
+
)
|
|
295
|
+
counter += 1
|
|
296
|
+
used[candidate] = identity
|
|
297
|
+
return candidate
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _evidence_fingerprint(evidence: EvidenceRefV2) -> tuple[str, ...]:
|
|
301
|
+
return (
|
|
302
|
+
evidence.source_id,
|
|
303
|
+
evidence.source_revision,
|
|
304
|
+
evidence.block_id,
|
|
305
|
+
evidence.quote,
|
|
306
|
+
locator_key(evidence.locator),
|
|
307
|
+
evidence.processor,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _unique_topic_id(topic: TopicGroup, used_topic_ids: set[str]) -> str:
|
|
312
|
+
if topic.topic_id not in used_topic_ids:
|
|
313
|
+
used_topic_ids.add(topic.topic_id)
|
|
314
|
+
return topic.topic_id
|
|
315
|
+
counter = 2
|
|
316
|
+
candidate = f"{topic.topic_id}~{counter}"
|
|
317
|
+
while candidate in used_topic_ids:
|
|
318
|
+
counter += 1
|
|
319
|
+
candidate = f"{topic.topic_id}~{counter}"
|
|
320
|
+
used_topic_ids.add(candidate)
|
|
321
|
+
return candidate
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def merge_bundles(
|
|
325
|
+
bundles: Sequence[ExtractionBundleV2],
|
|
326
|
+
*,
|
|
327
|
+
artifacts: Sequence[_Artifact] = (),
|
|
328
|
+
output_dir: Path | None = None,
|
|
329
|
+
) -> ExtractionBundleV2:
|
|
330
|
+
"""Merge already-validated v2 bundles into one self-contained bundle.
|
|
331
|
+
|
|
332
|
+
``output_dir`` (when given) receives the copied assets; every asset path in
|
|
333
|
+
the result is relative to it. The function is deterministic for a given
|
|
334
|
+
input order — inputs are expected in sorted artifact order.
|
|
335
|
+
"""
|
|
336
|
+
source_map: dict[str, ContentRef] = {}
|
|
337
|
+
content_map: dict[tuple[str, str], ContentBundle] = {}
|
|
338
|
+
evidence_index: dict[str, EvidenceRefV2] = {}
|
|
339
|
+
evidence_used: dict[str, tuple[str, str, str]] = {}
|
|
340
|
+
evidence_fingerprints: dict[str, tuple[str, ...]] = {}
|
|
341
|
+
topics: list[TopicGroup] = []
|
|
342
|
+
topic_ids: set[str] = set()
|
|
343
|
+
atoms: list[KnowledgeAtomV2] = []
|
|
344
|
+
|
|
345
|
+
for bundle in bundles:
|
|
346
|
+
_merge_sources(source_map, bundle.sources)
|
|
347
|
+
for content in bundle.contents:
|
|
348
|
+
key = (content.source.source_id, content.source_revision)
|
|
349
|
+
existing = content_map.get(key)
|
|
350
|
+
if existing is None:
|
|
351
|
+
content_map[key] = content
|
|
352
|
+
elif not _same_content(existing, content):
|
|
353
|
+
raise MergeConflictError(
|
|
354
|
+
f"conflicting content for revision {content.content_key}: two inputs "
|
|
355
|
+
"carry different blocks for the same source revision"
|
|
356
|
+
)
|
|
357
|
+
evidence_mapping: dict[str, str] = {}
|
|
358
|
+
for evidence_id, evidence in bundle.evidence_index.items():
|
|
359
|
+
new_id = _claim_evidence_id(evidence, evidence_used)
|
|
360
|
+
fingerprint = _evidence_fingerprint(evidence)
|
|
361
|
+
previous = evidence_fingerprints.get(new_id)
|
|
362
|
+
if previous is not None and previous != fingerprint:
|
|
363
|
+
raise MergeConflictError(
|
|
364
|
+
f"evidence {new_id!r} is registered twice with different content "
|
|
365
|
+
"(conflicting evidence id)"
|
|
366
|
+
)
|
|
367
|
+
evidence_mapping[evidence_id] = new_id
|
|
368
|
+
evidence_index[new_id] = evidence.model_copy(update={"evidence_id": new_id})
|
|
369
|
+
evidence_fingerprints[new_id] = fingerprint
|
|
370
|
+
for topic in bundle.topics:
|
|
371
|
+
new_topic_id = _unique_topic_id(topic, topic_ids)
|
|
372
|
+
topics.append(
|
|
373
|
+
topic.model_copy(update={"topic_id": new_topic_id})
|
|
374
|
+
if new_topic_id != topic.topic_id
|
|
375
|
+
else topic
|
|
376
|
+
)
|
|
377
|
+
for atom in bundle.knowledge:
|
|
378
|
+
mapped_ids: list[str] = []
|
|
379
|
+
for evidence_id in atom.evidence_ids:
|
|
380
|
+
if evidence_id not in evidence_mapping:
|
|
381
|
+
raise MergeInputError(
|
|
382
|
+
f"atom {atom.atom_id!r} references evidence {evidence_id!r} that is "
|
|
383
|
+
"missing from its bundle's evidence_index"
|
|
384
|
+
)
|
|
385
|
+
mapped_ids.append(evidence_mapping[evidence_id])
|
|
386
|
+
atoms.append(atom.model_copy(update={"evidence_ids": mapped_ids}))
|
|
387
|
+
|
|
388
|
+
if output_dir is not None:
|
|
389
|
+
assembled = ExtractionBundleV2.model_construct(
|
|
390
|
+
sources=list(source_map.values()),
|
|
391
|
+
contents=list(content_map.values()),
|
|
392
|
+
evidence_index=evidence_index,
|
|
393
|
+
topics=topics,
|
|
394
|
+
knowledge=atoms,
|
|
395
|
+
metadata={},
|
|
396
|
+
skill_spec=None,
|
|
397
|
+
)
|
|
398
|
+
relocated = _copy_assets(assembled, artifacts, Path(output_dir))
|
|
399
|
+
content_map = {
|
|
400
|
+
(content.source.source_id, content.source_revision): content
|
|
401
|
+
for content in relocated
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
deduped = DeduplicationService().merge_v2(atoms)
|
|
405
|
+
merged_metadata: dict[str, Any] = {
|
|
406
|
+
"title": f"Merged knowledge from {len(source_map)} sources",
|
|
407
|
+
"source_ids": [ref.source_id for ref in source_map.values()],
|
|
408
|
+
"merged_schema": "v2",
|
|
409
|
+
}
|
|
410
|
+
try:
|
|
411
|
+
return ExtractionBundleV2(
|
|
412
|
+
sources=list(source_map.values()),
|
|
413
|
+
contents=list(content_map.values()),
|
|
414
|
+
evidence_index=evidence_index,
|
|
415
|
+
topics=topics,
|
|
416
|
+
knowledge=deduped,
|
|
417
|
+
metadata=merged_metadata,
|
|
418
|
+
skill_spec=None,
|
|
419
|
+
)
|
|
420
|
+
except ValidationError as exc:
|
|
421
|
+
raise MergeConflictError(
|
|
422
|
+
"merged bundle failed reference validation: " + _format_validation_error(exc)
|
|
423
|
+
) from exc
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def merge_bundle_files(
|
|
427
|
+
paths: Sequence[Path],
|
|
428
|
+
*,
|
|
429
|
+
artifact_root: Path | None = None,
|
|
430
|
+
output_dir: Path | None = None,
|
|
431
|
+
) -> ExtractionBundleV2:
|
|
432
|
+
"""Validate and merge the given v1/v2 artifacts (no legacy minimal inputs)."""
|
|
433
|
+
artifacts: list[_Artifact] = []
|
|
434
|
+
bundles: list[ExtractionBundleV2] = []
|
|
435
|
+
for path in paths:
|
|
436
|
+
path = Path(path)
|
|
437
|
+
payload = load_knowledge(path)
|
|
438
|
+
kind = classify_payload(path, payload)
|
|
439
|
+
if kind == "legacy":
|
|
440
|
+
raise MergeInputError(
|
|
441
|
+
f"merge input {path} is a legacy minimal knowledge.json and cannot be "
|
|
442
|
+
"combined with extraction bundles"
|
|
443
|
+
)
|
|
444
|
+
bundle = load_bundle(path, payload)
|
|
445
|
+
artifacts.append(
|
|
446
|
+
_Artifact(
|
|
447
|
+
path=path,
|
|
448
|
+
root=Path(artifact_root) if artifact_root else path.parent,
|
|
449
|
+
bundle=bundle,
|
|
450
|
+
payload=payload,
|
|
451
|
+
kind=kind,
|
|
452
|
+
)
|
|
453
|
+
)
|
|
454
|
+
bundles.append(bundle)
|
|
455
|
+
return merge_bundles(bundles, artifacts=artifacts, output_dir=output_dir)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def merge_bundle_directories(input_dir: Path, output_dir: Path) -> ExtractionBundleV2:
|
|
459
|
+
"""Merge every ``knowledge.json`` under ``input_dir`` into ``output_dir``."""
|
|
460
|
+
input_dir = Path(input_dir)
|
|
461
|
+
paths = sorted(input_dir.rglob("knowledge.json"))
|
|
462
|
+
if not paths:
|
|
463
|
+
raise MergeInputError(f"no knowledge.json found under {input_dir}")
|
|
464
|
+
return merge_bundle_files(paths, artifact_root=input_dir, output_dir=output_dir)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
# ---------------------------------------------------------------------------
|
|
468
|
+
# Legacy (minimal v1) merge path — byte-compatible with the old CLI output
|
|
469
|
+
# ---------------------------------------------------------------------------
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _merge_legacy_artifacts(
|
|
473
|
+
artifacts: Sequence[tuple[Path, dict[str, Any]]], output_path: Path
|
|
474
|
+
) -> dict[str, Any]:
|
|
475
|
+
all_atoms: list[dict[str, Any]] = []
|
|
476
|
+
source_videos: list[str] = []
|
|
477
|
+
for path, data in artifacts:
|
|
478
|
+
video_id = data.get("video_id", path.parent.name)
|
|
479
|
+
source_videos.append(video_id)
|
|
480
|
+
for atom in data.get("knowledge_atoms", []):
|
|
481
|
+
if not isinstance(atom, dict):
|
|
482
|
+
raise MergeInputError(f"merge input {path} has a non-object knowledge atom")
|
|
483
|
+
atom = dict(atom)
|
|
484
|
+
atom["video_id"] = video_id
|
|
485
|
+
all_atoms.append(atom)
|
|
486
|
+
|
|
487
|
+
merged_atoms = deduplicate_atoms(all_atoms)
|
|
488
|
+
result: dict[str, Any] = {
|
|
489
|
+
"title": f"Merged knowledge from {len(source_videos)} videos",
|
|
490
|
+
"source_count": len(source_videos),
|
|
491
|
+
"source_videos": source_videos,
|
|
492
|
+
"atoms": merged_atoms,
|
|
493
|
+
}
|
|
494
|
+
output_path = Path(output_path)
|
|
495
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
496
|
+
output_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
497
|
+
return result
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _write_bundle(bundle: ExtractionBundleV2, output_path: Path) -> None:
|
|
501
|
+
output_path = Path(output_path)
|
|
502
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
503
|
+
output_path.write_text(
|
|
504
|
+
json.dumps(bundle.model_dump(mode="json"), indent=2, ensure_ascii=False) + "\n",
|
|
505
|
+
encoding="utf-8",
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def merge_knowledge_directories(input_dir: Path, output_path: Path) -> dict[str, Any]:
|
|
510
|
+
"""Merge every ``knowledge.json`` under ``input_dir`` into ``output_path``.
|
|
511
|
+
|
|
512
|
+
Dispatches on the input schema: all-legacy minimal v1 inputs keep the
|
|
513
|
+
historical dictionary output; any full v1/v2 artifact produces a v2
|
|
514
|
+
:class:`ExtractionBundleV2` JSON with self-contained relative assets.
|
|
515
|
+
"""
|
|
516
|
+
input_dir = Path(input_dir)
|
|
517
|
+
output_path = Path(output_path)
|
|
518
|
+
paths = sorted(input_dir.rglob("knowledge.json"))
|
|
519
|
+
if not paths:
|
|
520
|
+
raise MergeInputError(f"no knowledge.json found under {input_dir}")
|
|
521
|
+
|
|
522
|
+
payloads: list[tuple[Path, dict[str, Any], InputKind]] = []
|
|
523
|
+
for path in paths:
|
|
524
|
+
payload = load_knowledge(path)
|
|
525
|
+
payloads.append((path, payload, classify_payload(path, payload)))
|
|
526
|
+
|
|
527
|
+
if all(kind == "legacy" for _path, _payload, kind in payloads):
|
|
528
|
+
return _merge_legacy_artifacts(
|
|
529
|
+
[(path, payload) for path, payload, _kind in payloads], output_path
|
|
530
|
+
)
|
|
531
|
+
|
|
532
|
+
legacy_paths = [str(path) for path, _payload, kind in payloads if kind == "legacy"]
|
|
533
|
+
if legacy_paths:
|
|
534
|
+
raise MergeInputError(
|
|
535
|
+
"cannot mix legacy minimal knowledge.json with extraction bundles: "
|
|
536
|
+
+ ", ".join(legacy_paths)
|
|
537
|
+
)
|
|
538
|
+
unknown_paths = [str(path) for path, _payload, kind in payloads if kind == "unknown"]
|
|
539
|
+
if unknown_paths:
|
|
540
|
+
raise MergeInputError(
|
|
541
|
+
"these merge inputs are not YTK knowledge artifacts: " + ", ".join(unknown_paths)
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
bundle = merge_bundle_files(paths, artifact_root=input_dir, output_dir=output_path.parent)
|
|
545
|
+
_write_bundle(bundle, output_path)
|
|
546
|
+
return {
|
|
547
|
+
"title": f"Merged knowledge from {len(bundle.sources)} sources",
|
|
548
|
+
"source_count": len(bundle.sources),
|
|
549
|
+
"sources": [ref.public_dict() for ref in bundle.sources],
|
|
550
|
+
"schema_version": 2,
|
|
551
|
+
"atoms": [atom.model_dump(mode="json") for atom in bundle.knowledge],
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
# ---------------------------------------------------------------------------
|
|
556
|
+
# Legacy dictionary-level helpers (public API kept for existing callers)
|
|
557
|
+
# ---------------------------------------------------------------------------
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def deduplicate_atoms(atoms: list[dict]) -> list[dict]:
|
|
561
|
+
seen: dict[tuple[str, str], dict] = {}
|
|
562
|
+
for atom in atoms:
|
|
563
|
+
key = (atom.get("atom_type", ""), atom.get("content", "").casefold())
|
|
564
|
+
if key in seen:
|
|
565
|
+
existing = seen[key]
|
|
566
|
+
sources = existing.setdefault("source_videos", [])
|
|
567
|
+
vid = atom.get("video_id", "")
|
|
568
|
+
if vid and vid not in sources:
|
|
569
|
+
sources.append(vid)
|
|
570
|
+
if existing.get("video_id") and existing["video_id"] not in sources:
|
|
571
|
+
sources.insert(0, existing["video_id"])
|
|
572
|
+
if atom.get("confidence", 0) > existing.get("confidence", 0):
|
|
573
|
+
existing["confidence"] = atom["confidence"]
|
|
574
|
+
else:
|
|
575
|
+
atom_copy = dict(atom)
|
|
576
|
+
vid = atom_copy.pop("video_id", "")
|
|
577
|
+
if vid:
|
|
578
|
+
atom_copy["source_videos"] = [vid]
|
|
579
|
+
seen[key] = atom_copy
|
|
580
|
+
return list(seen.values())
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def merge_atom_refs(atoms: Iterable[KnowledgeAtomV2]) -> list[KnowledgeAtomV2]:
|
|
584
|
+
"""Merge v2 atoms without a bundle (thin wrapper over the dedup service)."""
|
|
585
|
+
return DeduplicationService().merge_v2(list(atoms))
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
__all__ = [
|
|
589
|
+
"MergeConflictError",
|
|
590
|
+
"MergeInputError",
|
|
591
|
+
"classify_payload",
|
|
592
|
+
"deduplicate_atoms",
|
|
593
|
+
"load_bundle",
|
|
594
|
+
"load_knowledge",
|
|
595
|
+
"merge_atom_refs",
|
|
596
|
+
"merge_bundle_directories",
|
|
597
|
+
"merge_bundle_files",
|
|
598
|
+
"merge_bundles",
|
|
599
|
+
"merge_knowledge_directories",
|
|
600
|
+
]
|